openapi: Remove yamole.

As explained in the previous commit, yamole preprocessed allOf with an
algorithm that is not standards compliant.  We replicate that
algorithm, but importantly, we only use it for our own code and not
for building the openapi_core RequestValidator.

This improves the time taken by OpenAPISpec().check_reload() from
1.69s to 0.53s, nearly all of which is inside
openapi_core.create_spec.

Closes #10484.  Significantly improves #16068.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

requirements/common.in

@@ -161,7 +161,7 @@ django-sendfile2
 disposable-email-domains
 
 # Needed for parsing YAML with JSON references from the REST API spec files
-yamole
+jsonref
 
 # Needed for signing thumbnail requests so that they can be authenticated on the
 # other end.

requirements/dev.txt

@@ -482,6 +482,10 @@ jsonpointer==2.0 \
     --hash=sha256:c192ba86648e05fdae4f08a17ec25180a9aef5008d973407b581798a83975362 \
     --hash=sha256:ff379fa021d1b81ab539f5ec467c7745beb1a5671463f9dcc2b2d458bd361c1e \
     # via jsonpatch
+jsonref==0.2 \
+    --hash=sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f \
+    --hash=sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697 \
+    # via -r requirements/common.in
 jsonschema==3.2.0 \
     --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
     --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a \
@@ -932,7 +936,7 @@ pyyaml==5.3.1 \
     --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
     --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
     --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
-    # via cfn-lint, libcst, moto, openapi-spec-validator, yamole
+    # via cfn-lint, libcst, moto, openapi-spec-validator
 qrcode==6.1 \
     --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \
     --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369 \
@@ -1303,10 +1307,6 @@ xmltodict==0.12.0 \
     --hash=sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21 \
     --hash=sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051 \
     # via moto
-yamole==2.1.7 \
-    --hash=sha256:cd37040d1b396d58ac5bd9864999b98700d37156b2e65d9498486874aee38fda \
-    --hash=sha256:f491345f18e9d4133eed196166136144e92bb4bad83e60d44ce5754adf130a36 \
-    # via -r requirements/common.in
 zipp==3.1.0 \
     --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
     --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \

requirements/prod.txt

@@ -331,6 +331,10 @@ jmespath==0.10.0 \
     --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
     --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
     # via boto3, botocore
+jsonref==0.2 \
+    --hash=sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f \
+    --hash=sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697 \
+    # via -r requirements/common.in
 jsonschema==3.2.0 \
     --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
     --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a \
@@ -655,7 +659,7 @@ pyyaml==5.3.1 \
     --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
     --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
     --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
-    # via openapi-spec-validator, yamole
+    # via openapi-spec-validator
 qrcode==6.1 \
     --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \
     --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369 \
@@ -833,10 +837,6 @@ xmlsec==1.3.8 \
     --hash=sha256:e3fe3a1256135edec4a35508b577355baaad780a60f4bb34eec9f3281f27cabd \
     --hash=sha256:e6bcac5ee9cd0cb5aa2d4d1e14f3714c5a09185607828285179c2c94fcc083dc \
     # via python3-saml
-yamole==2.1.7 \
-    --hash=sha256:cd37040d1b396d58ac5bd9864999b98700d37156b2e65d9498486874aee38fda \
-    --hash=sha256:f491345f18e9d4133eed196166136144e92bb4bad83e60d44ce5754adf130a36 \
-    # via -r requirements/common.in
 zipp==3.1.0 \
     --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
     --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \

version.py

@@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 33
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '110.0'
+PROVISION_VERSION = '111.0'

zerver/openapi/openapi.py

@@ -30,6 +30,44 @@ EXCLUDE_DOCUMENTED_ENDPOINTS = {
     ("/settings/notifications", "patch"),
 }
 
+# Most of our code expects allOf to be preprocessed away because that is what
+# yamole did.  Its algorithm for doing so is not standards compliant, but we
+# replicate it here.
+def naively_merge(a: Dict[str, object], b: Dict[str, object]) -> Dict[str, object]:
+    ret: Dict[str, object] = a.copy()
+    for key, b_value in b.items():
+        if key == "example" or key not in ret:
+            ret[key] = b_value
+            continue
+        a_value = ret[key]
+        if isinstance(b_value, list):
+            assert isinstance(a_value, list)
+            ret[key] = a_value + b_value
+        elif isinstance(b_value, dict):
+            assert isinstance(a_value, dict)
+            ret[key] = naively_merge(a_value, b_value)
+    return ret
+
+def naively_merge_allOf(obj: object) -> object:
+    if isinstance(obj, dict):
+        return naively_merge_allOf_dict(obj)
+    elif isinstance(obj, list):
+        return list(map(naively_merge_allOf, obj))
+    else:
+        return obj
+
+def naively_merge_allOf_dict(obj: Dict[str, object]) -> Dict[str, object]:
+    if "allOf" in obj:
+        ret = obj.copy()
+        subschemas = ret.pop("allOf")
+        ret = naively_merge_allOf_dict(ret)
+        assert isinstance(subschemas, list)
+        for subschema in subschemas:
+            assert isinstance(subschema, dict)
+            ret = naively_merge(ret, naively_merge_allOf_dict(subschema))
+        return ret
+    return {key: naively_merge_allOf(value) for key, value in obj.items()}
+
 class OpenAPISpec():
     def __init__(self, openapi_path: str) -> None:
         self.openapi_path = openapi_path
@@ -39,29 +77,31 @@ class OpenAPISpec():
         self._request_validator: Optional[RequestValidator] = None
 
     def check_reload(self) -> None:
-        # Because importing yamole (and in turn, yaml) takes
+        # Because importing yaml takes significant time, and we only
-        # significant time, and we only use python-yaml for our API
+        # use python-yaml for our API docs, importing it lazily here
-        # docs, importing it lazily here is a significant optimization
+        # is a significant optimization to `manage.py` startup.
-        # to `manage.py` startup.
         #
         # There is a bit of a race here...we may have two processes
         # accessing this module level object and both trying to
         # populate self.data at the same time.  Hopefully this will
         # only cause some extra processing at startup and not data
         # corruption.
-        from yamole import YamoleParser
 
-        mtime = os.path.getmtime(self.openapi_path)
+        import yaml
+        from jsonref import JsonRef
+
+        with open(self.openapi_path) as f:
+            mtime = os.fstat(f.fileno()).st_mtime
             # Using == rather than >= to cover the corner case of users placing an
             # earlier version than the current one
             if self.mtime == mtime:
                 return
 
-        with open(self.openapi_path) as f:
+            openapi = yaml.load(f, Loader=yaml.CSafeLoader)
-            yamole_parser = YamoleParser(f)
+
-        self._openapi = yamole_parser.data
+        spec = create_spec(openapi)
-        spec = create_spec(self._openapi)
         self._request_validator = RequestValidator(spec)
+        self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi))
         self.create_endpoints_dict()
         self.mtime = mtime