diff --git a/requirements/common.in b/requirements/common.in index 896ab23762..47ebe5a4ec 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -161,7 +161,7 @@ django-sendfile2 disposable-email-domains # Needed for parsing YAML with JSON references from the REST API spec files -yamole +jsonref # Needed for signing thumbnail requests so that they can be authenticated on the # other end. diff --git a/requirements/dev.txt b/requirements/dev.txt index 88d15efd3c..293e88fe62 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -482,6 +482,10 @@ jsonpointer==2.0 \ --hash=sha256:c192ba86648e05fdae4f08a17ec25180a9aef5008d973407b581798a83975362 \ --hash=sha256:ff379fa021d1b81ab539f5ec467c7745beb1a5671463f9dcc2b2d458bd361c1e \ # via jsonpatch +jsonref==0.2 \ + --hash=sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f \ + --hash=sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697 \ + # via -r requirements/common.in jsonschema==3.2.0 \ --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a \ @@ -932,7 +936,7 @@ pyyaml==5.3.1 \ --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \ --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \ --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \ - # via cfn-lint, libcst, moto, openapi-spec-validator, yamole + # via cfn-lint, libcst, moto, openapi-spec-validator qrcode==6.1 \ --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \ --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369 \ @@ -1303,10 +1307,6 @@ xmltodict==0.12.0 \ --hash=sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21 \ --hash=sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051 \ # via moto -yamole==2.1.7 \ - --hash=sha256:cd37040d1b396d58ac5bd9864999b98700d37156b2e65d9498486874aee38fda \ - --hash=sha256:f491345f18e9d4133eed196166136144e92bb4bad83e60d44ce5754adf130a36 \ - # via -r requirements/common.in zipp==3.1.0 \ --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \ --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \ diff --git a/requirements/prod.txt b/requirements/prod.txt index c058832a30..785c16edbe 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -331,6 +331,10 @@ jmespath==0.10.0 \ --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \ # via boto3, botocore +jsonref==0.2 \ + --hash=sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f \ + --hash=sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697 \ + # via -r requirements/common.in jsonschema==3.2.0 \ --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a \ @@ -655,7 +659,7 @@ pyyaml==5.3.1 \ --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \ --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \ --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \ - # via openapi-spec-validator, yamole + # via openapi-spec-validator qrcode==6.1 \ --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \ --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369 \ @@ -833,10 +837,6 @@ xmlsec==1.3.8 \ --hash=sha256:e3fe3a1256135edec4a35508b577355baaad780a60f4bb34eec9f3281f27cabd \ --hash=sha256:e6bcac5ee9cd0cb5aa2d4d1e14f3714c5a09185607828285179c2c94fcc083dc \ # via python3-saml -yamole==2.1.7 \ - --hash=sha256:cd37040d1b396d58ac5bd9864999b98700d37156b2e65d9498486874aee38fda \ - --hash=sha256:f491345f18e9d4133eed196166136144e92bb4bad83e60d44ce5754adf130a36 \ - # via -r requirements/common.in zipp==3.1.0 \ --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \ --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \ diff --git a/version.py b/version.py index a3032d9223..1d7abac1c3 100644 --- a/version.py +++ b/version.py @@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 33 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '110.0' +PROVISION_VERSION = '111.0' diff --git a/zerver/openapi/openapi.py b/zerver/openapi/openapi.py index 7210aca173..3335983b36 100644 --- a/zerver/openapi/openapi.py +++ b/zerver/openapi/openapi.py @@ -30,6 +30,44 @@ EXCLUDE_DOCUMENTED_ENDPOINTS = { ("/settings/notifications", "patch"), } +# Most of our code expects allOf to be preprocessed away because that is what +# yamole did. Its algorithm for doing so is not standards compliant, but we +# replicate it here. +def naively_merge(a: Dict[str, object], b: Dict[str, object]) -> Dict[str, object]: + ret: Dict[str, object] = a.copy() + for key, b_value in b.items(): + if key == "example" or key not in ret: + ret[key] = b_value + continue + a_value = ret[key] + if isinstance(b_value, list): + assert isinstance(a_value, list) + ret[key] = a_value + b_value + elif isinstance(b_value, dict): + assert isinstance(a_value, dict) + ret[key] = naively_merge(a_value, b_value) + return ret + +def naively_merge_allOf(obj: object) -> object: + if isinstance(obj, dict): + return naively_merge_allOf_dict(obj) + elif isinstance(obj, list): + return list(map(naively_merge_allOf, obj)) + else: + return obj + +def naively_merge_allOf_dict(obj: Dict[str, object]) -> Dict[str, object]: + if "allOf" in obj: + ret = obj.copy() + subschemas = ret.pop("allOf") + ret = naively_merge_allOf_dict(ret) + assert isinstance(subschemas, list) + for subschema in subschemas: + assert isinstance(subschema, dict) + ret = naively_merge(ret, naively_merge_allOf_dict(subschema)) + return ret + return {key: naively_merge_allOf(value) for key, value in obj.items()} + class OpenAPISpec(): def __init__(self, openapi_path: str) -> None: self.openapi_path = openapi_path @@ -39,29 +77,31 @@ class OpenAPISpec(): self._request_validator: Optional[RequestValidator] = None def check_reload(self) -> None: - # Because importing yamole (and in turn, yaml) takes - # significant time, and we only use python-yaml for our API - # docs, importing it lazily here is a significant optimization - # to `manage.py` startup. + # Because importing yaml takes significant time, and we only + # use python-yaml for our API docs, importing it lazily here + # is a significant optimization to `manage.py` startup. # # There is a bit of a race here...we may have two processes # accessing this module level object and both trying to # populate self.data at the same time. Hopefully this will # only cause some extra processing at startup and not data # corruption. - from yamole import YamoleParser - mtime = os.path.getmtime(self.openapi_path) - # Using == rather than >= to cover the corner case of users placing an - # earlier version than the current one - if self.mtime == mtime: - return + import yaml + from jsonref import JsonRef with open(self.openapi_path) as f: - yamole_parser = YamoleParser(f) - self._openapi = yamole_parser.data - spec = create_spec(self._openapi) + mtime = os.fstat(f.fileno()).st_mtime + # Using == rather than >= to cover the corner case of users placing an + # earlier version than the current one + if self.mtime == mtime: + return + + openapi = yaml.load(f, Loader=yaml.CSafeLoader) + + spec = create_spec(openapi) self._request_validator = RequestValidator(spec) + self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi)) self.create_endpoints_dict() self.mtime = mtime