openapi: Add test for validating examples.

Zulip's openapi specification in zulip.yaml has various examples
for various schemas. Validate the example with their respective
schemas to ensure that all the examples are schematically correct.

Part of #14100.

package.json

@@ -44,6 +44,7 @@
     "mini-css-extract-plugin": "^0.9.0",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.25",
+    "openapi-examples-validator": "^3.0.2",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
     "plotly.js": "^1.48.1",
     "postcss-calc": "^7.0.1",

tools/check-openapi

@@ -3,6 +3,7 @@
 const fs = require('fs');
 const jsyaml = require('js-yaml');
 const SwaggerParser = require('swagger-parser');
+const ExampleValidator = require('openapi-examples-validator');
 
 (async () => {
     // Iterate through the changed files, passed in the arguments.
@@ -16,6 +17,13 @@ const SwaggerParser = require('swagger-parser');
                 }).openapi !== undefined
             ) {
                 await SwaggerParser.validate(file);
+                const res = await ExampleValidator.validateFile(file);
+                if (!res.valid) {
+                    for (const error of res.errors) {
+                        console.error(error);
+                    }
+                    process.exitCode = 1;
+                }
             }
         } catch (error) {
             if (error instanceof jsyaml.YAMLException) {

yarn.lock

@@ -11,6 +11,15 @@
     orbit-camera-controller "^4.0.0"
     turntable-camera-controller "^3.0.0"
 
+"@apidevtools/json-schema-ref-parser@9.0.1":
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.1.tgz#c0ed0bd21a7397d2d7a83b69565268f2d78f2d7a"
+  integrity sha512-Qsdz0W0dyK84BuBh5KZATWXOtVDXIF2EeNRzpyWblPUeAmnIokwWcwrpAm5pTPMjuWoIQt9C67X3Af1OlL6oSw==
+  dependencies:
+    "@jsdevtools/ono" "^7.1.2"
+    call-me-maybe "^1.0.1"
+    js-yaml "^3.13.1"
+
 "@apidevtools/json-schema-ref-parser@^8.0.0":
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-8.0.0.tgz#9eb749499b3f8d919e90bb141e4b6f67aee4692d"
@@ -799,10 +808,10 @@
     pirates "^4.0.0"
     source-map-support "^0.5.16"
 
-"@babel/runtime@^7.3.1", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.0":
+"@babel/runtime@^7.3.1", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
-  version "7.9.2"
+  version "7.9.6"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.9.2.tgz#d90df0583a3a252f09aaa619665367bae518db06"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.9.6.tgz#a9102eb5cadedf3f31d08a9ecf294af7827ea29f"
-  integrity sha512-NE2DtOdufG7R5vnfQUTehdTfNycfUANEtCa9PssN9O/xmTzP4E08UI797ixaei6hBEVL9BI/PsdJS5x7mWoB9Q==
+  integrity sha512-64AF1xY3OAkFHqOb9s4jpgk1Mm5vDZ4L3acHvAml+53nO1XbXLuDodsVpO4OIUsmemlUHMxNdYMNJmsvOwLrvQ==
   dependencies:
     regenerator-runtime "^0.13.4"
 
@@ -861,10 +870,10 @@
   resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.2.tgz#26520bf09abe4a5644cd5414e37125a8954241dd"
   integrity sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw==
 
-"@jsdevtools/ono@^7.1.0":
+"@jsdevtools/ono@^7.1.0", "@jsdevtools/ono@^7.1.2":
-  version "7.1.1"
+  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.1.tgz#36034f9cb0fb456858c137a3f3e6d6db67ab5cc5"
+  resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.2.tgz#373995bb40a6686589a7fcfec06b0e6e304ef6c6"
-  integrity sha512-pu5fxkbLQWzRbBgfFbZfHXz0KlYojOfVdUhcNfy9lef8ZhBt0pckGr8g7zv4vPX4Out5vBNvqd/az4UaVWzZ9A==
+  integrity sha512-qS/a24RA5FEoiJS9wiv6Pwg2c/kiUo3IVUQcfeM9JvsR6pM8Yx+yl/6xWYLckZCT5jpLNhslgjiA8p/XcGyMRQ==
 
 "@mapbox/geojson-area@0.2.2":
   version "0.2.2"
@@ -1661,10 +1670,17 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1:
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.4.1.tgz#ef916e271c64ac12171fd8384eaae6b2345854da"
   integrity sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==
 
-ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.5.5:
+ajv-oai@1.2.0:
-  version "6.12.0"
+  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.0.tgz#06d60b96d87b8454a5adaba86e7854da629db4b7"
+  resolved "https://registry.yarnpkg.com/ajv-oai/-/ajv-oai-1.2.0.tgz#93ba0d3c64edf55e575c9d9f52fe494251c5b6d0"
-  integrity sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==
+  integrity sha512-BQ2HL/ZfiMm68Xdy7dkS49Vnnq+gSsxfOugJB4TA8Kmu4Ie9ZIa4K4VQYbcHxyW4ccg6l9VB57PjRA2RPh1elw==
+  dependencies:
+    decimal.js "^10.2.0"
+
+ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.5.5:
+  version "6.12.2"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.2.tgz#c629c5eced17baf314437918d2da88c99d5958cd"
+  integrity sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==
   dependencies:
     fast-deep-equal "^3.1.1"
     fast-json-stable-stringify "^2.0.0"
@@ -2969,6 +2985,11 @@ commander@^4.1.1:
   resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
   integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
 
+commander@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae"
+  integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
+
 commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@@ -4110,7 +4131,7 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4"
   integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==
 
-errno@^0.1.3, errno@~0.1.7:
+errno@0.1.7, errno@^0.1.3, errno@~0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618"
   integrity sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==
@@ -4888,7 +4909,7 @@ for-in@^1.0.2:
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
   integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=
 
-foreach@^2.0.5:
+foreach@^2.0.4, foreach@^2.0.5:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
   integrity sha1-C+4AUBiusmDQo6865ljdATbsG5k=
@@ -6946,6 +6967,20 @@ json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2:
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
   integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==
 
+json-pointer@0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/json-pointer/-/json-pointer-0.6.0.tgz#8e500550a6aac5464a473377da57aa6cc22828d7"
+  integrity sha1-jlAFUKaqxUZKRzN32leqbMIoKNc=
+  dependencies:
+    foreach "^2.0.4"
+
+json-schema-ref-parser@^9.0.1:
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/json-schema-ref-parser/-/json-schema-ref-parser-9.0.1.tgz#3c1fd01c159e3e6016190284752dcda93f8ea5b0"
+  integrity sha512-KLrCjRjW5hMXxsX4osVBWpwixXL9NtICfpyNNS0eHguN5mP/I4UatI7i7PFS8jU94b1NHF4EbirACdCn0RFPBA==
+  dependencies:
+    "@apidevtools/json-schema-ref-parser" "9.0.1"
+
 json-schema-traverse@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
@@ -6997,6 +7032,11 @@ jsonfile@^2.1.0:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonpath-plus@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/jsonpath-plus/-/jsonpath-plus-4.0.0.tgz#954b69faa3d8b07f30ae2f9e601176a4b0d2806e"
+  integrity sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==
+
 jsprim@^1.2.2:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -8369,6 +8409,22 @@ onetime@^5.1.0:
   dependencies:
     mimic-fn "^2.1.0"
 
+openapi-examples-validator@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/openapi-examples-validator/-/openapi-examples-validator-3.0.2.tgz#72bdaad8c6d56841640abec64f7eccf2b3aecee1"
+  integrity sha512-LRglZ/k/srjLgIrvZqdb1eJ5FA3Ul0d5rew6Vxi/TRgx2otSgmPXtOidkwSSKnU/fePg1RlqerICVC4ZPRXO1w==
+  dependencies:
+    ajv "^6.12.2"
+    ajv-oai "1.2.0"
+    commander "^5.1.0"
+    errno "0.1.7"
+    glob "^7.1.6"
+    json-pointer "0.6.0"
+    json-schema-ref-parser "^9.0.1"
+    jsonpath-plus "^4.0.0"
+    lodash "^4.17.15"
+    yaml "^1.9.2"
+
 openapi-types@^1.3.5:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-1.3.5.tgz#6718cfbc857fe6c6f1471f65b32bdebb9c10ce40"
@@ -13011,12 +13067,12 @@ yallist@^4.0.0:
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
   integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
 
-yaml@^1.6.0, yaml@^1.7.2:
+yaml@^1.6.0, yaml@^1.7.2, yaml@^1.9.2:
-  version "1.9.0"
+  version "1.9.2"
-  resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.9.0.tgz#dc1ff3e24837b62bc3c8ae02c28e16ee5742b9d6"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed"
-  integrity sha512-3GLZOj8A9Gsp0Fw3kOyj0zqk4xMq+YvhbHSDYALd2NMOfIpyZeBhz32ZiNU7AtX1MtXX/9JJgxSElGRwvv9enA==
+  integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg==
   dependencies:
-    "@babel/runtime" "^7.9.0"
+    "@babel/runtime" "^7.9.2"
 
 yargs-parser@^11.1.1:
   version "11.1.1"

zerver/lib/bugdown/api_arguments_table_generator.py

@@ -117,11 +117,15 @@ class APIArgumentsTablePreprocessor(Preprocessor):
 
             # TODO: OpenAPI allows indicating where the argument goes
             # (path, querystring, form data...).  We should document this detail.
+            example = ""
+            if 'example' in argument:
+                example = argument['example']
+            else:
+                example = json.dumps(argument['content']['application/json']['example'])
+
             table.append(argument_template.format(
                 argument=argument.get('argument') or argument.get('name'),
-                # Show this as JSON to avoid changing the quoting style, which
+                example=escape_html(example),
-                # may cause problems with JSON encoding.
-                example=escape_html(json.dumps(argument['example'])),
                 required='<span class="api-argument-required">required</span>' if argument.get('required')
                 else '<span class="api-argument-optional">optional</span>',
                 description=md_engine.convert(description),

zerver/openapi/markdown_extension.py

@@ -127,6 +127,11 @@ def curl_method_arguments(endpoint: str, method: str,
 
 def get_openapi_param_example_value_as_string(endpoint: str, method: str, param: Dict[str, Any],
                                               curl_argument: bool=False) -> str:
+    jsonify = False
+    param_name = param["name"]
+    if "content" in param:
+        param = param["content"]["application/json"]
+        jsonify = True
     if "type" in param["schema"]:
         param_type = param["schema"]["type"]
     else:
@@ -135,7 +140,6 @@ def get_openapi_param_example_value_as_string(endpoint: str, method: str, param:
         # union type.  But for this logic's purpose, it's good enough
         # to just check the first parameter.
         param_type = param["schema"]["oneOf"][0]["type"]
-    param_name = param["name"]
     if param_type in ["object", "array"]:
         example_value = param.get("example", None)
         if not example_value:
@@ -152,7 +156,7 @@ cURL example.""".format(endpoint, method, param_name)
         example_value = param.get("example", DEFAULT_EXAMPLE[param_type])
         if isinstance(example_value, bool):
             example_value = str(example_value).lower()
-        if param["schema"].get("format", "") == "json":
+        if jsonify:
             example_value = json.dumps(example_value)
         if curl_argument:
             return "    -d '{}={}'".format(param_name, example_value)

zerver/tests/test_openapi.py

@@ -442,6 +442,15 @@ do not match the types declared in the implementation of {}.\n""".format(functio
                     continue
 
             name: str = element["name"]
+            schema = {}
+            if "content" in element:
+                schema = element["content"]["application/json"]["schema"]
+                # If content_type is application/json then the
+                # data type is essentially string.
+                openapi_params.add((name, VARMAP["string"]))
+                continue
+
+            else:
                 schema = element["schema"]
             if 'oneOf' in schema:
                 # Hack: Just use the type of the first value
@@ -459,7 +468,7 @@ do not match the types declared in the implementation of {}.\n""".format(functio
                     self.assertTrue(len(subtypes) > 1)
                     sub_type = self.get_type_by_priority(subtypes)
                 else:
-                    sub_type = VARMAP[element["schema"]["items"]["type"]]
+                    sub_type = VARMAP[schema["items"]["type"]]
                     self.assertIsNotNone(sub_type)
                 openapi_params.add((name, (_type, sub_type)))
             else: