From f84c9b919b9634b41dd455adb5507357007445e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yago=20Gonz=C3=A1lez?= Date: Tue, 15 May 2018 19:28:42 +0200 Subject: [PATCH] api docs: Read parameters and response fixtures from OpenAPI files. --- mypy.ini | 3 ++ templates/zerver/api/fixtures.json | 4 -- templates/zerver/api/update-message.md | 8 ++-- version.py | 2 +- zerver/lib/api_test_helpers.py | 7 ++- .../bugdown/api_arguments_table_generator.py | 47 +++++++++++++------ zerver/lib/bugdown/api_code_examples.py | 12 +++-- zerver/lib/openapi.py | 26 ++++++++++ zerver/openapi/zulip.yaml | 23 +++++++-- zerver/templatetags/app_filters.py | 3 +- 10 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 zerver/lib/openapi.py diff --git a/mypy.ini b/mypy.ini index 929a279e64..61406f0331 100644 --- a/mypy.ini +++ b/mypy.ini @@ -254,6 +254,9 @@ ignore_missing_imports = True [mypy-two_factor,two_factor.*] ignore_missing_imports = True +[mypy-yamole] +ignore_missing_imports = True + diff --git a/templates/zerver/api/fixtures.json b/templates/zerver/api/fixtures.json index a609cfd44f..a85bf8a10d 100644 --- a/templates/zerver/api/fixtures.json +++ b/templates/zerver/api/fixtures.json @@ -360,10 +360,6 @@ "msg": "", "result": "success" }, - "update-message": { - "msg": "", - "result": "success" - }, "update-message-edit-permission-error": { "code": "BAD_REQUEST", "msg": "You don't have permission to edit this message", diff --git a/templates/zerver/api/update-message.md b/templates/zerver/api/update-message.md index 788c185598..efef8cac8a 100644 --- a/templates/zerver/api/update-message.md +++ b/templates/zerver/api/update-message.md @@ -27,7 +27,7 @@ curl -X "PATCH" {{ api_url }}/v1/messages/ \
-{generate_code_example(python)|update-message|example} +{generate_code_example(python)|/messages/{message_id}:patch|example}
@@ -67,7 +67,7 @@ You only have permission to edit a message if: ## Arguments -{generate_api_arguments_table|arguments.json|update-message.md} +{generate_api_arguments_table|zulip.yaml|/messages/{message_id}:patch} ## Response @@ -75,9 +75,9 @@ You only have permission to edit a message if: A typical successful JSON response may look like: -{generate_code_example|update-message|fixture} +{generate_code_example|/messages/{message_id}:patch|fixture(200)} A typical JSON response for when one doesn't have the permission to edit a particular message: -{generate_code_example|update-message-edit-permission-error|fixture} +{generate_code_example|/messages/{message_id}:patch|fixture(400)} diff --git a/version.py b/version.py index 7928c5cd03..2ce41403e2 100644 --- a/version.py +++ b/version.py @@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git" # Typically, adding a dependency only requires a minor version bump, and # removing a dependency requires a major version bump. -PROVISION_VERSION = '20.3' +PROVISION_VERSION = '20.4' diff --git a/zerver/lib/api_test_helpers.py b/zerver/lib/api_test_helpers.py index 7eec764edd..91f4c53811 100644 --- a/zerver/lib/api_test_helpers.py +++ b/zerver/lib/api_test_helpers.py @@ -1,10 +1,12 @@ from typing import Dict, Any, Optional, Iterable from io import StringIO +from yamole import YamoleParser import json import os from zerver.lib import mdiff +from zerver.lib.openapi import get_openapi_fixture if False: from zulip import Client @@ -383,7 +385,8 @@ def update_message(client, message_id): result = client.update_message(request) # {code_example|end} - fixture = FIXTURES['update-message'] + fixture = get_openapi_fixture('/messages/{message_id}', 'patch', '200') + test_against_fixture(result, fixture) # test it was actually updated @@ -491,7 +494,7 @@ TEST_FUNCTIONS = { 'render-message': render_message, 'stream-message': stream_message, 'private-message': private_message, - 'update-message': update_message, + '/messages/{message_id}:patch': update_message, 'get-stream-id': get_stream_id, 'get-subscribed-streams': list_subscriptions, 'get-all-streams': get_streams, diff --git a/zerver/lib/bugdown/api_arguments_table_generator.py b/zerver/lib/bugdown/api_arguments_table_generator.py index f064383b25..340f923b72 100644 --- a/zerver/lib/bugdown/api_arguments_table_generator.py +++ b/zerver/lib/bugdown/api_arguments_table_generator.py @@ -4,10 +4,11 @@ import ujson from markdown.extensions import Extension from markdown.preprocessors import Preprocessor +from zerver.lib.openapi import get_openapi_parameters from typing import Any, Dict, Optional, List import markdown -REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+?)\s*\}') +REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') class MarkdownArgumentsTableGenerator(Extension): @@ -16,8 +17,6 @@ class MarkdownArgumentsTableGenerator(Extension): configs = {} self.config = { 'base_path': ['.', 'Default location from which to evaluate relative paths for the JSON files.'], - 'openapi_path': ['../../../zerver/openapi', - 'Default location from which to evaluate relative paths for the YAML files.'] } for key, value in configs.items(): self.setConfig(key, value) @@ -41,19 +40,29 @@ class APIArgumentsTablePreprocessor(Preprocessor): match = REGEXP.search(line) if match: - json_filename = match.group(1) - doc_filename = match.group(2) - json_filename = os.path.expanduser(json_filename) - if not os.path.isabs(json_filename): - json_filename = os.path.normpath(os.path.join(self.base_path, json_filename)) + filename = match.group(1) + doc_name = match.group(2) + filename = os.path.expanduser(filename) + + is_openapi_format = filename.endswith('.yaml') + + if not os.path.isabs(filename): + parent_dir = self.base_path + filename = os.path.normpath(os.path.join(parent_dir, filename)) + try: - with open(json_filename, 'r') as fp: - json_obj = ujson.load(fp) - arguments = json_obj[doc_filename] - text = self.render_table(arguments) + if is_openapi_format: + endpoint, method = doc_name.rsplit(':', 1) + arguments = get_openapi_parameters(endpoint, method) + else: + with open(filename, 'r') as fp: + json_obj = ujson.load(fp) + arguments = json_obj[doc_name] + + text = self.render_table(arguments) except Exception as e: print('Warning: could not find file {}. Ignoring ' - 'statement. Error: {}'.format(json_filename, e)) + 'statement. Error: {}'.format(filename, e)) # If the file cannot be opened, just substitute an empty line # in place of the macro include line lines[loc] = REGEXP.sub('', line) @@ -101,11 +110,19 @@ class APIArgumentsTablePreprocessor(Preprocessor): md_engine = markdown.Markdown(extensions=[]) for argument in arguments: + oneof = ['`' + item + '`' + for item in argument.get('schema', {}).get('enum', [])] + description = argument['description'] + if oneof: + description += '\nMust be one of: {}.'.format(', '.join(oneof)) + # TODO: Swagger allows indicating where the argument goes + # (path, querystring, form data...). A column in the table should + # be added for this. table.append(tr.format( - argument=argument['argument'], + argument=argument.get('argument') or argument.get('name'), example=argument['example'], required='Yes' if argument.get('required') else 'No', - description=md_engine.convert(argument['description']), + description=md_engine.convert(description), )) table.append("") diff --git a/zerver/lib/bugdown/api_code_examples.py b/zerver/lib/bugdown/api_code_examples.py index 6d56787c34..a012a9d0d7 100644 --- a/zerver/lib/bugdown/api_code_examples.py +++ b/zerver/lib/bugdown/api_code_examples.py @@ -7,11 +7,13 @@ import inspect from markdown.extensions import Extension from markdown.preprocessors import Preprocessor from typing import Any, Dict, Optional, List +from yamole import YamoleParser import markdown import zerver.lib.api_test_helpers +from zerver.lib.openapi import get_openapi_fixture -MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+?)\s*\))?\}') +MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}') CODE_EXAMPLE_REGEX = re.compile(r'\# \{code_example\|\s*(.+?)\s*\}') PYTHON_CLIENT_CONFIG = """ @@ -138,8 +140,12 @@ class APICodeExamplesPreprocessor(Preprocessor): def render_fixture(self, function: str, name: Optional[str]=None) -> List[str]: fixture = [] - if name: - fixture_dict = zerver.lib.api_test_helpers.FIXTURES[function][name] + # We assume that if the function we're rendering starts with a slash + # it's a path in the endpoint and therefore it uses the new OpenAPI + # format. + if function.startswith('/'): + path, method = function.rsplit(':', 1) + fixture_dict = get_openapi_fixture(path, method, name) else: fixture_dict = zerver.lib.api_test_helpers.FIXTURES[function] diff --git a/zerver/lib/openapi.py b/zerver/lib/openapi.py new file mode 100644 index 0000000000..6fafb5e17d --- /dev/null +++ b/zerver/lib/openapi.py @@ -0,0 +1,26 @@ +# Set of helper functions to manipulate the OpenAPI files that define our REST +# API's specification. +import os +from typing import Any, Dict, List, Optional + +from yamole import YamoleParser + +OPENAPI_SPEC_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), + '../openapi/zulip.yaml')) + +with open(OPENAPI_SPEC_PATH) as file: + yaml_parser = YamoleParser(file) + +OPENAPI_SPEC = yaml_parser.data + + +def get_openapi_fixture(endpoint: str, method: str, + response: Optional[str]='200') -> Dict[str, Any]: + return (OPENAPI_SPEC['paths'][endpoint][method.lower()]['responses'] + [response]['content']['application/json']['schema'] + ['example']) + +def get_openapi_parameters(endpoint: str, + method: str) -> List[Dict[str, Any]]: + return (OPENAPI_SPEC['paths'][endpoint][method]['parameters']) diff --git a/zerver/openapi/zulip.yaml b/zerver/openapi/zulip.yaml index b585c8f9fd..17ff033a95 100644 --- a/zerver/openapi/zulip.yaml +++ b/zerver/openapi/zulip.yaml @@ -60,7 +60,8 @@ paths: example: change_all - name: content in: query - description: Message's new body. + description: | + The content of the message. Maximum message size of 10000 bytes. schema: type: string example: Hello @@ -68,7 +69,11 @@ paths: - basicAuth: [] responses: '200': - $ref: '#/components/responses/SimpleSuccess' + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/JsonSuccess' '400': description: Bad request. content: @@ -84,6 +89,12 @@ paths: - The time limit for editing this message has past - Nothing to change - Topic can't be empty + - example: + { + "code": "BAD_REQUEST", + "msg": "You don't have permission to edit this message", + "result": "error" + } components: ####################### # Security definitions @@ -96,13 +107,13 @@ components: Basic authentication, with the user's email as the username, and the API key as the password. The API key can be fetched using the `/fetch_api_key` or `/dev_fetch_api_key` endpoints. + schemas: JsonResponse: type: object properties: result: type: string - JsonSuccess: allOf: - $ref: '#/components/schemas/JsonResponse' @@ -115,7 +126,11 @@ components: - success msg: type: string - + - example: + { + "msg": "", + "result": "success" + } JsonError: allOf: - $ref: '#/components/schemas/JsonResponse' diff --git a/zerver/templatetags/app_filters.py b/zerver/templatetags/app_filters.py index 109166ec10..f386043ffb 100644 --- a/zerver/templatetags/app_filters.py +++ b/zerver/templatetags/app_filters.py @@ -96,8 +96,7 @@ def render_markdown_path(markdown_file_path: str, context: Optional[Dict[Any, An ), zerver.lib.bugdown.fenced_code.makeExtension(), zerver.lib.bugdown.api_arguments_table_generator.makeExtension( - base_path='templates/zerver/api/', - openapi_path='zerver/openapi'), + base_path='templates/zerver/api/'), zerver.lib.bugdown.api_code_examples.makeExtension(), zerver.lib.bugdown.help_settings_links.makeExtension(), ]