diff --git a/tools/test-api b/tools/test-api index 1b12e24fe4..5ba66898f8 100755 --- a/tools/test-api +++ b/tools/test-api @@ -29,6 +29,7 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981') # Zerver imports should happen after `django.setup()` is run # by the test_server_running decorator. from zerver.openapi.python_examples import test_the_api, test_invalid_api_key + from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success from zerver.lib.actions import do_create_user from zerver.lib.users import get_api_key from zerver.models import get_user, get_realm @@ -41,14 +42,24 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981') user = get_user(email, realm) api_key = get_api_key(user) site = 'http://zulip.zulipdev.com:9981' - client = Client( email=email, api_key=api_key, site=site ) - # Prepare the admin client + # Prepare a generic bot client for curl testing + email = 'default-bot@zulip.com' + realm = get_realm("zulip") + bot_user = get_user(email, realm) + api_key = get_api_key(bot_user) + bot_client = Client( + email=email, + api_key=api_key, + site=site + ) + + # Prepare the non-admin client email = 'guest@zulip.com' # guest is not an admin guest_user = do_create_user('guest@zulip.com', 'secret', get_realm('zulip'), 'Mr. Guest', 'guest') @@ -59,6 +70,7 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981') site=site ) + test_generated_curl_examples_for_success(bot_client) test_the_api(client, nonadmin_client) # Test error payloads diff --git a/zerver/openapi/test_curl_examples.py b/zerver/openapi/test_curl_examples.py new file mode 100644 index 0000000000..8a67b8ca1d --- /dev/null +++ b/zerver/openapi/test_curl_examples.py @@ -0,0 +1,84 @@ +import glob +import json +import shlex +import subprocess +import markdown + +from zulip import Client +from zerver.lib.bugdown import api_code_examples +from zerver.models import get_realm + +# whitelisted_files is a list of endpoints which have been +# migrated to the example generation system and can thus be +# tested. +whitelisted_files = [ + "get-messages.md", +] + +def test_generated_curl_examples_for_success(client: Client) -> None: + authentication_line = "{}:{}".format(client.email, client.api_key) + # A limited markdown engine that just processes the code example syntax. + realm = get_realm("zulip") + md_engine = markdown.Markdown(extensions=[api_code_examples.makeExtension( + api_url=realm.uri + "/api")]) + + for file_name in glob.glob("templates/zerver/api/*.md"): + documentation_lines = open(file_name, "r").readlines() + for line in documentation_lines: + # A typical example from the markdown source looks like this: + # {generate_code_example(curl, ...} + if not line.startswith("{generate_code_example(curl"): + continue + + # To do an end-to-end test on the documentation examples + # that will be actually shown to users, we use the + # markdown rendering pipeline to compute the user-facing + # example, and then run that to test it. + curl_command_html = md_engine.convert(line.strip()) + curl_command_text = curl_command_html[len("

curl\n"):-len("

")] + + curl_command_text = curl_command_text.replace( + "BOT_EMAIL_ADDRESS:BOT_API_KEY", authentication_line) + + print("Testing %s ..." % (curl_command_text.split("\n")[0],)) + + # Turn the text into an arguments list. + generated_curl_command = [ + x for x in shlex.split(curl_command_text) if x != "\n"] + + response = None + try: + # We split this across two lines so if curl fails and + # returns non-JSON output, we'll still print it. + response = subprocess.check_output(generated_curl_command).decode('utf-8') + response = json.loads(response) + assert(response["result"] == "success") + except (AssertionError, Exception): + error_template = """ +Error verifying the success of the API documentation curl example. + +File: {file_name} +Line: {line} +Curl Command: +{curl_command} +Response: +{response} + +This test is designed to check each generate_code_example(curl) instance in the +API documentation for success. If this fails then it means that the curl example +that was generated was faulty and when tried, it resulted in an unsuccessful +response. + +Common reasons for why this could occur: + 1. One or more example values in zerver/openapi/zulip.yaml for this endpoint + do not line up with the values in the test database. + 2. One or more mandatory parameters were included in the "exclude" list. + +To learn more about the test itself, see zerver/openapi/test_curl_examples.py. +""" + print(error_template.format( + file_name=file_name, + line=line, + curl_command=generated_curl_command, + response=json.dumps(response, indent=4))) + raise