openapi: Add a test to ensure generated curl examples work.

This new test runs each generated curl example against the Zulip API,
checking whether it returns successfully without errors.

Significantly modified by tabbott for simplicity.

tools/test-api

@@ -29,6 +29,7 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981')
     # Zerver imports should happen after `django.setup()` is run
     # by the test_server_running decorator.
     from zerver.openapi.python_examples import test_the_api, test_invalid_api_key
+    from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success
     from zerver.lib.actions import do_create_user
     from zerver.lib.users import get_api_key
     from zerver.models import get_user, get_realm
@@ -41,14 +42,24 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981')
     user = get_user(email, realm)
     api_key = get_api_key(user)
     site = 'http://zulip.zulipdev.com:9981'
-
     client = Client(
         email=email,
         api_key=api_key,
         site=site
     )
 
-    # Prepare the admin client
+    # Prepare a generic bot client for curl testing
+    email = 'default-bot@zulip.com'
+    realm = get_realm("zulip")
+    bot_user = get_user(email, realm)
+    api_key = get_api_key(bot_user)
+    bot_client = Client(
+        email=email,
+        api_key=api_key,
+        site=site
+    )
+
+    # Prepare the non-admin client
     email = 'guest@zulip.com'  # guest is not an admin
     guest_user = do_create_user('guest@zulip.com', 'secret',
                                 get_realm('zulip'), 'Mr. Guest', 'guest')
@@ -59,6 +70,7 @@ with test_server_running(force=options.force, external_host='zulipdev.com:9981')
         site=site
     )
 
+    test_generated_curl_examples_for_success(bot_client)
     test_the_api(client, nonadmin_client)
 
     # Test error payloads

zerver/openapi/test_curl_examples.py

@@ -0,0 +1,84 @@
+import glob
+import json
+import shlex
+import subprocess
+import markdown
+
+from zulip import Client
+from zerver.lib.bugdown import api_code_examples
+from zerver.models import get_realm
+
+# whitelisted_files is a list of endpoints which have been
+# migrated to the example generation system and can thus be
+# tested.
+whitelisted_files = [
+    "get-messages.md",
+]
+
+def test_generated_curl_examples_for_success(client: Client) -> None:
+    authentication_line = "{}:{}".format(client.email, client.api_key)
+    # A limited markdown engine that just processes the code example syntax.
+    realm = get_realm("zulip")
+    md_engine = markdown.Markdown(extensions=[api_code_examples.makeExtension(
+        api_url=realm.uri + "/api")])
+
+    for file_name in glob.glob("templates/zerver/api/*.md"):
+        documentation_lines = open(file_name, "r").readlines()
+        for line in documentation_lines:
+            # A typical example from the markdown source looks like this:
+            #     {generate_code_example(curl, ...}
+            if not line.startswith("{generate_code_example(curl"):
+                continue
+
+            # To do an end-to-end test on the documentation examples
+            # that will be actually shown to users, we use the
+            # markdown rendering pipeline to compute the user-facing
+            # example, and then run that to test it.
+            curl_command_html = md_engine.convert(line.strip())
+            curl_command_text = curl_command_html[len("<p><code>curl\n"):-len("</code></p>")]
+
+            curl_command_text = curl_command_text.replace(
+                "BOT_EMAIL_ADDRESS:BOT_API_KEY", authentication_line)
+
+            print("Testing %s ..." % (curl_command_text.split("\n")[0],))
+
+            # Turn the text into an arguments list.
+            generated_curl_command = [
+                x for x in shlex.split(curl_command_text) if x != "\n"]
+
+            response = None
+            try:
+                # We split this across two lines so if curl fails and
+                # returns non-JSON output, we'll still print it.
+                response = subprocess.check_output(generated_curl_command).decode('utf-8')
+                response = json.loads(response)
+                assert(response["result"] == "success")
+            except (AssertionError, Exception):
+                error_template = """
+Error verifying the success of the API documentation curl example.
+
+File: {file_name}
+Line: {line}
+Curl Command:
+{curl_command}
+Response:
+{response}
+
+This test is designed to check each generate_code_example(curl) instance in the
+API documentation for success. If this fails then it means that the curl example
+that was generated was faulty and when tried, it resulted in an unsuccessful
+response.
+
+Common reasons for why this could occur:
+    1. One or more example values in zerver/openapi/zulip.yaml for this endpoint
+       do not line up with the values in the test database.
+    2. One or more mandatory parameters were included in the "exclude" list.
+
+To learn more about the test itself, see zerver/openapi/test_curl_examples.py.
+"""
+                print(error_template.format(
+                    file_name=file_name,
+                    line=line,
+                    curl_command=generated_curl_command,
+                    response=json.dumps(response, indent=4)))
+                raise