bugdown/api_code_examples: Add support for multiple languages.

This commit modifies the Markdown extension in bugdown/api_code_examples.py
to support rendering code examples in multiple languages by specifying
the language like so:

{generate_code_example(python)|doc.md|example}

This makes us one step closer towards adding support for testable
JavaScript code examples.

templates/zerver/api/add-subscriptions.md

@@ -35,7 +35,7 @@ curl {{ api_url }}/v1/users/me/subscriptions \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|add-subscriptions|example}
+{generate_code_example(python)|add-subscriptions|example}
 
 </div>
 

templates/zerver/api/create-user.md

@@ -31,7 +31,7 @@ curl {{ api_url }}/v1/users \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|create-user|example(admin_config=True)}
+{generate_code_example(python)|create-user|example(admin_config=True)}
 
 </div>
 

templates/zerver/api/delete-queue.md

@@ -25,7 +25,7 @@ curl -X "DELETE" {{ api_url }}/v1/events \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|delete-queue|example}
+{generate_code_example(python)|delete-queue|example}
 
 </div>
 

templates/zerver/api/get-all-streams.md

@@ -31,7 +31,7 @@ curl {{ api_url }}/v1/streams?include_public=false \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|get-all-streams|example}
+{generate_code_example(python)|get-all-streams|example}
 
 </div>
 

templates/zerver/api/get-all-users.md

@@ -30,7 +30,7 @@ curl {{ api_url }}/v1/users?client_gravatar=true \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|get-all-users|example}
+{generate_code_example(python)|get-all-users|example}
 
 </div>
 

templates/zerver/api/get-profile.md

@@ -25,7 +25,7 @@ curl {{ api_url }}/v1/users/me \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|get-profile|example}
+{generate_code_example(python)|get-profile|example}
 
 </div>
 

templates/zerver/api/get-stream-id.md

@@ -24,7 +24,7 @@ curl {{ api_url }}/v1/get_stream_id?stream=Denmark \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|get-stream-id|example}
+{generate_code_example(python)|get-stream-id|example}
 
 </div>
 

templates/zerver/api/get-subscribed-streams.md

@@ -24,7 +24,7 @@ curl {{ api_url }}/v1/users/me/subscriptions \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|get-subscribed-streams|example}
+{generate_code_example(python)|get-subscribed-streams|example}
 
 </div>
 

templates/zerver/api/private-message.md

@@ -27,7 +27,7 @@ curl {{ api_url }}/v1/messages \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|private-message|example}
+{generate_code_example(python)|private-message|example}
 
 </div>
 

templates/zerver/api/register-queue.md

@@ -31,7 +31,7 @@ curl {{ api_url }}/v1/register \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|register-queue|example}
+{generate_code_example(python)|register-queue|example}
 
 </div>
 

templates/zerver/api/remove-subscriptions.md

@@ -36,7 +36,7 @@ administrative privileges.
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|remove-subscriptions|example}
+{generate_code_example(python)|remove-subscriptions|example}
 
 </div>
 

templates/zerver/api/render-message.md

@@ -25,7 +25,7 @@ curl {{ api_url }}/v1/messages/render \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|render-message|example}
+{generate_code_example(python)|render-message|example}
 
 </div>
 

templates/zerver/api/stream-message.md

@@ -29,7 +29,7 @@ curl {{ api_url }}/v1/messages \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|stream-message|example}
+{generate_code_example(python)|stream-message|example}
 
 </div>
 

templates/zerver/api/update-message.md

@@ -27,7 +27,7 @@ curl -X "PATCH" {{ api_url }}/v1/messages/<msg_id> \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example|update-message|example}
+{generate_code_example(python)|update-message|example}
 
 </div>
 

zerver/lib/bugdown/api_code_examples.py

@@ -11,9 +11,10 @@ import markdown
 
 import zerver.lib.api_test_helpers
 
-REGEXP = re.compile(r'\{generate_code_example\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+?)\s*\))?\}')
+MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+?)\s*\))?\}')
+CODE_EXAMPLE_REGEX = re.compile(r'\# \{code_example\|\s*(.+?)\s*\}')
 
-PYTHON_CLIENT_CONFIG_LINES = """
+PYTHON_CLIENT_CONFIG = """
 #!/usr/bin/env python3
 
 import zulip
@@ -33,6 +34,57 @@ client = zulip.Client(config_file="~/zuliprc-admin")
 
 """
 
+def extract_python_code_example(source: List[str], snippet: List[str]) -> List[str]:
+    start = -1
+    end = -1
+    for line in source:
+        match = CODE_EXAMPLE_REGEX.search(line)
+        if match:
+            if match.group(1) == 'start':
+                start = source.index(line)
+            elif match.group(1) == 'end':
+                end = source.index(line)
+                break
+
+    if (start == -1 and end == -1):
+        return snippet
+
+    snippet.extend(source[start + 1: end])
+    snippet.append('    print(result)')
+    snippet.append('\n')
+    source = source[end + 1:]
+    return extract_python_code_example(source, snippet)
+
+def render_python_code_example(function: str, admin_config: Optional[bool]=False) -> List[str]:
+    method = zerver.lib.api_test_helpers.TEST_FUNCTIONS[function]
+    function_source_lines = inspect.getsourcelines(method)[0]
+
+    if admin_config:
+        config = PYTHON_CLIENT_ADMIN_CONFIG.splitlines()
+    else:
+        config = PYTHON_CLIENT_CONFIG.splitlines()
+
+    snippet = extract_python_code_example(function_source_lines, [])
+
+    code_example = []
+    code_example.append('```python')
+    code_example.extend(config)
+
+    for line in snippet:
+        # Remove one level of indentation and strip newlines
+        code_example.append(line[4:].rstrip())
+
+    code_example.append('```')
+
+    return code_example
+
+SUPPORTED_LANGUAGES = {
+    'python': {
+        'client_config': PYTHON_CLIENT_CONFIG,
+        'admin_config': PYTHON_CLIENT_ADMIN_CONFIG,
+        'render': render_python_code_example,
+    }
+}  # type: Dict[str, Any]
 
 class APICodeExamplesGenerator(Extension):
     def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
@@ -50,12 +102,13 @@ class APICodeExamplesPreprocessor(Preprocessor):
         while not done:
             for line in lines:
                 loc = lines.index(line)
-                match = REGEXP.search(line)
+                match = MACRO_REGEXP.search(line)
 
                 if match:
-                    function = match.group(1)
+                    language = match.group(2)
-                    key = match.group(2)
+                    function = match.group(3)
-                    argument = match.group(4)
+                    key = match.group(4)
+                    argument = match.group(6)
 
                     if key == 'fixture':
                         if argument:
@@ -64,15 +117,15 @@ class APICodeExamplesPreprocessor(Preprocessor):
                             text = self.render_fixture(function)
                     elif key == 'example':
                         if argument == 'admin_config=True':
-                            text = self.render_code_example(function, admin_config=True)
+                            text = SUPPORTED_LANGUAGES[language]['render'](function, admin_config=True)
                         else:
-                            text = self.render_code_example(function)
+                            text = SUPPORTED_LANGUAGES[language]['render'](function)
 
                     # The line that contains the directive to include the macro
                     # may be preceded or followed by text or tags, in that case
                     # we need to make sure that any preceding or following text
                     # stays the same.
-                    line_split = REGEXP.split(line, maxsplit=0)
+                    line_split = MACRO_REGEXP.split(line, maxsplit=0)
                     preceding = line_split[0]
                     following = line_split[-1]
                     text = [preceding] + text + [following]
@@ -98,52 +151,5 @@ class APICodeExamplesPreprocessor(Preprocessor):
 
         return fixture
 
-    def render_code_example(self, function: str, admin_config: Optional[bool]=False) -> List[str]:
-        method = zerver.lib.api_test_helpers.TEST_FUNCTIONS[function]
-        function_source_lines = inspect.getsourcelines(method)[0]
-
-        if admin_config:
-            config = PYTHON_CLIENT_ADMIN_CONFIG.splitlines()
-        else:
-            config = PYTHON_CLIENT_CONFIG_LINES.splitlines()
-
-        snippet = self.extractCodeExample(function_source_lines, [])
-
-        code_example = []
-        code_example.append('```python')
-        code_example.extend(config)
-
-        for line in snippet:
-            # Remove one level of indentation and strip newlines
-            code_example.append(line[4:].rstrip())
-
-        code_example.append('```')
-
-        return code_example
-
-    def extractCodeExample(self, source: List[str], snippet: List[str]) -> List[str]:
-        ce_regex = re.compile(r'\# \{code_example\|\s*(.+?)\s*\}')
-
-        start = -1
-        end = -1
-        for line in source:
-            match = ce_regex.search(line)
-            if match:
-                if match.group(1) == 'start':
-                    start = source.index(line)
-                elif match.group(1) == 'end':
-                    end = source.index(line)
-                    break
-
-        if (start == -1 and end == -1):
-            return snippet
-
-        snippet.extend(source[start + 1: end])
-        snippet.append('    print(result)')
-        snippet.append('\n')
-        source = source[end + 1:]
-        return self.extractCodeExample(source, snippet)
-
-
 def makeExtension(*args: Any, **kwargs: str) -> APICodeExamplesGenerator:
     return APICodeExamplesGenerator(kwargs)