webhooks/freshdesk/doc.md: Render example JSON correctly.

We've been getting reports from users that our Freshdesk webhook
isn't working correctly. It turns out that the issue had nothing
to do with the webhook implementation itself!

In freshdesk/doc.md, we have a JSON template we ask users to
copy/paste into a textbox in the Freshdesk UI. That JSON template
contains "{{" and "}}" characters which we escaped as Unicode
decimals to prevent clashes with Jinja2 syntax in other parts
of the same template. This worked for a while!

But thanks to the changes introduced as part of the
nested_code_blocks extension, such escaped characters were never
decoded, leading users to copy/paste the same template but with
raw escaped unicode representations of "{{" and "}}" inside. And
that eventually broke our webhook implementation.

This commit makes sure that such characters are properly "unescaped",
just for Freshdesk docs.

templates/tests/test_unicode_decimals.html

@@ -0,0 +1,5 @@
+header
+
+{{ render_markdown_path("zerver/tests/markdown/test_unicode_decimals.md", {"unescape_rendered_html": unescape_rendered_html}) }}
+
+footer

templates/zerver/tests/markdown/test_unicode_decimals.md

@@ -0,0 +1 @@
+{}

zerver/templatetags/app_filters.py

@@ -1,4 +1,5 @@
 import os
+from html import unescape
 from typing import Any, Dict, List, Optional
 
 import markdown
@@ -141,4 +142,13 @@ def render_markdown_path(markdown_file_path: str,
     html = md_engine.convert(markdown_string)
     rendered_html = jinja.from_string(html).render(context)
 
+    if context.get('unescape_rendered_html', False):
+        # In some exceptional cases (such as our Freshdesk webhook docs),
+        # code blocks in some of our Markdown templates have characters such
+        # as '{' encoded as '{' to prevent clashes with Jinja2 syntax,
+        # but the encoded form never gets decoded because the text ends up
+        # inside a <pre> tag. So here, we explicitly "unescape" such characters
+        # if 'unescape_rendered_html' is True.
+        rendered_html = unescape(rendered_html)
+
     return mark_safe(rendered_html)

zerver/tests/test_templates.py

@@ -216,6 +216,22 @@ class TemplateTestCase(ZulipTestCase):
         self.assertEqual(content_sans_whitespace,
                          'header<h1id="hello">Hello!</h1><p>Thisissome<em>boldtext</em>.</p>footer')
 
+    def test_encoded_unicode_decimals_in_markdown_template(self) -> None:
+        template = get_template("tests/test_unicode_decimals.html")
+        context = {'unescape_rendered_html': False}
+        content = template.render(context)
+
+        content_sans_whitespace = content.replace(" ", "").replace('\n', '')
+        self.assertEqual(content_sans_whitespace,
+                         'header<p>&#123;&#125;</p>footer')
+
+        context = {'unescape_rendered_html': True}
+        content = template.render(context)
+
+        content_sans_whitespace = content.replace(" ", "").replace('\n', '')
+        self.assertEqual(content_sans_whitespace,
+                         'header<p>{}</p>footer')
+
     def test_markdown_nested_code_blocks(self) -> None:
         template = get_template("tests/test_markdown.html")
         context = {

zerver/views/integrations.py

@@ -146,6 +146,14 @@ def integration_doc(request: HttpRequest, integration_name: str=REQ(default=None
         context['hubot_docs_url'] = integration.hubot_docs_url
     if isinstance(integration, EmailIntegration):
         context['email_gateway_example'] = settings.EMAIL_GATEWAY_EXAMPLE
+    if integration.name == 'freshdesk':
+        # In our Freshdesk docs, some nested code blocks have characters such
+        # as '{' encoded as '{' to prevent clashes with Jinja2 syntax,
+        # but the encoded form never gets rendered because the text ends up
+        # inside a <pre> tag. So here, we explicitly set a directive that
+        # a particular template should be "unescaped" before being displayed.
+        # Note that this value is used by render_markdown_path.
+        context['unescape_rendered_html'] = True
 
     doc_html_str = render_markdown_path(integration.doc, context)