markdown: Add TeX typesetting support.

Co-authored-by: Reid Barton <rwbarton@gmail.com>

Fixes #2056.

.eslintrc.json

@@ -114,7 +114,8 @@
         "user_events": false,
         "Plotly": false,
         "emoji_codes": false,
-        "drafts": false
+        "drafts": false,
+        "katex": false
     },
     "rules": {
         "no-restricted-syntax": 0,

frontend_tests/node_tests/echo.js

@@ -37,6 +37,7 @@ add_dependencies({
     hash_util: 'js/hash_util',
     hashchange: 'js/hashchange',
     fenced_code: 'js/fenced_code.js',
+    katex: 'node_modules/katex/dist/katex.min.js',
 });
 
 var doc = "";

package.json

@@ -13,6 +13,7 @@
     "i18next-localstorage-cache": "0.3.0",
     "jquery": "1.12.1",
     "jquery-validation": "1.16.0",
+    "katex": "0.7.1",
     "plotly.js": "1.19.2",
     "string.prototype.codepointat": "0.2.0",
     "underscore": "1.8.3",

static/js/echo.js

@@ -373,6 +373,14 @@ function handleRealmFilter(pattern, matches) {
     return url;
 }
 
+function handleTex(tex, fullmatch) {
+    try {
+        return katex.renderToString(tex);
+    } catch (ex) {
+        return '<span class="tex-error">' + escape(fullmatch) + '</span>';
+    }
+}
+
 function python_to_js_filter(pattern, url) {
     // Converts a python named-group regex to a javascript-compatible numbered
     // group regex... with a regex!
@@ -514,6 +522,7 @@ $(function () {
         unicodeEmojiHandler: handleUnicodeEmoji,
         streamHandler: handleStream,
         realmFilterHandler: handleRealmFilter,
+        texHandler: handleTex,
         renderer: r,
         preprocessors: [preprocess_code_blocks],
     });

static/js/fenced_code.js

@@ -52,6 +52,16 @@ function wrap_quote(text) {
     return quoted_paragraphs.join('\n\n');
 }
 
+function wrap_tex(tex) {
+    try {
+        return katex.renderToString(tex, {
+            displayMode: true,
+        });
+    } catch (ex) {
+        return '<span class="tex-error">' + escape_func(tex) + '</span>';
+    }
+}
+
 exports.set_stash_func = function (stash_handler) {
     stash_func = stash_handler;
 };
@@ -90,6 +100,28 @@ exports.process_fenced_code = function (content) {
                     },
                 };
             }
+
+            if (lang === 'math' || lang === 'tex' || lang === 'latex') {
+                return {
+                    handle_line: function (line) {
+                        if (line === fence) {
+                            this.done();
+                        } else {
+                            consume_line(lines, line);
+                        }
+                    },
+
+                    done: function () {
+                        var text = wrap_tex(lines.join('\n'));
+                        var placeholder = stash_func(text, true);
+                        output_lines.push('');
+                        output_lines.push(placeholder);
+                        output_lines.push('');
+                        handler_stack.pop();
+                    },
+                };
+            }
+
             return {
                 handle_line: function (line) {
                     if (line === fence) {

static/styles/zulip.css

@@ -1931,6 +1931,15 @@ div.floating_recipient {
     font-weight: bold;
 }
 
+.katex-html {
+    line-height: initial;
+    white-space: initial;
+}
+
+.tex-error {
+    color: red;
+}
+
 .popover {
     width: auto;
 }

static/third/katex/cli.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+/*
+The MIT License (MIT)
+
+Copyright (c) 2015 Khan Academy
+
+This software also uses portions of the underscore.js project, which is
+MIT licensed with the following copyright:
+
+Copyright (c) 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative
+Reporters & Editors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Simple CLI for KaTeX.
+// Reads TeX from stdin, outputs HTML to stdout.
+
+let katex;
+try {
+    // Attempt to import KaTeX from the production bundle
+    katex = require("../../serve/min/katex.js");
+} catch (ex) {
+    // Import KaTeX from node_modules (development environment) otherwise
+    katex = require("../../node_modules/katex/katex.js");
+}
+
+let input = "";
+
+// Skip the first two args, which are just "node" and "cli.js"
+const args = process.argv.slice(2);
+
+if (args.indexOf("--help") !== -1) {
+    console.log(process.argv[0] + " " + process.argv[1] +
+                " [ --help ]" +
+                " [ --display-mode ]");
+
+    console.log("\n" +
+                "Options:");
+    console.log("  --help            Display this help message");
+    console.log("  --display-mode    Render in display mode (not inline mode)");
+    process.exit();
+}
+
+process.stdin.on("data", function(chunk) {
+    input += chunk.toString();
+});
+
+process.stdin.on("end", function() {
+    var options = { displayMode: args.indexOf("--display-mode") !== -1 };
+    var output = katex.renderToString(input, options);
+    console.log(output);
+});

static/third/marked/lib/marked.js

@@ -479,9 +479,10 @@ var inline = {
   usermention: noop,
   stream: noop,
   avatar: noop,
+  tex: noop,
   gravatar: noop,
   realm_filters: [],
-  text: /^[\s\S]+?(?=[\\<!\[_*`]| {2,}\n|$)/
+  text: /^[\s\S]+?(?=[\\<!\[_*`$]| {2,}\n|$)/
 };
 
 inline._inside = /(?:\[[^\]]*\]|[^\[\]]|\](?=[^\[]*\]))*/;
@@ -541,6 +542,7 @@ inline.zulip = merge({}, inline.breaks, {
   stream: /^#\*\*([^\*]+)\*\*/m,
   avatar: /^!avatar\(([^)]+)\)/,
   gravatar: /^!gravatar\(([^)]+)\)/,
+  tex: /^(\$\$([^ _$](\\\$|[^$])*)(?! )\$\$)\B/,
   realm_filters: [],
   text: replace(inline.breaks.text)
     ('|', '|(\ud83c[\udf00-\udfff]|\ud83d[\udc00-\ude4f]|\ud83d[\ude80-\udeff]|[\u2600-\u26FF]|[\u2700-\u27BF])|')
@@ -797,6 +799,13 @@ InlineLexer.prototype.output = function(src) {
       continue;
     }
 
+    // tex
+    if (cap = this.rules.tex.exec(src)) {
+      src = src.substring(cap[0].length);
+      out += this.tex(cap[2], cap[0]);
+      continue;
+    }
+
     // text
     if (cap = this.rules.text.exec(src)) {
       src = src.substring(cap[0].length);
@@ -838,6 +847,12 @@ InlineLexer.prototype.unicodeEmoji = function (name) {
   return this.options.unicodeEmojiHandler(name);
 };
 
+InlineLexer.prototype.tex = function (tex, fullmatch) {
+  if (typeof this.options.texHandler !== 'function')
+    return fullmatch;
+  return this.options.texHandler(tex, fullmatch);
+};
+
 InlineLexer.prototype.userAvatar = function (email) {
   if (typeof this.options.avatarHandler !== 'function')
     return '!avatar(' + email + ')';
@@ -850,7 +865,6 @@ InlineLexer.prototype.userGravatar = function (email) {
   return this.options.avatarHandler(email);
 };
 
-
 InlineLexer.prototype.realm_filter = function (filter, matches, orig) {
   if (typeof this.options.realmFilterHandler !== 'function')
     return;

templates/zerver/help/format-your-message-using-markdown.md

@@ -143,3 +143,25 @@ a space before your phrase or submit it as a quote block by following
 the code syntax highlighting format.
 
 ![Quotes](/static/images/help/quotes-screenshot.png)
+
+## TeX math
+
+You can display mathematical symbols, expressions and equations using Zulip's
+[TeX](http://www.tug.org/interest.html#doc) typesetting implementation,
+based on [KaTeX](https://github.com/Khan/KaTeX).
+
+!!! tip ""
+    Visit the [KaTeX Wiki](https://github.com/Khan/KaTeX/wiki/Function-Support-in-KaTeX)
+    to view a complete of compatible commands.
+
+Surround elements in valid TeX syntax with `$$two dollar signs$$` to display it
+as inline content.
+
+![Inline TeX](/static/images/help/inline-tex-screenshot.png)
+
+Also, you can show expressions, such as expanded integrals, in TeX
+*display mode* to present them fully-sized in the center of the messages by
+fencing them with three back-ticks ` ``` ` or tildes `~~~`, with **math**,
+**tex** or **latex** immediately after the first set of back-ticks.
+
+![Display mode TeX](/static/images/help/display-mode-tex-screenshot.png)

tools/update-prod-static

@@ -46,6 +46,22 @@ subprocess.check_call(['./tools/minify-js'] +
                       (['--prev-deploy', prev_deploy] if prev_deploy else []),
                       stdout=fp, stderr=fp)
 
+# Copy the KaTeX files outside node_modules
+subprocess.check_call(['mkdir', '-p',
+                       os.path.join(settings.STATIC_ROOT,
+                                    'node_modules/katex/dist/')],
+                      stdout=fp, stderr=fp)
+
+subprocess.check_call(['cp', 'node_modules/katex/dist/katex.css',
+                       os.path.join(settings.STATIC_ROOT,
+                                    'node_modules/katex/dist/')],
+                      stdout=fp, stderr=fp)
+
+subprocess.check_call(['cp', '-R', 'node_modules/katex/dist/fonts',
+                       os.path.join(settings.STATIC_ROOT,
+                                    'node_modules/katex/dist/fonts')],
+                      stdout=fp, stderr=fp)
+
 # Collect the files that we're going to serve; this creates prod-static/serve.
 subprocess.check_call(['./manage.py', 'collectstatic', '--no-default-ignore',
                        '--noinput', '-i', 'assets', '-i' 'node_modules'],

version.py

@@ -1,2 +1,2 @@
 ZULIP_VERSION = "1.5.1+git"
-PROVISION_VERSION = '4.11'
+PROVISION_VERSION = '4.12'

zerver/lib/bugdown/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import subprocess
 # Zulip's main markdown implementation.  See docs/markdown.md for
 # detailed documentation on our markdown syntax.
 from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Text, Tuple, TypeVar, Union
@@ -40,7 +41,8 @@ from zerver.lib.url_preview import preview as link_preview
 from zerver.models import Message, Realm, UserProfile, get_user_profile_by_email
 import zerver.lib.alert_words as alert_words
 import zerver.lib.mention as mention
-from zerver.lib.str_utils import force_text, force_str
+from zerver.lib.str_utils import force_str, force_text
+from zerver.lib.tex import render_tex
 import six
 from six.moves import range, html_parser
 from typing import Text
@@ -762,6 +764,18 @@ class ModalLink(markdown.inlinepatterns.Pattern):
 
         return a_tag
 
+class Tex(markdown.inlinepatterns.Pattern):
+    def handleMatch(self, match):
+        # type: (Match[Text]) -> Element
+        rendered = render_tex(match.group('body'), is_inline=True)
+        if rendered is not None:
+            return etree.fromstring(rendered.encode('utf-8'))
+        else: # Something went wrong while rendering
+            span = markdown.util.etree.Element('span')
+            span.set('class', 'tex-error')
+            span.text = '$$' + match.group('body') + '$$'
+            return span
+
 upload_title_re = re.compile(u"^(https?://[^/]*)?(/user_uploads/\\d+)(/[^/]*)?/[^/]*/(?P<filename>[^/]*)$")
 def url_filename(url):
     # type: (Text) -> Text
@@ -1146,6 +1160,7 @@ class Bugdown(markdown.Extension):
                         \*\*                         # ends by double asterisks
                        """
         md.inlinePatterns.add('stream', StreamPattern(stream_group), '>backtick')
+        md.inlinePatterns.add('tex', Tex(r'\B\$\$(?P<body>[^ _$](\\\$|[^$])*)(?! )\$\$\B'), '>backtick')
         md.inlinePatterns.add('emoji', Emoji(r'(?P<syntax>:[\w\-\+]+:)'), '_end')
         md.inlinePatterns.add('unicodeemoji', UnicodeEmoji(
             u'(?P<syntax>[\U0001F300-\U0001F64F\U0001F680-\U0001F6FF\u2600-\u26FF\u2700-\u27BF])'),

zerver/lib/bugdown/fenced_code.py

@@ -62,8 +62,13 @@ Dependencies:
 """
 
 import re
+import subprocess
 import markdown
+import six
+from django.utils.html import escape
 from markdown.extensions.codehilite import CodeHilite, CodeHiliteExtension
+from zerver.lib.str_utils import force_bytes
+from zerver.lib.tex import render_tex
 from typing import Any, Dict, Iterable, List, MutableSequence, Optional, Tuple, Union, Text
 
 # Global vars
@@ -170,9 +175,37 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
             # type: (MutableSequence[Text], Text, Text) -> BaseHandler
             if lang in ('quote', 'quoted'):
                 return QuoteHandler(output, fence)
+            elif lang in ('math', 'tex', 'latex'):
+                return TexHandler(output, fence)
             else:
                 return CodeHandler(output, fence, lang)
 
+        class CodeHandler(BaseHandler):
+            def __init__(self, output, fence, lang):
+                # type: (MutableSequence[Text], Text, Text) -> None
+                self.output = output
+                self.fence = fence
+                self.lang = lang
+                self.lines = [] # type: List[Text]
+
+            def handle_line(self, line):
+                # type: (Text) -> None
+                if line.rstrip() == self.fence:
+                    self.done()
+                else:
+                    self.lines.append(line)
+
+            def done(self):
+                # type: () -> None
+                text = '\n'.join(self.lines)
+                text = processor.format_code(self.lang, text)
+                text = processor.placeholder(text)
+                processed_lines = text.split('\n')
+                self.output.append('')
+                self.output.extend(processed_lines)
+                self.output.append('')
+                pop()
+
         class QuoteHandler(BaseHandler):
             def __init__(self, output, fence):
                 # type: (MutableSequence[Text], Text) -> None
@@ -197,12 +230,11 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
                 self.output.append('')
                 pop()
 
-        class CodeHandler(BaseHandler):
-            def __init__(self, output, fence, lang):
-                # type: (MutableSequence[Text], Text, Text) -> None
+        class TexHandler(BaseHandler):
+            def __init__(self, output, fence):
+                # type: (MutableSequence[Text], Text) -> None
                 self.output = output
                 self.fence = fence
-                self.lang = lang
                 self.lines = [] # type: List[Text]
 
             def handle_line(self, line):
@@ -210,12 +242,12 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
                 if line.rstrip() == self.fence:
                     self.done()
                 else:
-                    self.lines.append(line)
+                    check_for_new_fence(self.lines, line)
 
             def done(self):
                 # type: () -> None
                 text = '\n'.join(self.lines)
-                text = processor.format_code(self.lang, text)
+                text = processor.format_tex(text)
                 text = processor.placeholder(text)
                 processed_lines = text.split('\n')
                 self.output.append('')
@@ -282,6 +314,19 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
             quoted_paragraphs.append("\n".join("> " + line for line in lines if line != ''))
         return "\n\n".join(quoted_paragraphs)
 
+    def format_tex(self, text):
+        # type: (Text) -> Text
+        paragraphs = text.split("\n\n")
+        tex_paragraphs = []
+        for paragraph in paragraphs:
+            html = render_tex(paragraph, is_inline=False)
+            if html is not None:
+                tex_paragraphs.append(html)
+            else:
+                tex_paragraphs.append('<span class="tex-error">' +
+                                      escape(paragraph) + '</span>')
+        return "\n\n".join(tex_paragraphs)
+
     def placeholder(self, code):
         # type: (Text) -> Text
         return self.markdown.htmlStash.store(code, safe=True)

zerver/lib/tex.py

@@ -0,0 +1,41 @@
+from __future__ import absolute_import
+
+import logging
+import os
+import subprocess
+from django.conf import settings
+from typing import Text
+from zerver.lib.str_utils import force_bytes
+
+def render_tex(tex, is_inline=True):
+    # type: (Text, bool) -> Text
+    """Render a TeX string into HTML using KaTeX
+
+    Returns the HTML string, or None if there was some error in the TeX syntax
+
+    Keyword arguments:
+    tex -- Text string with the TeX to render
+           Don't include delimiters ('$$', '\[ \]', etc.)
+    is_inline -- Boolean setting that indicates whether the render should be
+                 inline (i.e. for embedding it in text) or not. The latter
+                 will show the content centered, and in the "expanded" form
+                 (default True)
+    """
+
+    katex_path = os.path.join(settings.STATIC_ROOT, 'third/katex/cli.js')
+    if not os.path.isfile(katex_path):
+        logging.error("Cannot find KaTeX for latex rendering!")
+        return None
+    command = ['node', katex_path]
+    if not is_inline:
+        command.extend(['--', '--display-mode'])
+    katex = subprocess.Popen(command,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+    stdout = katex.communicate(input=force_bytes(tex))[0]
+    if katex.returncode == 0:
+        # stdout contains a newline at the end
+        return stdout.decode('utf-8').strip()
+    else:
+        return None

zproject/settings.py

@@ -715,6 +715,7 @@ PIPELINE = {
                 'third/bootstrap-notify/css/bootstrap-notify.css',
                 'third/spectrum/spectrum.css',
                 'third/jquery-perfect-scrollbar/css/perfect-scrollbar.css',
+                'node_modules/katex/dist/katex.css',
                 'styles/components.css',
                 'styles/zulip.css',
                 'styles/settings.css',
@@ -927,6 +928,12 @@ JS_SPECS = {
         'source_filenames': ['third/sockjs/sockjs-0.3.4.js'],
         'output_filename': 'min/sockjs-0.3.4.min.js'
     },
+    'katex': {
+        'source_filenames': [
+            'node_modules/katex/dist/katex.js',
+        ],
+        'output_filename': 'min/katex.js'
+    }
 }
 
 if PIPELINE_ENABLED: