diff --git a/.eslintrc.json b/.eslintrc.json index 24e43ec51a..563180173d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -114,7 +114,8 @@ "user_events": false, "Plotly": false, "emoji_codes": false, - "drafts": false + "drafts": false, + "katex": false }, "rules": { "no-restricted-syntax": 0, diff --git a/frontend_tests/node_tests/echo.js b/frontend_tests/node_tests/echo.js index d7cf64dc52..e105189eff 100644 --- a/frontend_tests/node_tests/echo.js +++ b/frontend_tests/node_tests/echo.js @@ -37,6 +37,7 @@ add_dependencies({ hash_util: 'js/hash_util', hashchange: 'js/hashchange', fenced_code: 'js/fenced_code.js', + katex: 'node_modules/katex/dist/katex.min.js', }); var doc = ""; diff --git a/package.json b/package.json index f7cc3ff0b4..2e462c8870 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "i18next-localstorage-cache": "0.3.0", "jquery": "1.12.1", "jquery-validation": "1.16.0", + "katex": "0.7.1", "plotly.js": "1.19.2", "string.prototype.codepointat": "0.2.0", "underscore": "1.8.3", diff --git a/static/images/help/display-mode-tex-screenshot.png b/static/images/help/display-mode-tex-screenshot.png new file mode 100644 index 0000000000..eec72f23f7 Binary files /dev/null and b/static/images/help/display-mode-tex-screenshot.png differ diff --git a/static/images/help/inline-tex-screenshot.png b/static/images/help/inline-tex-screenshot.png new file mode 100644 index 0000000000..efc4a72175 Binary files /dev/null and b/static/images/help/inline-tex-screenshot.png differ diff --git a/static/js/echo.js b/static/js/echo.js index 7c321a7390..1243a5a5f0 100644 --- a/static/js/echo.js +++ b/static/js/echo.js @@ -373,6 +373,14 @@ function handleRealmFilter(pattern, matches) { return url; } +function handleTex(tex, fullmatch) { + try { + return katex.renderToString(tex); + } catch (ex) { + return '' + escape(fullmatch) + ''; + } +} + function python_to_js_filter(pattern, url) { // Converts a python named-group regex to a javascript-compatible numbered // group regex... with a regex! @@ -514,6 +522,7 @@ $(function () { unicodeEmojiHandler: handleUnicodeEmoji, streamHandler: handleStream, realmFilterHandler: handleRealmFilter, + texHandler: handleTex, renderer: r, preprocessors: [preprocess_code_blocks], }); diff --git a/static/js/fenced_code.js b/static/js/fenced_code.js index 2c045f8c01..3daac4929c 100644 --- a/static/js/fenced_code.js +++ b/static/js/fenced_code.js @@ -52,6 +52,16 @@ function wrap_quote(text) { return quoted_paragraphs.join('\n\n'); } +function wrap_tex(tex) { + try { + return katex.renderToString(tex, { + displayMode: true, + }); + } catch (ex) { + return '' + escape_func(tex) + ''; + } +} + exports.set_stash_func = function (stash_handler) { stash_func = stash_handler; }; @@ -90,6 +100,28 @@ exports.process_fenced_code = function (content) { }, }; } + + if (lang === 'math' || lang === 'tex' || lang === 'latex') { + return { + handle_line: function (line) { + if (line === fence) { + this.done(); + } else { + consume_line(lines, line); + } + }, + + done: function () { + var text = wrap_tex(lines.join('\n')); + var placeholder = stash_func(text, true); + output_lines.push(''); + output_lines.push(placeholder); + output_lines.push(''); + handler_stack.pop(); + }, + }; + } + return { handle_line: function (line) { if (line === fence) { diff --git a/static/styles/zulip.css b/static/styles/zulip.css index b16e74f4f2..1372e636c3 100644 --- a/static/styles/zulip.css +++ b/static/styles/zulip.css @@ -1931,6 +1931,15 @@ div.floating_recipient { font-weight: bold; } +.katex-html { + line-height: initial; + white-space: initial; +} + +.tex-error { + color: red; +} + .popover { width: auto; } diff --git a/static/third/katex/cli.js b/static/third/katex/cli.js new file mode 100644 index 0000000000..b99f825626 --- /dev/null +++ b/static/third/katex/cli.js @@ -0,0 +1,70 @@ +#!/usr/bin/env node + +/* +The MIT License (MIT) + +Copyright (c) 2015 Khan Academy + +This software also uses portions of the underscore.js project, which is +MIT licensed with the following copyright: + +Copyright (c) 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative +Reporters & Editors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Simple CLI for KaTeX. +// Reads TeX from stdin, outputs HTML to stdout. + +let katex; +try { + // Attempt to import KaTeX from the production bundle + katex = require("../../serve/min/katex.js"); +} catch (ex) { + // Import KaTeX from node_modules (development environment) otherwise + katex = require("../../node_modules/katex/katex.js"); +} + +let input = ""; + +// Skip the first two args, which are just "node" and "cli.js" +const args = process.argv.slice(2); + +if (args.indexOf("--help") !== -1) { + console.log(process.argv[0] + " " + process.argv[1] + + " [ --help ]" + + " [ --display-mode ]"); + + console.log("\n" + + "Options:"); + console.log(" --help Display this help message"); + console.log(" --display-mode Render in display mode (not inline mode)"); + process.exit(); +} + +process.stdin.on("data", function(chunk) { + input += chunk.toString(); +}); + +process.stdin.on("end", function() { + var options = { displayMode: args.indexOf("--display-mode") !== -1 }; + var output = katex.renderToString(input, options); + console.log(output); +}); diff --git a/static/third/marked/lib/marked.js b/static/third/marked/lib/marked.js index bd7acf3ba9..2189ecb612 100644 --- a/static/third/marked/lib/marked.js +++ b/static/third/marked/lib/marked.js @@ -479,9 +479,10 @@ var inline = { usermention: noop, stream: noop, avatar: noop, + tex: noop, gravatar: noop, realm_filters: [], - text: /^[\s\S]+?(?=[\\!_stream_subscribe_button(streamname)

", "bugdown_matches_marked": true + }, + { + "name": "tex_inline", + "input": "$$1 \\oplus 0 = 1$$", + "expected_output": "

1⊕0=11 \\oplus 0 = 11⊕0=1

", + "bugdown_matches_marked": false + }, + { + "name": "tex_complex", + "input": "$$\\Phi_E = \\oint E \\cdot dA$$", + "expected_output": "

ΦE=∮E⋅dA\\Phi_E = \\oint E \\cdot dAΦ​E​​=∮E⋅dA

", + "bugdown_matches_marked": false + }, + { + "name": "tex_escaped", + "input": "`$$a$$`", + "expected_output": "

$$a$$

", + "bugdown_matches_marked": true + }, + { + "name": "tex_fenced_math", + "input": "```math\na^2 = b^2 + c^2\n```", + "expected_output": "

a2=b2+c2a^2 = b^2 + c^2a​2​​=b​2​​+c​2​​

", + "bugdown_matches_marked": false + }, + { + "name": "tex_fenced_tex", + "input": "```tex\n\n\\pi \\textbf{ is not } 3.14\n```", + "expected_output": "

π is not 3.14\n\\pi \\textbf{ is not } 3.14π is not 3.14

", + "bugdown_matches_marked": false + }, + { + "name": "tex_fenced_latex", + "input": "```latex\n\n\\frac{1}{\\sqrt{2}}\n\\begin{bmatrix}\n1 & 1 \\\\\n1 & -1\n\\end{bmatrix}\n```", + "expected_output": "

12[111−1]\n\\frac{1}{\\sqrt{2}}\n\\begin{bmatrix}\n1 & 1 \\\\\n1 & -1\n\\end{bmatrix}​√​2​​​​​1​​[​1​1​​​1​−1​​]

", + "bugdown_matches_marked": false + }, + { + "name": "tex_money", + "input": "Tickets are $5 to $20 for youth, $10-$30 for adults, so we are hoping to bring in $500 from the event ($$x \\approx 500\\$$$)", + "expected_output": "

Tickets are $5 to $20 for youth, $10-$30 for adults, so we are hoping to bring in $500 from the event (x≈500$x \\approx 500\\$x≈500$)

", + "bugdown_matches_marked": false + }, + { + "name": "tex_dollar_smiley", + "input": "$$_$$ is a fun money-related smiley!", + "expected_output": "

$$_$$ is a fun money-related smiley!

", + "bugdown_matches_marked": true + }, + { + "name": "tex_multiple_dollars", + "input": "We are going to make some $$$ or maybe even $$$$!", + "expected_output": "

We are going to make some $$$ or maybe even $$$$!

", + "bugdown_matches_marked": true + }, + { + "name": "tex_non_matching_dollar_count", + "input": "$foo is$$", + "expected_output": "

$foo is$$

", + "bugdown_matches_marked": true + }, + { + "name": "tex_safe_script_tag", + "input": "$$$$\n\n~~~math\n\n~~~", + "expected_output": "

<scripttype=′text/javascript′>alert(′xss′);</script><script type='text/javascript'>alert('xss');</script><scripttype=​′​​text/javascript​′​​>alert(​′​​xss​′​​);</script>

\n

<scripttype=′text/javascript′>alert(′xss′);</script><script type='text/javascript'>alert('xss');</script><scripttype=​′​​text/javascript​′​​>alert(​′​​xss​′​​);</script>

", + "bugdown_matches_marked": false + }, + { + "name": "tex_error_safe_script_tag", + "input": "$$\\$$\n\n~~~math\n\\\n~~~", + "expected_output": "

$$\\<script type='text/javascript'>alert('xss');</script>$$

\n

\\<script type='text/javascript'>alert('xss');</script>

", + "bugdown_matches_marked": false } ], "linkify_tests": [ diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index a6cacf534a..0c89810360 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import subprocess # Zulip's main markdown implementation. See docs/markdown.md for # detailed documentation on our markdown syntax. from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Text, Tuple, TypeVar, Union @@ -40,7 +41,8 @@ from zerver.lib.url_preview import preview as link_preview from zerver.models import Message, Realm, UserProfile, get_user_profile_by_email import zerver.lib.alert_words as alert_words import zerver.lib.mention as mention -from zerver.lib.str_utils import force_text, force_str +from zerver.lib.str_utils import force_str, force_text +from zerver.lib.tex import render_tex import six from six.moves import range, html_parser from typing import Text @@ -762,6 +764,18 @@ class ModalLink(markdown.inlinepatterns.Pattern): return a_tag +class Tex(markdown.inlinepatterns.Pattern): + def handleMatch(self, match): + # type: (Match[Text]) -> Element + rendered = render_tex(match.group('body'), is_inline=True) + if rendered is not None: + return etree.fromstring(rendered.encode('utf-8')) + else: # Something went wrong while rendering + span = markdown.util.etree.Element('span') + span.set('class', 'tex-error') + span.text = '$$' + match.group('body') + '$$' + return span + upload_title_re = re.compile(u"^(https?://[^/]*)?(/user_uploads/\\d+)(/[^/]*)?/[^/]*/(?P[^/]*)$") def url_filename(url): # type: (Text) -> Text @@ -1146,6 +1160,7 @@ class Bugdown(markdown.Extension): \*\* # ends by double asterisks """ md.inlinePatterns.add('stream', StreamPattern(stream_group), '>backtick') + md.inlinePatterns.add('tex', Tex(r'\B\$\$(?P[^ _$](\\\$|[^$])*)(?! )\$\$\B'), '>backtick') md.inlinePatterns.add('emoji', Emoji(r'(?P:[\w\-\+]+:)'), '_end') md.inlinePatterns.add('unicodeemoji', UnicodeEmoji( u'(?P[\U0001F300-\U0001F64F\U0001F680-\U0001F6FF\u2600-\u26FF\u2700-\u27BF])'), diff --git a/zerver/lib/bugdown/fenced_code.py b/zerver/lib/bugdown/fenced_code.py index 3c4d2341f8..830ce0dd11 100644 --- a/zerver/lib/bugdown/fenced_code.py +++ b/zerver/lib/bugdown/fenced_code.py @@ -62,8 +62,13 @@ Dependencies: """ import re +import subprocess import markdown +import six +from django.utils.html import escape from markdown.extensions.codehilite import CodeHilite, CodeHiliteExtension +from zerver.lib.str_utils import force_bytes +from zerver.lib.tex import render_tex from typing import Any, Dict, Iterable, List, MutableSequence, Optional, Tuple, Union, Text # Global vars @@ -170,9 +175,37 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): # type: (MutableSequence[Text], Text, Text) -> BaseHandler if lang in ('quote', 'quoted'): return QuoteHandler(output, fence) + elif lang in ('math', 'tex', 'latex'): + return TexHandler(output, fence) else: return CodeHandler(output, fence, lang) + class CodeHandler(BaseHandler): + def __init__(self, output, fence, lang): + # type: (MutableSequence[Text], Text, Text) -> None + self.output = output + self.fence = fence + self.lang = lang + self.lines = [] # type: List[Text] + + def handle_line(self, line): + # type: (Text) -> None + if line.rstrip() == self.fence: + self.done() + else: + self.lines.append(line) + + def done(self): + # type: () -> None + text = '\n'.join(self.lines) + text = processor.format_code(self.lang, text) + text = processor.placeholder(text) + processed_lines = text.split('\n') + self.output.append('') + self.output.extend(processed_lines) + self.output.append('') + pop() + class QuoteHandler(BaseHandler): def __init__(self, output, fence): # type: (MutableSequence[Text], Text) -> None @@ -197,12 +230,11 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): self.output.append('') pop() - class CodeHandler(BaseHandler): - def __init__(self, output, fence, lang): - # type: (MutableSequence[Text], Text, Text) -> None + class TexHandler(BaseHandler): + def __init__(self, output, fence): + # type: (MutableSequence[Text], Text) -> None self.output = output self.fence = fence - self.lang = lang self.lines = [] # type: List[Text] def handle_line(self, line): @@ -210,12 +242,12 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): if line.rstrip() == self.fence: self.done() else: - self.lines.append(line) + check_for_new_fence(self.lines, line) def done(self): # type: () -> None text = '\n'.join(self.lines) - text = processor.format_code(self.lang, text) + text = processor.format_tex(text) text = processor.placeholder(text) processed_lines = text.split('\n') self.output.append('') @@ -282,6 +314,19 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): quoted_paragraphs.append("\n".join("> " + line for line in lines if line != '')) return "\n\n".join(quoted_paragraphs) + def format_tex(self, text): + # type: (Text) -> Text + paragraphs = text.split("\n\n") + tex_paragraphs = [] + for paragraph in paragraphs: + html = render_tex(paragraph, is_inline=False) + if html is not None: + tex_paragraphs.append(html) + else: + tex_paragraphs.append('' + + escape(paragraph) + '') + return "\n\n".join(tex_paragraphs) + def placeholder(self, code): # type: (Text) -> Text return self.markdown.htmlStash.store(code, safe=True) diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py new file mode 100644 index 0000000000..53f895c06c --- /dev/null +++ b/zerver/lib/tex.py @@ -0,0 +1,41 @@ +from __future__ import absolute_import + +import logging +import os +import subprocess +from django.conf import settings +from typing import Text +from zerver.lib.str_utils import force_bytes + +def render_tex(tex, is_inline=True): + # type: (Text, bool) -> Text + """Render a TeX string into HTML using KaTeX + + Returns the HTML string, or None if there was some error in the TeX syntax + + Keyword arguments: + tex -- Text string with the TeX to render + Don't include delimiters ('$$', '\[ \]', etc.) + is_inline -- Boolean setting that indicates whether the render should be + inline (i.e. for embedding it in text) or not. The latter + will show the content centered, and in the "expanded" form + (default True) + """ + + katex_path = os.path.join(settings.STATIC_ROOT, 'third/katex/cli.js') + if not os.path.isfile(katex_path): + logging.error("Cannot find KaTeX for latex rendering!") + return None + command = ['node', katex_path] + if not is_inline: + command.extend(['--', '--display-mode']) + katex = subprocess.Popen(command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout = katex.communicate(input=force_bytes(tex))[0] + if katex.returncode == 0: + # stdout contains a newline at the end + return stdout.decode('utf-8').strip() + else: + return None diff --git a/zproject/settings.py b/zproject/settings.py index 7f55afb95f..6e40406da9 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -715,6 +715,7 @@ PIPELINE = { 'third/bootstrap-notify/css/bootstrap-notify.css', 'third/spectrum/spectrum.css', 'third/jquery-perfect-scrollbar/css/perfect-scrollbar.css', + 'node_modules/katex/dist/katex.css', 'styles/components.css', 'styles/zulip.css', 'styles/settings.css', @@ -927,6 +928,12 @@ JS_SPECS = { 'source_filenames': ['third/sockjs/sockjs-0.3.4.js'], 'output_filename': 'min/sockjs-0.3.4.min.js' }, + 'katex': { + 'source_filenames': [ + 'node_modules/katex/dist/katex.js', + ], + 'output_filename': 'min/katex.js' + } } if PIPELINE_ENABLED: