From 0965c43238849a917abf3a4e5d6052d9bf94b0aa Mon Sep 17 00:00:00 2001 From: Cory Lynch Date: Wed, 26 Apr 2017 03:32:46 -0400 Subject: [PATCH] Add typeahead for syntax highlighting languages. Modified composebox_typeahead.js to recognize the triple backtick and tilde for code blocks, and added appropriate typeahead functions in that file and in typeahead_helper.js. Additionally, a new file pygments_data.js contains a dictionary of the supported languages, mapping to relative popularity rankings. These rankings determine the order of sort of the languages in the typeahead. This JavaScript file is actually in static/generated/pygments_data.js, as it is generated by a Python script, tools/build_pymgents_data.py. This is so that if Pygments adds support for new languages, the JavaScript file will be updated appropriately. This python script uses a set of popularity rankings defined in lang.json. Corresponding unit tests were also added. Fixes #4111. --- .eslintrc.json | 1 + .gitignore | 1 + docs/dev-setup-non-vagrant.md | 2 +- .../node_tests/composebox_typeahead.js | 19 ++++++- frontend_tests/node_tests/typeahead_helper.js | 10 ++++ static/js/composebox_typeahead.js | 54 +++++++++++++++++- static/js/typeahead_helper.js | 18 ++++++ tools/lib/provision.py | 1 + tools/setup/build_pygments_data.py | 34 +++++++++++ tools/setup/lang.json | 57 +++++++++++++++++++ tools/update-prod-static | 4 ++ version.py | 2 +- zproject/settings.py | 1 + 13 files changed, 199 insertions(+), 5 deletions(-) create mode 100755 tools/setup/build_pygments_data.py create mode 100644 tools/setup/lang.json diff --git a/.eslintrc.json b/.eslintrc.json index 3fc7b0476e..31500943ac 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -23,6 +23,7 @@ "attachments_ui": false, "csrf_token": false, "typeahead_helper": false, + "pygments_data": false, "popovers": false, "server_events": false, "ui": false, diff --git a/.gitignore b/.gitignore index 4d6bcefbf1..0f81a2685b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ coverage/ /zproject/dev-secrets.conf static/js/bundle.js static/generated/emoji +static/generated/pygments_data.js static/generated/github-contributors.json static/locale/language_options.json static/third/emoji-data diff --git a/docs/dev-setup-non-vagrant.md b/docs/dev-setup-non-vagrant.md index 864b495da6..6e54257894 100644 --- a/docs/dev-setup-non-vagrant.md +++ b/docs/dev-setup-non-vagrant.md @@ -321,6 +321,7 @@ Now run these commands: ``` ./tools/install-mypy ./tools/setup/emoji/build_emoji +./tools/setup/build_pygments_data.py ./scripts/setup/generate_secrets.py --development if [ $(uname) = "OpenBSD" ]; then sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/ @@ -461,4 +462,3 @@ the results in your browser. Currently, the Docker workflow is substantially less convenient than the Vagrant workflow and less documented; please contribute to this guide and the Docker tooling if you are using Docker to develop Zulip! - diff --git a/frontend_tests/node_tests/composebox_typeahead.js b/frontend_tests/node_tests/composebox_typeahead.js index 7d9df5d7de..6c43631f48 100644 --- a/frontend_tests/node_tests/composebox_typeahead.js +++ b/frontend_tests/node_tests/composebox_typeahead.js @@ -8,6 +8,9 @@ set_global('emoji', {emojis: emoji_list}); set_global('stream_data', {subscribed_subs: function () { return stream_list; }}); +set_global('pygments_data', {langs: + {python: 0, javscript: 1, html: 2, css: 3}, +}); global.stub_out_jquery(); add_dependencies({ @@ -46,7 +49,7 @@ global.people.add({ ct.split_at_cursor = function (word) { return [word, '']; }; var begin_typehead_this = {options: {completions: { - emoji: true, mention: true, stream: true}}}; + emoji: true, mention: true, stream: true, syntax: true}}}; function assert_typeahead_equals(input, reference) { var returned = ct.compose_content_begins_typeahead.call(begin_typehead_this, input); @@ -90,6 +93,17 @@ global.people.add({ assert_typeahead_equals("test #", false); assert_typeahead_equals("test #D", stream_list); assert_typeahead_equals("#s", stream_list); + + var lang_list = Object.keys(pygments_data.langs); + assert_typeahead_equals("``` ", false); + assert_typeahead_equals("test ``` py", false); + assert_typeahead_equals("test ```a", false); + assert_typeahead_equals("```b", lang_list); + assert_typeahead_equals("``c", false); + assert_typeahead_equals("``` d", lang_list); + assert_typeahead_equals("~~~e", lang_list); + assert_typeahead_equals("~~~ f", lang_list); + assert_typeahead_equals("test ~~~", false); }()); (function test_tokenizing() { @@ -102,6 +116,9 @@ global.people.add({ assert.equal(ct.tokenize_compose_str("foo bar :smil"), ":smil"); assert.equal(ct.tokenize_compose_str(":smil"), ":smil"); assert.equal(ct.tokenize_compose_str("foo @alice sm"), "@alice sm"); + assert.equal(ct.tokenize_compose_str("foo ```p"), ""); + assert.equal(ct.tokenize_compose_str("``` py"), "``` py"); + assert.equal(ct.tokenize_compose_str("foo``bar ~~~ py"), ""); // The following cases are kinda judgment calls... assert.equal(ct.tokenize_compose_str( diff --git a/frontend_tests/node_tests/typeahead_helper.js b/frontend_tests/node_tests/typeahead_helper.js index 1c7010e9b8..cc12f7e393 100644 --- a/frontend_tests/node_tests/typeahead_helper.js +++ b/frontend_tests/node_tests/typeahead_helper.js @@ -43,6 +43,16 @@ assert.deepEqual(test_streams[2].name, "Derp"); // Less subscribers assert.deepEqual(test_streams[3].name, "Dev"); // Alphabetically last assert.deepEqual(test_streams[4].name, "dead"); // Inactive streams last +set_global('pygments_data', {langs: + {python: 40, javscript: 50, php: 38, pascal: 29, perl: 22, css: 0}, +}); + +var test_langs = ["pascal", "perl", "php", "python", "javascript"]; +test_langs = typeahead_helper.sort_languages(test_langs, "p"); + +// Sort languages by matching first letter, and then by popularity +assert.deepEqual(test_langs, ["python", "php", "pascal", "perl", "javascript"]); + var matches = [ { email: "a_bot@zulip.com", diff --git a/static/js/composebox_typeahead.js b/static/js/composebox_typeahead.js index e041daf193..8f68e1dfab 100644 --- a/static/js/composebox_typeahead.js +++ b/static/js/composebox_typeahead.js @@ -48,6 +48,11 @@ function composebox_typeahead_highlighter(item) { return typeahead_helper.highlight_with_escaping(this.query, item); } +function query_matches_language(query, lang) { + query = query.toLowerCase(); + return lang.indexOf(query) !== -1; +} + function query_matches_person(query, person) { // Case-insensitive. query = query.toLowerCase(); @@ -212,6 +217,15 @@ exports.tokenize_compose_str = function (s) { while (i > min_i) { i -= 1; switch (s[i]) { + case '`': + case '~': + // Code block must start on a new line + if (i === 2) { + return s.slice(0); + } else if (i > 2 && s[i-3] === "\n") { + return s.slice(i-2); + } + break; case '#': case '@': case ':': @@ -234,6 +248,29 @@ exports.compose_content_begins_typeahead = function (query) { return false; } + // Start syntax highlighting autocompleter if the first three characters are ``` + var syntax_token = current_token.substring(0,3); + if (this.options.completions.syntax && (syntax_token === '```' || syntax_token === "~~~")) { + // Only autocomplete if user starts typing a language after ``` + if (current_token.length === 3) { + return false; + } + + // If the only input is a space, don't autocomplete + current_token = current_token.substring(3); + if (current_token === " ") { + return false; + } + + // Trim the first whitespace if it is there + if (current_token[0] === " ") { + current_token = current_token.substring(1); + } + this.completing = 'syntax'; + this.token = current_token; + return Object.keys(pygments_data.langs); + } + // Only start the emoji autocompleter if : is directly after one // of the whitespace or punctuation chars we split on. if (this.options.completions.emoji && current_token[0] === ':') { @@ -309,6 +346,8 @@ exports.content_highlighter = function (item) { return typeahead_helper.highlight_with_escaping(this.token, item_formatted); } else if (this.completing === 'stream') { return typeahead_helper.render_stream(this.token, item); + } else if (this.completing === 'syntax') { + return typeahead_helper.highlight_with_escaping(this.token, item); } }; @@ -332,18 +371,25 @@ exports.content_typeahead_selected = function (item) { beginning = (beginning.substring(0, beginning.length - this.token.length-1) + '#**' + item.name + '** '); $(document).trigger('streamname_completed.zulip', {stream: item}); + } else if (this.completing === 'syntax') { + rest = "\n" + beginning.substring(beginning.length - this.token.length - 4, + beginning.length - this.token.length).trim() + rest; + beginning = beginning.substring(0, beginning.length - this.token.length) + item + "\n"; } // Keep the cursor after the newly inserted text, as Bootstrap will call textbox.change() to // overwrite the text in the textbox. setTimeout(function () { $('#new_message_content').caret(beginning.length, beginning.length); + // Also, trigger autosize to check if compose box needs to be resized. + compose.autosize_textarea(); }, 0); return beginning + rest; }; exports.initialize_compose_typeahead = function (selector, completions) { - completions = $.extend({mention: false, emoji: false, stream: false}, completions); + completions = $.extend( + {mention: false, emoji: false, stream: false, syntax: false}, completions); $(selector).typeahead({ items: 5, @@ -358,6 +404,8 @@ exports.initialize_compose_typeahead = function (selector, completions) { return query_matches_person(this.token, item); } else if (this.completing === 'stream') { return query_matches_stream(this.token, item); + } else if (this.completing === 'syntax') { + return query_matches_language(this.token, item); } }, sorter: function (matches) { @@ -368,6 +416,8 @@ exports.initialize_compose_typeahead = function (selector, completions) { compose_state.stream_name()); } else if (this.completing === 'stream') { return typeahead_helper.sort_streams(matches, this.token); + } else if (this.completing === 'syntax') { + return typeahead_helper.sort_languages(matches, this.token); } }, updater: exports.content_typeahead_selected, @@ -493,7 +543,7 @@ exports.initialize = function () { stopAdvance: true, // Do not advance to the next field on a tab or enter }); - exports.initialize_compose_typeahead("#new_message_content", {mention: true, emoji: true, stream: true}); + exports.initialize_compose_typeahead("#new_message_content", {mention: true, emoji: true, stream: true, syntax: true}); $( "#private_message_recipient" ).blur(function () { var val = $(this).val(); diff --git a/static/js/typeahead_helper.js b/static/js/typeahead_helper.js index ef6744f3bc..f59fb077c9 100644 --- a/static/js/typeahead_helper.js +++ b/static/js/typeahead_helper.js @@ -182,6 +182,24 @@ exports.sort_for_at_mentioning = function (objs, current_stream) { return subs_sorted.concat(non_subs_sorted); }; +exports.compare_by_popularity = function (lang_a, lang_b) { + var diff = pygments_data.langs[lang_b] - pygments_data.langs[lang_a]; + if (diff !== 0) { + return diff; + } + return util.strcmp(lang_a, lang_b); +}; + +exports.sort_languages = function (matches, query) { + var results = prefix_sort(query, matches, function (x) { return x; }); + + // Languages that start with the query + results.matches = results.matches.sort(exports.compare_by_popularity); + // Languages that have the query somewhere in their name + results.rest = results.rest.sort(exports.compare_by_popularity); + return results.matches.concat(results.rest); +}; + exports.sort_recipients = function (matches, query, current_stream) { var name_results = prefix_sort(query, matches, function (x) { return x.full_name; }); var email_results = prefix_sort(query, name_results.rest, function (x) { return x.email; }); diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 4909f5b9a6..40bab9fcf4 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -219,6 +219,7 @@ def main(options): run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH]) run(["tools/setup/emoji/download-emoji-data"]) run(["tools/setup/emoji/build_emoji"]) + run(["tools/setup/build_pygments_data.py"]) run(["scripts/setup/generate_secrets.py", "--development"]) run(["tools/update-authors-json", "--use-fixture"]) if options.is_travis and not options.is_production_travis: diff --git a/tools/setup/build_pygments_data.py b/tools/setup/build_pygments_data.py new file mode 100755 index 0000000000..bdc102409f --- /dev/null +++ b/tools/setup/build_pygments_data.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +from pygments.lexers import get_all_lexers +import json +import os + +ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') +DATA_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'lang.json') +JS_PATH = os.path.join(ZULIP_PATH, 'static', 'generated', 'pygments_data.js') + +with open(DATA_PATH) as f: + langs = json.load(f) + +lexers = get_all_lexers() +for lexer in lexers: + for name in lexer[1]: + if name not in langs: + langs[name] = 0 + +template = '''var pygments_data = (function () { + +var exports = {}; + +exports.langs = %s; + +return exports; + +}()); +if (typeof module !== 'undefined') { + module.exports = pygments_data; +}''' % json.dumps(langs) + +with open(JS_PATH, 'w') as f: + f.write(template) diff --git a/tools/setup/lang.json b/tools/setup/lang.json new file mode 100644 index 0000000000..d1faca3883 --- /dev/null +++ b/tools/setup/lang.json @@ -0,0 +1,57 @@ +{ + "abap": 27, + "ada": 25, + "awk": 1, + "bash": 7, + "c": 49, + "c#": 47, + "c++": 48, + "cobol": 26, + "cpp": 48, + "csharp": 47, + "css": 48, + "d": 29, + "dart": 28, + "delphi": 42, + "erlang": 10, + "fsharp": 19, + "go": 34, + "groovy": 13, + "haskell": 15, + "html": 30, + "java": 50, + "javascript": 51, + "js": 43, + "julia": 4, + "latex": 40, + "lisp": 18, + "lua": 22, + "mask": 2, + "math": 50, + "matlab": 33, + "mql": 9, + "mql4": 9, + "objective-c": 35, + "objectivec": 35, + "objectpascal": 42, + "pascal": 42, + "perl": 40, + "php": 44, + "pl": 40, + "prolog": 16, + "python": 46, + "quote": 50, + "r": 37, + "rb": 39, + "ruby": 39, + "rust": 8, + "sas": 30, + "scala": 21, + "scheme": 14, + "sql": 32, + "swift": 41, + "tex": 40, + "vb.net": 45, + "vbnet": 45, + "xml": 1 +} diff --git a/tools/update-prod-static b/tools/update-prod-static index f5d2c2803e..dd1b47c7c6 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -45,6 +45,10 @@ subprocess.check_call(['./tools/setup/emoji/download-emoji-data'], subprocess.check_call(['./tools/setup/emoji/build_emoji'], stdout=fp, stderr=fp) +# Build pygment data +subprocess.check_call(['./tools/setup/build_pygments_data.py'], + stdout=fp, stderr=fp) + # Compile Handlebars templates and minify JavaScript. subprocess.check_call(['./tools/minify-js'] + (['--prev-deploy', prev_deploy] if prev_deploy else []), diff --git a/version.py b/version.py index d3514b7715..735f521dcd 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.5.1+git" -PROVISION_VERSION = '4.20' +PROVISION_VERSION = '4.21' diff --git a/zproject/settings.py b/zproject/settings.py index 1091b436c4..335c50629d 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -827,6 +827,7 @@ JS_SPECS = { 'node_modules/handlebars/dist/handlebars.runtime.js', 'third/marked/lib/marked.js', 'generated/emoji/emoji_codes.js', + 'generated/pygments_data.js', 'templates/compiled.js', 'js/feature_flags.js', 'js/loading.js',