emoji_codes: Replace JS module with JSON module.

webpack optimizes JSON modules using JSON.parse("{…}"), which is
faster than the normal JavaScript parser.

Update the backend to use emoji_codes.json too instead of the three
separate JSON files.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

.eslintrc.json

@@ -56,7 +56,6 @@
         "drafts": false,
         "echo": false,
         "emoji": false,
-        "emoji_codes": false,
         "emoji_picker": false,
         "favicon": false,
         "feature_flags": false,

docs/subsystems/emoji.md

@@ -65,7 +65,7 @@ emoji tooling).  See [our dependencies document](../subsystems/dependencies.md)
 for more details on this strategy.
 
 The emoji tree generated by this process contains several import elements:
-* `emoji_codes.js`: A set of mappings used by the Zulip frontend to
+* `emoji_codes.json`: A set of mappings used by the Zulip frontend to
   understand what unicode emoji exist and what their shortnames are,
   used for autocomplete, emoji pickers, etc.  This has been
   deduplicated using the logic in

frontend_tests/node_tests/emoji.js

@@ -6,7 +6,7 @@ set_global('page_params', {
 set_global('upload_widget', {});
 set_global('blueslip', global.make_zblueslip());
 
-set_global('emoji_codes', zrequire('emoji_codes', 'generated/emoji/emoji_codes'));
+const emoji_codes = zrequire('emoji_codes', 'generated/emoji/emoji_codes.json');
 zrequire('emoji');
 zrequire('markdown');
 zrequire('util');
@@ -68,29 +68,19 @@ run_test('get_canonical_name', () => {
     let canonical_name = emoji.get_canonical_name('realm_emoji');
     assert.equal(canonical_name, 'realm_emoji');
 
-    const orig_emoji_codes = global.emoji_codes;
-    global.emoji_codes = {
-        name_to_codepoint: {
-            '+1': '1f44d',
-        },
-        codepoint_to_name: {
-            '1f44d': 'thumbs_up',
-        },
-    };
-    canonical_name = emoji.get_canonical_name('+1');
-    assert.equal(canonical_name, 'thumbs_up');
+    canonical_name = emoji.get_canonical_name('thumbs_up');
+    assert.equal(canonical_name, '+1');
 
-    emoji.active_realm_emojis = new Map(Object.entries({
-        '+1': 'TBD',
-    }));
     canonical_name = emoji.get_canonical_name('+1');
     assert.equal(canonical_name, '+1');
 
+    canonical_name = emoji.get_canonical_name('airplane');
+    assert.equal(canonical_name, 'airplane');
+
     blueslip.set_test_data('error', 'Invalid emoji name: non_existent');
     emoji.get_canonical_name('non_existent');
     assert.equal(blueslip.get_test_logs('error').length, 1);
     blueslip.clear_test_data();
-    global.emoji_codes = orig_emoji_codes;
 });
 
 run_test('translate_emoticons_to_names', () => {

frontend_tests/node_tests/emoji_picker.js

@@ -1,4 +1,3 @@
-set_global('emoji_codes', zrequire('emoji_codes', 'generated/emoji/emoji_codes'));
 zrequire('emoji');
 zrequire('emoji_picker');
 

frontend_tests/node_tests/hotkey.js

@@ -192,11 +192,6 @@ run_test('basic_chars', () => {
     set_global('emoji_picker', {
         reactions_popped: return_false,
     });
-    set_global('emoji_codes', {
-        codepoint_to_name: {
-            '1f44d': 'thumbs_up',
-        },
-    });
     set_global('hotspots', {
         is_open: return_false,
     });

frontend_tests/node_tests/markdown.js

@@ -8,7 +8,7 @@ zrequire('fenced_code');
 zrequire('stream_data');
 zrequire('people');
 zrequire('user_groups');
-set_global('emoji_codes', zrequire('emoji_codes', 'generated/emoji/emoji_codes'));
+const emoji_codes = zrequire('emoji_codes', 'generated/emoji/emoji_codes.json');
 zrequire('emoji');
 zrequire('message_store');
 zrequire('markdown');

frontend_tests/node_tests/reactions.js

@@ -50,14 +50,6 @@ set_global('blueslip', global.make_zblueslip());
 set_global('page_params', {user_id: 5});
 
 set_global('channel', {});
-set_global('emoji_codes', {
-    name_to_codepoint: {
-        alien: '1f47d',
-        smile: '1f604',
-        frown: '1f626',
-        octopus: '1f419',
-    },
-});
 set_global('emoji_picker', {
     hide_emoji_popover: function () {},
 });
@@ -84,15 +76,15 @@ people.add_in_realm(cali);
 const message = {
     id: 1001,
     reactions: [
-        {emoji_name: 'smile', user: {id: 5}, reaction_type: 'unicode_emoji', emoji_code: '1f604'},
-        {emoji_name: 'smile', user: {id: 6}, reaction_type: 'unicode_emoji', emoji_code: '1f604'},
-        {emoji_name: 'frown', user: {id: 7}, reaction_type: 'unicode_emoji', emoji_code: '1f626'},
+        {emoji_name: 'smile', user: {id: 5}, reaction_type: 'unicode_emoji', emoji_code: '263a'},
+        {emoji_name: 'smile', user: {id: 6}, reaction_type: 'unicode_emoji', emoji_code: '263a'},
+        {emoji_name: 'frown', user: {id: 7}, reaction_type: 'unicode_emoji', emoji_code: '1f641'},
         {emoji_name: 'inactive_realm_emoji', user: {id: 5}, reaction_type: 'realm_emoji',
          emoji_code: '992'},
 
         // add some bogus user_ids
         {emoji_name: 'octopus', user: {id: 8888}, reaction_type: 'unicode_emoji', emoji_code: '1f419'},
-        {emoji_name: 'frown', user: {id: 9999}, reaction_type: 'unicode_emoji', emoji_code: '1f626'},
+        {emoji_name: 'frown', user: {id: 9999}, reaction_type: 'unicode_emoji', emoji_code: '1f641'},
     ],
 };
 
@@ -148,8 +140,8 @@ run_test('basics', () => {
     const result = reactions.get_message_reactions(message);
     assert.equal(blueslip.get_test_logs('warn').length, 2);
     blueslip.clear_test_data();
-    assert(reactions.current_user_has_reacted_to_emoji(message, '1f604', 'unicode_emoji'));
-    assert(!reactions.current_user_has_reacted_to_emoji(message, '1f626', 'unicode_emoji'));
+    assert(reactions.current_user_has_reacted_to_emoji(message, '263a', 'unicode_emoji'));
+    assert(!reactions.current_user_has_reacted_to_emoji(message, '1f641', 'unicode_emoji'));
 
     result.sort(function (a, b) { return a.count - b.count; });
 
@@ -157,8 +149,8 @@ run_test('basics', () => {
         {
             emoji_name: 'frown',
             reaction_type: 'unicode_emoji',
-            emoji_code: '1f626',
-            local_id: 'unicode_emoji,frown,1f626',
+            emoji_code: '1f641',
+            local_id: 'unicode_emoji,frown,1f641',
             count: 1,
             user_ids: [7],
             label: 'Cali reacted with :frown:',
@@ -181,8 +173,8 @@ run_test('basics', () => {
         {
             emoji_name: 'smile',
             reaction_type: 'unicode_emoji',
-            emoji_code: '1f604',
-            local_id: 'unicode_emoji,smile,1f604',
+            emoji_code: '263a',
+            local_id: 'unicode_emoji,smile,263a',
             count: 2,
             user_ids: [5, 6],
             label: 'You (click to remove) and Bob van Roberts reacted with :smile:',
@@ -210,7 +202,7 @@ run_test('sending', () => {
         assert.deepEqual(args.data, {
             reaction_type: 'unicode_emoji',
             emoji_name: 'smile',
-            emoji_code: '1f604',
+            emoji_code: '263a',
         });
         // args.success() does nothing; just make sure it doesn't crash
         args.success();
@@ -301,7 +293,7 @@ run_test('get_reaction_section', () => {
 
 run_test('emoji_reaction_title', () => {
     const message_id = 1001;
-    const local_id = 'unicode_emoji,smile,1f604';
+    const local_id = 'unicode_emoji,smile,263a';
 
     assert.equal(reactions.get_reaction_title_data(message_id, local_id),
                  "You (click to remove) and Bob van Roberts reacted with :smile:");
@@ -704,11 +696,11 @@ run_test('process_reaction_click', () => {
     expected_reaction_info = {
         reaction_type: 'unicode_emoji',
         emoji_name: 'smile',
-        emoji_code: '1f604',
+        emoji_code: '263a',
     };
     global.with_stub(function (stub) {
         global.channel.del = stub.f;
-        reactions.process_reaction_click(message_id, 'unicode_emoji,smile,1f604');
+        reactions.process_reaction_click(message_id, 'unicode_emoji,smile,263a');
         const args = stub.get_args('args').args;
         assert.equal(args.url, '/json/messages/1001/reactions');
         assert.deepEqual(args.data, expected_reaction_info);

static/js/bundles/app.js

@@ -17,7 +17,6 @@ import "handlebars/dist/cjs/handlebars.runtime.js";
 import "flatpickr/dist/flatpickr.js";
 import "flatpickr/dist/plugins/confirmDate/confirmDate.js";
 import "sortablejs/Sortable.js";
-import "../../generated/emoji/emoji_codes.js";
 import "../../generated/pygments_data.js";
 
 // Import App JS

static/js/emoji.js

@@ -1,3 +1,5 @@
+const emoji_codes = require("../generated/emoji/emoji_codes.json");
+
 // `emojis_by_name` is the central data source that is supposed to be
 // used by every widget in the webapp for gathering data for displaying
 // emojis. Emoji picker uses this data to derive data for its own use.

static/js/emoji_picker.js

@@ -1,3 +1,4 @@
+const emoji_codes = require("../generated/emoji/emoji_codes.json");
 const typeahead = require("../shared/js/typeahead");
 
 const render_emoji_popover = require('../templates/emoji_popover.hbs');

static/js/hotkey.js

@@ -1,3 +1,5 @@
+const emoji_codes = require("../generated/emoji/emoji_codes.json");
+
 function do_narrow_action(action) {
     action(current_msg_list.selected_id(), {trigger: 'hotkey'});
     return true;

static/js/markdown.js

@@ -6,6 +6,8 @@
 
 // Docs: https://zulip.readthedocs.io/en/latest/subsystems/markdown.html
 
+const emoji_codes = require("../generated/emoji/emoji_codes.json");
+
 let realm_filter_map = {};
 let realm_filter_list = [];
 

static/js/reactions.js

@@ -1,3 +1,4 @@
+const emoji_codes = require("../generated/emoji/emoji_codes.json");
 const render_message_reaction = require('../templates/message_reaction.hbs');
 const Dict = require('./dict').Dict;
 

tools/setup/emoji/build_emoji

@@ -24,27 +24,6 @@ EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
 EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'emoji')
 NODE_MODULES_PATH = os.path.join(ZULIP_PATH, 'node_modules')
 
-EMOJI_CODES_FILE_TEMPLATE = """\
-var emoji_codes = (function () {
-var exports = {};
-
-exports.names = %(names)s;
-
-exports.name_to_codepoint = %(name_to_codepoint)s;
-
-exports.codepoint_to_name = %(codepoint_to_name)s;
-
-exports.emoji_catalog = %(emoji_catalog)s;
-
-exports.emoticon_conversions = %(emoticon_conversions)s;
-
-return exports;
-}());
-if (typeof module !== 'undefined') {
-    module.exports = emoji_codes;
-}
-"""
-
 SPRITE_CSS_FILE_TEMPLATE = """\
 div.emoji,
 span.emoji
@@ -244,32 +223,20 @@ def setup_old_emoji_farm(cache_path: str,
             pass
 
 def generate_map_files(cache_path: str, emoji_catalog: Dict[str, List[str]]) -> None:
-    # This function generates the various data files consumed by webapp, mobile apps, bugdown etc.
+    # This function generates the data file consumed by webapp, mobile apps, bugdown etc.
     names = emoji_names_for_picker(EMOJI_NAME_MAPS)
     codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
     name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)
 
-    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.js')
+    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.json')
     with open(EMOJI_CODES_FILE_PATH, 'w') as emoji_codes_file:
-        emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % {
+        ujson.dump({
             'names': names,
             'name_to_codepoint': name_to_codepoint,
             'codepoint_to_name': codepoint_to_name,
             'emoji_catalog': emoji_catalog,
             'emoticon_conversions': EMOTICON_CONVERSIONS,
-        })
-
-    NAME_TO_CODEPOINT_PATH = os.path.join(cache_path, 'name_to_codepoint.json')
-    with open(NAME_TO_CODEPOINT_PATH, 'w') as name_to_codepoint_file:
-        name_to_codepoint_file.write(ujson.dumps(name_to_codepoint))
-
-    CODEPOINT_TO_NAME_PATH = os.path.join(cache_path, 'codepoint_to_name.json')
-    with open(CODEPOINT_TO_NAME_PATH, 'w') as codepoint_to_name_file:
-        codepoint_to_name_file.write(ujson.dumps(codepoint_to_name))
-
-    EMOTICON_CONVERSIONS_PATH = os.path.join(cache_path, 'emoticon_conversions.json')
-    with open(EMOTICON_CONVERSIONS_PATH, 'w') as emoticon_conversions_file:
-        emoticon_conversions_file.write(ujson.dumps(EMOTICON_CONVERSIONS))
+        }, emoji_codes_file)
 
 def dump_emojis(cache_path: str) -> None:
     with open('emoji_map.json') as emoji_map_file:

tools/webpack.config.ts

@@ -221,7 +221,6 @@ export default (env?: string): webpack.Configuration[] => {
         { path: "clipboard/dist/clipboard.js", name: "ClipboardJS" },
         { path: "xdate/src/xdate.js", name: "XDate" },
         { path: "../static/third/marked/lib/marked.js" },
-        { path: "../static/generated/emoji/emoji_codes.js" },
         { path: "../static/generated/pygments_data.js" },
         { path: "../static/js/debug.js" },
         { path: "../static/js/blueslip.js" },

version.py

@@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/12/13/zulip-2-1-relea
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '70.1'
+PROVISION_VERSION = '71.0'

zerver/data_import/mattermost.py

@@ -19,7 +19,7 @@ from zerver.models import Recipient, RealmEmoji, Reaction, UserProfile
 from zerver.lib.utils import (
     process_list_in_batches,
 )
-from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
+from zerver.lib.emoji import name_to_codepoint
 from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
     build_stream, build_realm, build_message, create_converted_data_files, \
     make_subscriber_map, build_recipients, build_user_profile, \
@@ -233,12 +233,8 @@ def convert_huddle_data(huddle_data: List[ZerverFieldsT],
             zerver_huddle.append(huddle_dict)
     return zerver_huddle
 
-def get_name_to_codepoint_dict() -> Dict[str, str]:
-    with open(NAME_TO_CODEPOINT_PATH) as fp:
-        return ujson.load(fp)
-
 def build_reactions(realm_id: int, total_reactions: List[ZerverFieldsT], reactions: List[ZerverFieldsT],
-                    message_id: int, name_to_codepoint: ZerverFieldsT,
+                    message_id: int,
                     user_id_mapper: IdMapper, zerver_realmemoji: List[ZerverFieldsT]) -> None:
     realmemoji = {}
     for realm_emoji in zerver_realmemoji:
@@ -320,7 +316,6 @@ def process_raw_message_batch(realm_id: int,
     import html2text
     h = html2text.HTML2Text()
 
-    name_to_codepoint = get_name_to_codepoint_dict()
     pm_members = {}
 
     for raw_message in raw_messages:
@@ -371,7 +366,7 @@ def process_raw_message_batch(realm_id: int,
         )
         zerver_message.append(message)
         build_reactions(realm_id, total_reactions, raw_message["reactions"], message_id,
-                        name_to_codepoint, user_id_mapper, zerver_realmemoji)
+                        user_id_mapper, zerver_realmemoji)
 
     zerver_usermessage = make_user_messages(
         zerver_message=zerver_message,

zerver/data_import/slack.py

@@ -24,7 +24,7 @@ from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
 from zerver.data_import.sequencer import NEXT_ID
 from zerver.lib.upload import random_name, sanitize_name
 from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE
-from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
+from zerver.lib.emoji import name_to_codepoint
 from zerver.lib.upload import resize_logo
 from urllib.parse import urlencode
 
@@ -681,10 +681,6 @@ def channel_message_to_zerver_message(realm_id: int,
     zerver_attachment = []  # type: List[ZerverFieldsT]
     reaction_list = []  # type: List[ZerverFieldsT]
 
-    # For unicode emoji
-    with open(NAME_TO_CODEPOINT_PATH) as fp:
-        name_to_codepoint = ujson.load(fp)
-
     total_user_messages = 0
     total_skipped_user_messages = 0
     for message in all_messages:
@@ -736,7 +732,7 @@ def channel_message_to_zerver_message(realm_id: int,
 
         if 'reactions' in message.keys():
             build_reactions(reaction_list, message['reactions'], slack_user_id_to_zulip_user_id,
-                            message_id, name_to_codepoint, zerver_realmemoji)
+                            message_id, zerver_realmemoji)
 
         # Process different subtypes of slack messages
 
@@ -895,7 +891,6 @@ def get_attachment_path_and_content(fileinfo: ZerverFieldsT, realm_id: int) -> T
 
 def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT],
                     slack_user_id_to_zulip_user_id: SlackToZulipUserIDT, message_id: int,
-                    name_to_codepoint: ZerverFieldsT,
                     zerver_realmemoji: List[ZerverFieldsT]) -> None:
     realmemoji = {}
     for realm_emoji in zerver_realmemoji:

zerver/lib/bugdown/__init__.py

@@ -16,7 +16,6 @@ import html
 import time
 import functools
 from io import StringIO
-import ujson
 import xml.etree.cElementTree as etree
 from xml.etree.cElementTree import Element
 import ahocorasick
@@ -33,10 +32,10 @@ from markdown.extensions import codehilite, nl2br, tables, sane_lists
 from zerver.lib.bugdown import fenced_code
 from zerver.lib.bugdown.fenced_code import FENCE_RE
 from zerver.lib.camo import get_camo_url
-from zerver.lib.emoji import translate_emoticons, emoticon_regex
+from zerver.lib.emoji import translate_emoticons, emoticon_regex, \
+    name_to_codepoint, codepoint_to_name
 from zerver.lib.mention import possible_mentions, \
     possible_user_group_mentions, extract_user_group
-from zerver.lib.storage import static_path
 from zerver.lib.url_encoding import encode_stream, hash_util_encode
 from zerver.lib.thumbnail import user_uploads_or_external
 from zerver.lib.timeout import timeout, TimeoutExpired
@@ -1204,14 +1203,6 @@ def possible_avatar_emails(content: str) -> Set[str]:
 
     return emails
 
-path_to_name_to_codepoint = static_path("generated/emoji/name_to_codepoint.json")
-with open(path_to_name_to_codepoint) as name_to_codepoint_file:
-    name_to_codepoint = ujson.load(name_to_codepoint_file)
-
-path_to_codepoint_to_name = static_path("generated/emoji/codepoint_to_name.json")
-with open(path_to_codepoint_to_name) as codepoint_to_name_file:
-    codepoint_to_name = ujson.load(codepoint_to_name_file)
-
 # All of our emojis(non ZWJ sequences) belong to one of these unicode blocks:
 # \U0001f100-\U0001f1ff - Enclosed Alphanumeric Supplement
 # \U0001f200-\U0001f2ff - Enclosed Ideographic Supplement

zerver/lib/emoji.py

@@ -11,19 +11,12 @@ from zerver.lib.upload import upload_backend
 from zerver.lib.exceptions import OrganizationAdministratorRequired
 from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
 
-EMOJI_PATH = static_path("generated/emoji")
-NAME_TO_CODEPOINT_PATH = os.path.join(EMOJI_PATH, "name_to_codepoint.json")
-CODEPOINT_TO_NAME_PATH = os.path.join(EMOJI_PATH, "codepoint_to_name.json")
-EMOTICON_CONVERSIONS_PATH = os.path.join(EMOJI_PATH, "emoticon_conversions.json")
+with open(static_path("generated/emoji/emoji_codes.json")) as fp:
+    emoji_codes = ujson.load(fp)
 
-with open(NAME_TO_CODEPOINT_PATH) as fp:
-    name_to_codepoint = ujson.load(fp)
-
-with open(CODEPOINT_TO_NAME_PATH) as fp:
-    codepoint_to_name = ujson.load(fp)
-
-with open(EMOTICON_CONVERSIONS_PATH) as fp:
-    EMOTICON_CONVERSIONS = ujson.load(fp)
+name_to_codepoint = emoji_codes["name_to_codepoint"]
+codepoint_to_name = emoji_codes["codepoint_to_name"]
+EMOTICON_CONVERSIONS = emoji_codes["emoticon_conversions"]
 
 possible_emoticons = EMOTICON_CONVERSIONS.keys()
 possible_emoticon_regexes = (re.escape(emoticon) for emoticon in possible_emoticons)

zerver/tests/test_mattermost_importer.py

@@ -5,6 +5,7 @@ import logging
 
 from typing import Dict, Any, List
 
+from zerver.lib.emoji import name_to_codepoint
 from zerver.lib.import_realm import (
     do_import_realm,
 )
@@ -16,7 +17,7 @@ from zerver.data_import.mattermost_user import UserHandler
 from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \
     create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \
     convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \
-    build_reactions, get_name_to_codepoint_dict, do_convert_data, convert_huddle_data, \
+    build_reactions, do_convert_data, convert_huddle_data, \
     generate_huddle_name
 from zerver.data_import.sequencer import IdMapper
 from zerver.data_import.import_util import SubscriberHandler
@@ -457,7 +458,6 @@ class MatterMostImporter(ZulipTestCase):
         self.assertEqual(zerver_realmemoji[1]["name"], "tick")
         tick_emoji_code = zerver_realmemoji[1]["id"]
 
-        name_to_codepoint = get_name_to_codepoint_dict()
         user_id_mapper = IdMapper()
         harry_id = user_id_mapper.get("harry")
         ron_id = user_id_mapper.get("ron")
@@ -467,7 +467,6 @@ class MatterMostImporter(ZulipTestCase):
             total_reactions=total_reactions,
             reactions=reactions,
             message_id=5,
-            name_to_codepoint=name_to_codepoint,
             user_id_mapper=user_id_mapper,
             zerver_realmemoji=zerver_realmemoji
         )