build_emoji: Extend emoji mappings to use new emojis from iamcal dataset.

Emojis which are represented by a sequence of codepoints or emojis
with ZWJ are not included until we implement a mechanism for dealing
with their unicode versions.

Fixes: #6279.

frontend_tests/node_tests/emoji_picker.js

@@ -11,7 +11,7 @@ var emoji_picker = require('js/emoji_picker.js');
 
     var complete_emoji_catalog = _.sortBy(emoji_picker.complete_emoji_catalog, 'name');
     assert.equal(complete_emoji_catalog.length, 9);
-    assert.equal(_.keys(emoji_picker.emoji_collection).length, 977);
+    assert.equal(_.keys(emoji_picker.emoji_collection).length, 1065);
 
     function assert_emoji_category(ele, icon, num) {
         assert.equal(ele.icon, icon);
@@ -27,13 +27,13 @@ var emoji_picker = require('js/emoji_picker.js');
             check_emojis(false);
         }
     }
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 243);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 251);
     assert_emoji_category(complete_emoji_catalog.pop(), 'fa-thumbs-o-up', 6);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 115);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 185);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 165);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 131);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 68);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 118);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 209);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 172);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 147);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 85);
     assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cog', 1);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-soccer-ball-o', 54);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-soccer-ball-o', 67);
 }());

tools/setup/emoji/build_emoji

@@ -9,7 +9,8 @@ import ujson
 from typing import Any, Dict, List, Text, Union
 
 from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \
-    emoji_names_for_picker, EMOJISETS
+    get_extended_codepoint_to_name, get_extended_name_to_codepoint, get_extended_names_list, \
+    get_new_emoji_dicts, emoji_names_for_picker, EMOJISETS
 
 ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
 sys.path.append(ZULIP_PATH)
@@ -198,6 +199,12 @@ def generate_map_files(cache_path, emoji_map, emoji_data, emoji_catalog, unified
     name_to_codepoint = {name: unified_reactions_data[name] for name in names}
     codepoint_to_name = generate_codepoint_to_name_map(names, unified_reactions_data)
 
+    # Extend to use new emojis from iamcal dataset(See `emoji_can_be_included()` in emoji_setup_utils).
+    new_emoji_dicts = get_new_emoji_dicts(unified_reactions_data, emoji_data)
+    names = get_extended_names_list(names, new_emoji_dicts)
+    name_to_codepoint = get_extended_name_to_codepoint(name_to_codepoint, new_emoji_dicts)
+    codepoint_to_name = get_extended_codepoint_to_name(codepoint_to_name, new_emoji_dicts)
+
     emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % {
         'names': names,
         'name_to_codepoint': name_to_codepoint,

tools/setup/emoji/emoji_setup_utils.py

@@ -273,6 +273,10 @@ def emoji_can_be_included(emoji_dict, unified_reactions_codepoints):
     # safe to be included. Currently emojis which are represented by a sequence of codepoints
     # or emojis with ZWJ are not to be included until we implement a mechanism for dealing with
     # their unicode versions.
+    # `:fried_egg:` emoji is banned for now, due to a name collision with `:egg:` emoji in
+    # `unified_reactions.json` dataset, until we completely switch to iamcal dataset.
+    if emoji_dict["short_name"] == "fried_egg":
+        return False
     codepoint = emoji_dict["unified"].lower()
     if '-' not in codepoint and emoji_dict["category"] != "Skin Tones" and \
             emoji_is_universal(emoji_dict) and codepoint not in unified_reactions_codepoints:

version.py

@@ -1,2 +1,2 @@
 ZULIP_VERSION = "1.6.0+git"
-PROVISION_VERSION = '10.2'
+PROVISION_VERSION = '10.3'

zerver/fixtures/markdown_test_cases.json

@@ -342,6 +342,30 @@
       "expected_output": "<p><span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>, <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>; <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span></p>",
       "bugdown_matches_marked": true
     },
+    {
+      "name": "new_emoji_test",
+      "input": ":avocado:, :kiwifruit:, :selfie:, :gear:, :comet:, :first_place_medal:",
+      "expected_output": "<p><span class=\"emoji emoji-1f951\" title=\"avocado\">:avocado:</span>, <span class=\"emoji emoji-1f95d\" title=\"kiwifruit\">:kiwifruit:</span>, <span class=\"emoji emoji-1f933\" title=\"selfie\">:selfie:</span>, <span class=\"emoji emoji-2699\" title=\"gear\">:gear:</span>, <span class=\"emoji emoji-2604\" title=\"comet\">:comet:</span>, <span class=\"emoji emoji-1f947\" title=\"first place medal\">:first_place_medal:</span></p>",
+      "bugdown_matches_marked": true
+    },
+    {
+      "name": "skin_tones_are_banned",
+      "input": ":skin-tone-2: :skin-tone-3:",
+      "expected_output": "<p>:skin-tone-2: :skin-tone-3:</p>",
+      "bugdown_matches_marked": true
+    },
+    {
+      "name": "fried_egg",
+      "input": "new :fried_egg: emoji is banned for now.",
+      "expected_output": "<p>new :fried_egg: emoji is banned for now.</p>",
+      "bugdown_matches_marked": true
+    },
+    {
+      "name": "emojis_with_zwj_are_banned",
+      "input": "Emojis like :man-girl-girl: which have ZWJ are banned for now.",
+      "expected_output": "<p>Emojis like :man-girl-girl: which have ZWJ are banned for now.</p>",
+      "bugdown_matches_marked": true
+    },
     {
       "name": "safe_html",
       "input": "<h1>stay normal</h1> thanks",