emoji: Fallback to Google Modern for unsupported new Twitter emoji.

The Twitter emoji team was laid off in 2022, so new emoji aren't supported.
https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143.
The "twitter" sprite sheet we’re using does have images in those locations,
but they’re fallback images that emoji-datasource fills in from the Apple
sprite sheet, which has unclear licensing implications.

To be able to support newer emoji, we fallback to Google Modern for any emoji
not covered by Twemoji.

CZO conversation:
https://chat.zulip.org/#narrow/stream/107-kandra/topic/emoji.20changes.20in.208.2E0/near/1689918

tools/setup/emoji/build_emoji

@@ -17,6 +17,7 @@ from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
 from tools.setup.emoji.emoji_setup_utils import (
     EMOTICON_CONVERSIONS,
     REMAPPED_EMOJIS,
+    emoji_is_supported,
     emoji_names_for_picker,
     generate_codepoint_to_name_map,
     generate_codepoint_to_names_map,
@@ -178,13 +179,7 @@ def generate_sprite_css_files(
 
     emoji_positions = ""
     for emoji in emoji_data:
-        if emoji["has_img_google"]:
-            # Why is the test here has_img_google and not
-            # emoji_is_universal?  Because we briefly supported all
-            # Google emoji (not just the universal ones), we need to
-            # ensure the spritesheet is set up to correctly display
-            # those Google emoji (in case anyone used them).
-
+        if emoji_is_supported(emoji):
             """
             For background-position we need to use percentages.
             Absolute pixel values won't work, because the size
@@ -260,6 +255,30 @@ def generate_sprite_css_files(
         with open(SPRITE_CSS_PATH, "a") as f:
             f.write(extra_emoji_positions)
 
+    # The Twitter emoji team was laid off in 2022, so new emoji aren't supported.
+    # https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143.
+    # The "twitter" sprite sheet we’re using does have images in those locations,
+    # but they’re fallback images that emoji-datasource fills in from the Apple
+    # sprite sheet, which has unclear licensing implications.
+    # To be able to support newer emoji, we fallback to Google Modern for any emoji
+    # not covered by Twemoji.
+    if emojiset == "twitter":
+        extra_emoji_positions = ""
+        twitter_covered_emoji_codes = {
+            get_emoji_code(emoji) for emoji in emoji_data if emoji["has_img_twitter"]
+        }
+        for emoji in emoji_data:
+            code = get_emoji_code(emoji)
+            if emoji["has_img_google"] and code not in twitter_covered_emoji_codes:
+                extra_emoji_positions += EMOJI_POS_INFO_OVERRIDE_TEMPLATE.format(
+                    codepoint=code,
+                    pos_x=percent(emoji["sheet_x"] / (n - 1)),
+                    pos_y=percent(emoji["sheet_y"] / (n - 1)),
+                    background_size=background_size,
+                )
+        with open(SPRITE_CSS_PATH, "a") as f:
+            f.write(extra_emoji_positions)
+
 
 def setup_emoji_farms(cache_path: str, emoji_data: List[Dict[str, Any]]) -> None:
     def ensure_emoji_image(

tools/setup/emoji/emoji_setup_utils.py

@@ -5,9 +5,6 @@ from typing import Any, Dict, List
 
 from zerver.lib.emoji_utils import emoji_to_hex_codepoint, hex_codepoint_to_emoji, unqualify_emoji
 
-# Emoji sets that we currently support.
-EMOJISETS = ["google", "twitter"]
-
 # Some image files in the old emoji farm had a different name than in the new emoji
 # farm. `remapped_emojis` is a map that contains a mapping of their name in the old
 # emoji farm to their name in the new emoji farm.
@@ -82,7 +79,7 @@ def generate_emoji_catalog(
 
     for emoji_dict in emoji_data:
         emoji_code = get_emoji_code(emoji_dict)
-        if not emoji_is_universal(emoji_dict) or emoji_code not in emoji_name_maps:
+        if not emoji_is_supported(emoji_dict) or emoji_code not in emoji_name_maps:
             continue
         category = emoji_dict["category"]
         sort_order[emoji_code] = emoji_dict["sort_order"]
@@ -96,12 +93,6 @@ def generate_emoji_catalog(
     return dict(emoji_catalog)
 
 
-# Use only those names for which images are present in all
-# the emoji sets so that we can switch emoji sets seamlessly.
-def emoji_is_universal(emoji_dict: Dict[str, Any]) -> bool:
-    return all(emoji_dict["has_img_" + emoji_set] for emoji_set in EMOJISETS)
-
-
 def generate_codepoint_to_name_map(emoji_name_maps: Dict[str, Dict[str, Any]]) -> Dict[str, str]:
     codepoint_to_name: Dict[str, str] = {}
     for emoji_code, name_info in emoji_name_maps.items():
@@ -109,6 +100,12 @@ def generate_codepoint_to_name_map(emoji_name_maps: Dict[str, Dict[str, Any]]) -
     return codepoint_to_name
 
 
+# We support Google Modern, and fall back to Google Modern when emoji
+# aren't supported by the other styles we use.
+def emoji_is_supported(emoji_dict: Dict[str, Any]) -> bool:
+    return emoji_dict["has_img_google"]
+
+
 def generate_codepoint_to_names_map(
     emoji_name_maps: Dict[str, Dict[str, Any]]
 ) -> Dict[str, List[str]]:

tools/setup/emoji/generate_emoji_names_table

@@ -16,7 +16,7 @@ ZULIP_PATH = os.path.dirname(TOOLS_DIR)
 sys.path.append(ZULIP_PATH)
 
 from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
-from tools.setup.emoji.emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code
+from tools.setup.emoji.emoji_setup_utils import emoji_is_supported, get_emoji_code
 
 UNIFIED_REACTIONS_FILE = os.path.join(
     ZULIP_PATH, "zerver", "management", "data", "unified_reactions.json"
@@ -25,6 +25,9 @@ EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-goo
 EMOJI_CACHE = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
 OUTPUT_FILE = os.path.join(EMOJI_CACHE, "emoji_names_table.html")
 
+# Emoji sets that we currently support.
+EMOJISETS = ["google", "twitter"]
+
 with open(EMOJI_DATA_FILE, "rb") as fp:
     EMOJI_DATA = orjson.loads(fp.read())
 with open(UNIFIED_REACTIONS_FILE, "rb") as fp:
@@ -175,7 +178,7 @@ def generate_emoji_collection() -> None:
     generate_emoji_code_to_emoji_names_maps()
     # Prepare `emoji_collection`.
     for emoji_dict in EMOJI_DATA:
-        if not emoji_is_universal(emoji_dict):
+        if not emoji_is_supported(emoji_dict):
             continue
         category = emoji_dict["category"]
         emoji_code = get_emoji_code(emoji_dict)

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 230
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (254, 0)
+PROVISION_VERSION = (255, 0)

web/tests/emoji_picker.test.js

@@ -23,7 +23,7 @@ run_test("initialize", () => {
 
     const complete_emoji_catalog = _.sortBy(emoji_picker.complete_emoji_catalog, "name");
     assert.equal(complete_emoji_catalog.length, 11);
-    assert.equal(emoji.emojis_by_name.size, 1827);
+    assert.equal(emoji.emojis_by_name.size, 1848);
 
     let total_emoji_in_categories = 0;
 
@@ -45,15 +45,15 @@ run_test("initialize", () => {
     const popular_emoji_count = 6;
     const zulip_emoji_count = 1;
     assert_emoji_category(complete_emoji_catalog.pop(), "fa-car", 195);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-hashtag", 221);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-smile-o", 162);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-hashtag", 223);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-smile-o", 166);
     assert_emoji_category(complete_emoji_catalog.pop(), "fa-star-o", popular_emoji_count);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-thumbs-o-up", 361);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-lightbulb-o", 257);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-cutlery", 131);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-thumbs-o-up", 363);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-lightbulb-o", 261);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-cutlery", 133);
     assert_emoji_category(complete_emoji_catalog.pop(), "fa-flag", 269);
     assert_emoji_category(complete_emoji_catalog.pop(), "fa-cog", 1);
-    assert_emoji_category(complete_emoji_catalog.pop(), "fa-leaf", 145);
+    assert_emoji_category(complete_emoji_catalog.pop(), "fa-leaf", 152);
     assert_emoji_category(complete_emoji_catalog.pop(), "fa-soccer-ball-o", 85);
 
     // The popular emoji appear twice in the picker, and the zulip emoji is special