From 32c730e5308095ce86eb60dd034fe35ac869504f Mon Sep 17 00:00:00 2001 From: evykassirer Date: Tue, 28 Nov 2023 21:47:59 -0800 Subject: [PATCH] emoji: Fallback to Google Modern for unsupported new Twitter emoji. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Twitter emoji team was laid off in 2022, so new emoji aren't supported. https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143. The "twitter" sprite sheet we’re using does have images in those locations, but they’re fallback images that emoji-datasource fills in from the Apple sprite sheet, which has unclear licensing implications. To be able to support newer emoji, we fallback to Google Modern for any emoji not covered by Twemoji. CZO conversation: https://chat.zulip.org/#narrow/stream/107-kandra/topic/emoji.20changes.20in.208.2E0/near/1689918 --- tools/setup/emoji/build_emoji | 33 +++++++++++++++----- tools/setup/emoji/emoji_setup_utils.py | 17 +++++----- tools/setup/emoji/generate_emoji_names_table | 7 +++-- version.py | 2 +- web/tests/emoji_picker.test.js | 14 ++++----- 5 files changed, 46 insertions(+), 27 deletions(-) diff --git a/tools/setup/emoji/build_emoji b/tools/setup/emoji/build_emoji index 7fd3780efe..c68c5bfce3 100755 --- a/tools/setup/emoji/build_emoji +++ b/tools/setup/emoji/build_emoji @@ -17,6 +17,7 @@ from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS from tools.setup.emoji.emoji_setup_utils import ( EMOTICON_CONVERSIONS, REMAPPED_EMOJIS, + emoji_is_supported, emoji_names_for_picker, generate_codepoint_to_name_map, generate_codepoint_to_names_map, @@ -178,13 +179,7 @@ def generate_sprite_css_files( emoji_positions = "" for emoji in emoji_data: - if emoji["has_img_google"]: - # Why is the test here has_img_google and not - # emoji_is_universal? Because we briefly supported all - # Google emoji (not just the universal ones), we need to - # ensure the spritesheet is set up to correctly display - # those Google emoji (in case anyone used them). - + if emoji_is_supported(emoji): """ For background-position we need to use percentages. Absolute pixel values won't work, because the size @@ -260,6 +255,30 @@ def generate_sprite_css_files( with open(SPRITE_CSS_PATH, "a") as f: f.write(extra_emoji_positions) + # The Twitter emoji team was laid off in 2022, so new emoji aren't supported. + # https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143. + # The "twitter" sprite sheet we’re using does have images in those locations, + # but they’re fallback images that emoji-datasource fills in from the Apple + # sprite sheet, which has unclear licensing implications. + # To be able to support newer emoji, we fallback to Google Modern for any emoji + # not covered by Twemoji. + if emojiset == "twitter": + extra_emoji_positions = "" + twitter_covered_emoji_codes = { + get_emoji_code(emoji) for emoji in emoji_data if emoji["has_img_twitter"] + } + for emoji in emoji_data: + code = get_emoji_code(emoji) + if emoji["has_img_google"] and code not in twitter_covered_emoji_codes: + extra_emoji_positions += EMOJI_POS_INFO_OVERRIDE_TEMPLATE.format( + codepoint=code, + pos_x=percent(emoji["sheet_x"] / (n - 1)), + pos_y=percent(emoji["sheet_y"] / (n - 1)), + background_size=background_size, + ) + with open(SPRITE_CSS_PATH, "a") as f: + f.write(extra_emoji_positions) + def setup_emoji_farms(cache_path: str, emoji_data: List[Dict[str, Any]]) -> None: def ensure_emoji_image( diff --git a/tools/setup/emoji/emoji_setup_utils.py b/tools/setup/emoji/emoji_setup_utils.py index 95837aa3a7..d8860da7cc 100644 --- a/tools/setup/emoji/emoji_setup_utils.py +++ b/tools/setup/emoji/emoji_setup_utils.py @@ -5,9 +5,6 @@ from typing import Any, Dict, List from zerver.lib.emoji_utils import emoji_to_hex_codepoint, hex_codepoint_to_emoji, unqualify_emoji -# Emoji sets that we currently support. -EMOJISETS = ["google", "twitter"] - # Some image files in the old emoji farm had a different name than in the new emoji # farm. `remapped_emojis` is a map that contains a mapping of their name in the old # emoji farm to their name in the new emoji farm. @@ -82,7 +79,7 @@ def generate_emoji_catalog( for emoji_dict in emoji_data: emoji_code = get_emoji_code(emoji_dict) - if not emoji_is_universal(emoji_dict) or emoji_code not in emoji_name_maps: + if not emoji_is_supported(emoji_dict) or emoji_code not in emoji_name_maps: continue category = emoji_dict["category"] sort_order[emoji_code] = emoji_dict["sort_order"] @@ -96,12 +93,6 @@ def generate_emoji_catalog( return dict(emoji_catalog) -# Use only those names for which images are present in all -# the emoji sets so that we can switch emoji sets seamlessly. -def emoji_is_universal(emoji_dict: Dict[str, Any]) -> bool: - return all(emoji_dict["has_img_" + emoji_set] for emoji_set in EMOJISETS) - - def generate_codepoint_to_name_map(emoji_name_maps: Dict[str, Dict[str, Any]]) -> Dict[str, str]: codepoint_to_name: Dict[str, str] = {} for emoji_code, name_info in emoji_name_maps.items(): @@ -109,6 +100,12 @@ def generate_codepoint_to_name_map(emoji_name_maps: Dict[str, Dict[str, Any]]) - return codepoint_to_name +# We support Google Modern, and fall back to Google Modern when emoji +# aren't supported by the other styles we use. +def emoji_is_supported(emoji_dict: Dict[str, Any]) -> bool: + return emoji_dict["has_img_google"] + + def generate_codepoint_to_names_map( emoji_name_maps: Dict[str, Dict[str, Any]] ) -> Dict[str, List[str]]: diff --git a/tools/setup/emoji/generate_emoji_names_table b/tools/setup/emoji/generate_emoji_names_table index ea9b109e61..76d569f10d 100755 --- a/tools/setup/emoji/generate_emoji_names_table +++ b/tools/setup/emoji/generate_emoji_names_table @@ -16,7 +16,7 @@ ZULIP_PATH = os.path.dirname(TOOLS_DIR) sys.path.append(ZULIP_PATH) from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS -from tools.setup.emoji.emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code +from tools.setup.emoji.emoji_setup_utils import emoji_is_supported, get_emoji_code UNIFIED_REACTIONS_FILE = os.path.join( ZULIP_PATH, "zerver", "management", "data", "unified_reactions.json" @@ -25,6 +25,9 @@ EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-goo EMOJI_CACHE = os.path.join(ZULIP_PATH, "static", "generated", "emoji") OUTPUT_FILE = os.path.join(EMOJI_CACHE, "emoji_names_table.html") +# Emoji sets that we currently support. +EMOJISETS = ["google", "twitter"] + with open(EMOJI_DATA_FILE, "rb") as fp: EMOJI_DATA = orjson.loads(fp.read()) with open(UNIFIED_REACTIONS_FILE, "rb") as fp: @@ -175,7 +178,7 @@ def generate_emoji_collection() -> None: generate_emoji_code_to_emoji_names_maps() # Prepare `emoji_collection`. for emoji_dict in EMOJI_DATA: - if not emoji_is_universal(emoji_dict): + if not emoji_is_supported(emoji_dict): continue category = emoji_dict["category"] emoji_code = get_emoji_code(emoji_dict) diff --git a/version.py b/version.py index 3148f33c42..f82920ecd6 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 230 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (254, 0) +PROVISION_VERSION = (255, 0) diff --git a/web/tests/emoji_picker.test.js b/web/tests/emoji_picker.test.js index 6ed98f8866..448c2f42cd 100644 --- a/web/tests/emoji_picker.test.js +++ b/web/tests/emoji_picker.test.js @@ -23,7 +23,7 @@ run_test("initialize", () => { const complete_emoji_catalog = _.sortBy(emoji_picker.complete_emoji_catalog, "name"); assert.equal(complete_emoji_catalog.length, 11); - assert.equal(emoji.emojis_by_name.size, 1827); + assert.equal(emoji.emojis_by_name.size, 1848); let total_emoji_in_categories = 0; @@ -45,15 +45,15 @@ run_test("initialize", () => { const popular_emoji_count = 6; const zulip_emoji_count = 1; assert_emoji_category(complete_emoji_catalog.pop(), "fa-car", 195); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-hashtag", 221); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-smile-o", 162); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-hashtag", 223); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-smile-o", 166); assert_emoji_category(complete_emoji_catalog.pop(), "fa-star-o", popular_emoji_count); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-thumbs-o-up", 361); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-lightbulb-o", 257); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-cutlery", 131); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-thumbs-o-up", 363); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-lightbulb-o", 261); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-cutlery", 133); assert_emoji_category(complete_emoji_catalog.pop(), "fa-flag", 269); assert_emoji_category(complete_emoji_catalog.pop(), "fa-cog", 1); - assert_emoji_category(complete_emoji_catalog.pop(), "fa-leaf", 145); + assert_emoji_category(complete_emoji_catalog.pop(), "fa-leaf", 152); assert_emoji_category(complete_emoji_catalog.pop(), "fa-soccer-ball-o", 85); // The popular emoji appear twice in the picker, and the zulip emoji is special