zulip/tools/setup/emoji/build_emoji

#!/usr/bin/env python3
#
# See docs/subsystems/emoji.md for a high-level explanation of how this system
# works.
import os
import shutil
import sys
import ujson

from typing import Any, Dict, List, Optional

from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \
    get_emoji_code, generate_name_to_codepoint_map, emoji_names_for_picker, \
    EMOTICON_CONVERSIONS, REMAPPED_EMOJIS
from emoji_names import EMOJI_NAME_MAPS

ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
sys.path.append(ZULIP_PATH)

from scripts.lib.zulip_tools import generate_sha1sum_emoji

TARGET_EMOJI_DUMP = os.path.join(ZULIP_PATH, 'static', 'generated', 'emoji')
EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'emoji')
NODE_MODULES_PATH = os.path.join(ZULIP_PATH, 'node_modules')

SPRITE_CSS_FILE_TEMPLATE = """\
div.emoji,
span.emoji
{
    display: inline-block;
    background-image: url('sheet-%(emojiset)s-64.png');
    -webkit-background-size: 5200%%;
    -moz-background-size: 5200%%;
    background-size: 5200%%;
    background-repeat: no-repeat;

    /* Hide the text. */
    text-indent: 100%%;
    white-space: nowrap;
    overflow: hidden;
}

.emoji-1f419
{
    background-image: url('images-google-64/1f419.png') !important;
    background-position: 0%% 0%% !important;
    background-size: contain !important;
}

%(emoji_positions)s
"""

EMOJI_POS_INFO_TEMPLATE = """\
.emoji-%(codepoint)s {
    background-position: %(pos_x)s%% %(pos_y)s%%;
}
"""

# change directory
os.chdir(EMOJI_SCRIPT_DIR_PATH)

if 'TRAVIS' in os.environ:
    # In Travis CI, we don't have root access
    EMOJI_CACHE_PATH = "/home/travis/zulip-emoji-cache"

def main() -> None:
    success_stamp = get_success_stamp()
    source_emoji_dump = os.path.dirname(success_stamp)

    if not os.path.exists(success_stamp):
        print("Dumping emojis ...")
        dump_emojis(source_emoji_dump)
        with open(success_stamp, 'w') as f:
            f.close()

    print("Using cached emojis from {}".format(source_emoji_dump))
    if os.path.lexists(TARGET_EMOJI_DUMP):
        os.remove(TARGET_EMOJI_DUMP)
    os.symlink(source_emoji_dump, TARGET_EMOJI_DUMP)

def get_success_stamp() -> str:
    sha1_hexdigest = generate_sha1sum_emoji(ZULIP_PATH)
    return os.path.join(EMOJI_CACHE_PATH, sha1_hexdigest, 'emoji', '.success-stamp')

def generate_sprite_css_files(cache_path: str,
                              emoji_data: List[Dict[str, Any]],
                              emojiset: str) -> None:
    def get_max_val(field: str, emoji_data: List[Dict[str, Any]]) -> int:
        max_val = 0
        for emoji_dict in emoji_data:
            max_val = max(max_val, emoji_dict[field])
            if 'skin_variations' in emoji_dict:
                for skin_tone, img_info in emoji_dict['skin_variations'].items():
                    max_val = max(max_val, img_info[field])
        return max_val

    # Spritesheet CSS generation code. Spritesheets are squared using
    # padding, so we have to take only the maximum of two dimensions.
    nrows = get_max_val('sheet_x', emoji_data)
    ncols = get_max_val('sheet_y', emoji_data)
    max_dim = max(nrows, ncols)
    emoji_positions = ""
    for emoji in emoji_data:
        if emoji["has_img_google"]:
            # Why is the test here has_img_google and not
            # emoji_is_universal?  Because we briefly supported all
            # Google emoji (not just the universal ones), we need to
            # ensure the spritesheet is setup to correctly display
            # those google emoji (in case anyone used them).
            emoji_positions += EMOJI_POS_INFO_TEMPLATE % {
                'codepoint': get_emoji_code(emoji),
                'pos_x': (emoji["sheet_x"] * 100) / max_dim,
                'pos_y': (emoji["sheet_y"] * 100) / max_dim,
            }

    SPRITE_CSS_PATH = os.path.join(cache_path, '%s-sprite.css' % (emojiset,))
    with open(SPRITE_CSS_PATH, 'w') as f:
        f.write(SPRITE_CSS_FILE_TEMPLATE % {'emojiset': emojiset,
                                            'emoji_positions': emoji_positions,
                                            })

def setup_emoji_farms(cache_path: str, emoji_data: List[Dict[str, Any]]) -> None:
    def ensure_emoji_image(emoji_dict: Dict[str, Any],
                           src_emoji_farm: str,
                           target_emoji_farm: str) -> None:
        # We use individual images from emoji farm for rendering emojis
        # in notification messages. We have a custom emoji formatter in
        # notifications processing code that converts `span` tags to
        # `img` and that logic requires us to have non-qualified
        # `emoji_code` as file name for emoji.
        emoji_code = get_emoji_code(emoji_dict)
        img_file_name = emoji_code + '.png'
        src_file = os.path.join(src_emoji_farm, emoji_dict['image'])
        dst_file = os.path.join(target_emoji_farm, img_file_name)
        shutil.copy2(src_file, dst_file)

    def setup_emoji_farm(emojiset: str,
                         emoji_data: List[Dict[str, Any]],
                         alt_name: Optional[str]=None) -> None:
        # `alt_name` is an optional parameter that we use to avoid duplicating below
        # code. It is only used while setting up google-blob emojiset as it is just
        # a wrapper for an older version of emoji-datasource package due to which we
        # need to use 'google' at some places in this code. It has no meaning for other
        # emojisets and is just equivalent to `emojiset`.
        alt_name = alt_name or emojiset

        # Copy individual emoji images from npm packages.
        src_emoji_farm = os.path.join(
            NODE_MODULES_PATH, 'emoji-datasource-' + emojiset, 'img', alt_name, '64')
        target_emoji_farm = os.path.join(cache_path, 'images-' + emojiset + '-64')
        os.makedirs(target_emoji_farm, exist_ok=True)
        print("Copying individual image files...")
        for emoji_dict in emoji_data:
            if emoji_dict['has_img_' + alt_name]:
                ensure_emoji_image(emoji_dict, src_emoji_farm, target_emoji_farm)
                skin_variations = emoji_dict.get('skin_variations', {})
                for skin_tone, img_info in skin_variations.items():
                    if img_info['has_img_' + alt_name]:
                        ensure_emoji_image(img_info, src_emoji_farm, target_emoji_farm)

        # Copy zulip.png to the emoji farm.
        zulip_image = os.path.join(ZULIP_PATH, 'static', 'assets', 'zulip-emoji')
        for f in os.listdir(zulip_image):
            shutil.copy2(os.path.join(zulip_image, f), target_emoji_farm, follow_symlinks=False)

        # Copy spritesheets.
        emoji_data_path = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-' + emojiset)
        input_sprite_sheet = os.path.join(emoji_data_path, 'img', alt_name, 'sheets-256', '64.png')
        output_sprite_sheet = os.path.join(cache_path, 'sheet-%s-64.png' % (emojiset,))
        shutil.copyfile(input_sprite_sheet, output_sprite_sheet)

        # We hardcode octopus emoji image to Google emojiset's old
        # "cute octopus" image. Copy it to the emoji farms.
        input_img_file = os.path.join(EMOJI_SCRIPT_DIR_PATH, '1f419.png')
        output_img_file = os.path.join(cache_path, 'images-' + emojiset + '-64', '1f419.png')
        shutil.copyfile(input_img_file, output_img_file)

        generate_sprite_css_files(cache_path, emoji_data, emojiset)

    # Setup standard emojisets.
    for emojiset in ['google', 'twitter']:
        setup_emoji_farm(emojiset, emoji_data)

    # Setup old google "blobs" emojiset.
    GOOGLE_BLOB_EMOJI_DATA_PATH = os.path.join(NODE_MODULES_PATH,
                                               'emoji-datasource-google-blob',
                                               'emoji.json')
    with open(GOOGLE_BLOB_EMOJI_DATA_PATH) as fp:
        blob_emoji_data = ujson.load(fp)
    setup_emoji_farm('google-blob', blob_emoji_data, 'google')

def setup_old_emoji_farm(cache_path: str,
                         emoji_map: Dict[str, str],
                         emoji_data: List[Dict[str, Any]]) -> None:
    # Code for setting up old emoji farm.
    os.chdir(cache_path)
    emoji_cache_path = os.path.join(cache_path, 'images', 'emoji')
    unicode_emoji_cache_path = os.path.join(cache_path, 'images', 'emoji', 'unicode')
    google_emoji_cache_path = os.path.join(cache_path, 'images-google-64')
    os.makedirs(emoji_cache_path, exist_ok=True)
    os.makedirs(unicode_emoji_cache_path, exist_ok=True)

    # Symlink zulip.png image file.
    image_file_path = os.path.join(google_emoji_cache_path, 'zulip.png')
    symlink_path = os.path.join(emoji_cache_path, 'zulip.png')
    os.symlink(image_file_path, symlink_path)

    unicode_symlink_path = os.path.join(unicode_emoji_cache_path, 'zulip.png')
    os.symlink(image_file_path, unicode_symlink_path)

    for name, codepoint in emoji_map.items():
        mapped_codepoint = REMAPPED_EMOJIS.get(codepoint, codepoint)
        image_file_path = os.path.join(google_emoji_cache_path, '{}.png'.format(mapped_codepoint))
        symlink_path = os.path.join(emoji_cache_path, '{}.png'.format(name))
        os.symlink(image_file_path, symlink_path)
        try:
            # `emoji_map` contains duplicate entries for the same codepoint with different
            # names. So creation of symlink for <codepoint>.png may throw `FileExistsError`.
            unicode_symlink_path = os.path.join(unicode_emoji_cache_path, '{}.png'.format(codepoint))
            os.symlink(image_file_path, unicode_symlink_path)
        except FileExistsError:
            pass

def generate_map_files(cache_path: str, emoji_catalog: Dict[str, List[str]]) -> None:
    # This function generates the data file consumed by webapp, mobile apps, bugdown etc.
    names = emoji_names_for_picker(EMOJI_NAME_MAPS)
    codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
    name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)

    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.json')
    with open(EMOJI_CODES_FILE_PATH, 'w') as emoji_codes_file:
        ujson.dump({
            'names': names,
            'name_to_codepoint': name_to_codepoint,
            'codepoint_to_name': codepoint_to_name,
            'emoji_catalog': emoji_catalog,
            'emoticon_conversions': EMOTICON_CONVERSIONS,
        }, emoji_codes_file)

def dump_emojis(cache_path: str) -> None:
    with open('emoji_map.json') as emoji_map_file:
        emoji_map = ujson.load(emoji_map_file)

    # `emoji.json` or any other data file can be sourced from any of the supported
    # emojiset packages, they all contain the same data files.
    EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-google', 'emoji.json')
    with open(EMOJI_DATA_FILE_PATH) as emoji_data_file:
        emoji_data = ujson.load(emoji_data_file)
    emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS)

    # Setup emoji farms.
    if os.path.exists(cache_path):
        shutil.rmtree(cache_path)
    setup_emoji_farms(cache_path, emoji_data)
    setup_old_emoji_farm(cache_path, emoji_map, emoji_data)

    # Generate various map files.
    generate_map_files(cache_path, emoji_catalog)

if __name__ == "__main__":
    main()