zulip/tools/setup/emoji/build_emoji

#!/usr/bin/env python3
#
# See docs/emoji.md for a high-level explanation of how this system
# works.
from __future__ import division, print_function
import os
import glob
import shutil
import subprocess
import ujson
import sys
import hashlib
import xml.etree.ElementTree as ET
from six import unichr
from typing import Dict, Text, Union
from os.path import dirname
from PIL import Image, ImageDraw, ImageFont

from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \
    emoji_names_for_picker, EMOJISETS

ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
sys.path.append(ZULIP_PATH)

from scripts.lib.zulip_tools import run

AA_SCALE = 8
SIZE = (136, 136)
BIG_SIZE = tuple([x * AA_SCALE for x in SIZE])

EMOJI_DUMP_DIR_PATH = os.path.join(ZULIP_PATH, 'var', 'emoji_dump')
EMOJI_DUMP_PATH = lambda p: os.path.join(EMOJI_DUMP_DIR_PATH, p)
TARGET_EMOJI_DUMP = os.path.join(ZULIP_PATH, 'static', 'generated', 'emoji')
EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'emoji')
EMOJI_DATA_PATH = os.path.join(ZULIP_PATH, 'node_modules', 'emoji-datasource')

EMOJI_CODES_FILE_TEMPLATE = """\
var emoji_codes = (function () {
var exports = {};

exports.names = %(names)s;

exports.codepoints = %(codepoints)s;

exports.name_to_codepoint = %(name_to_codepoint)s;

exports.codepoint_to_name = %(codepoint_to_name)s;

exports.emoji_catalog = %(emoji_catalog)s;

exports.patched_css_classes = %(patched_css_classes)s;

return exports;
}());
if (typeof module !== 'undefined') {
    module.exports = emoji_codes;
}
"""

SPRITE_CSS_FILE_TEMPLATE = """\
div.emoji,
span.emoji
{
    display: inline-block;
    background-image: url('sheet_%(emojiset)s_32.png');
    -webkit-background-size: 4900%%;
    -moz-background-size: 4900%%;
    background-size: 4900%%;
    background-repeat: no-repeat;

    /* Hide the text. */
    text-indent: 100%%;
    white-space: nowrap;
    overflow: hidden;
}

%(emoji_positions)s
"""

EMOJI_POS_INFO_TEMPLATE = """\
.emoji-%(codepoint)s {
    background-position: %(pos_x)s%% %(pos_y)s%%;
}
"""

# change directory
os.chdir(EMOJI_SCRIPT_DIR_PATH)

if 'TRAVIS' in os.environ:
    # In Travis CI, we don't have root access
    EMOJI_CACHE_PATH = "/home/travis/zulip-emoji-cache"

class MissingGlyphError(Exception):
    pass

def color_font(name, code_point, code_point_to_fname_map):
    # type: (str, str, Dict[int, Union[Text, bytes]]) -> None
    glyph_name = code_point_to_fname_map[int(code_point, 16)]

    in_name = 'bitmaps/strike0/{}.png'.format(glyph_name)
    out_name = 'out/unicode/{}.png'.format(code_point)

    # These emojis are colored white and need to be recolored
    white_emojis = ['eight', 'five', 'four', 'hash', 'nine', 'one',
                    'seven', 'six', 'three', 'two', 'zero']

    try:
        if name in white_emojis:
            white_emoji_image = Image.open(in_name).convert('RGBA')
            # Reduced image size for having a 4-pixel dark yellow background
            # on right and bottom of the image.
            light_yellow_background = Image.new('RGBA', (124, 124), '#FCC21B')
            dark_yellow_background = Image.new('RGBA', SIZE, '#F79329')
            # Paste the image on a light yellow background and the resulting
            # image on a dark yellow background.
            light_yellow_background.paste(white_emoji_image, mask=white_emoji_image)
            dark_yellow_background.paste(light_yellow_background, mask=light_yellow_background)
            dark_yellow_background.save(in_name)
        shutil.copyfile(in_name, out_name)
    except IOError:
        raise MissingGlyphError('code_point: %r' % (code_point))


def bw_font(name, code_point):
    # type: (str, str) -> None
    char = unichr(int(code_point, 16))

    # AndroidEmoji.ttf is from
    # https://android.googlesource.com/platform/frameworks/base.git/+/master/data/fonts/AndroidEmoji.ttf
    # commit 07912f876c8639f811b06831465c14c4a3b17663
    font = ImageFont.truetype('AndroidEmoji.ttf', 65 * AA_SCALE)
    image = Image.new('RGBA', BIG_SIZE)
    draw = ImageDraw.Draw(image)
    draw.text((0, 0), char, font=font, fill='black')
    image.resize(SIZE, Image.ANTIALIAS).save(
        'out/unicode/{}.png'.format(code_point), 'PNG'
    )

def code_point_to_file_name_map(ttx):
    # type: (str) -> Dict[int, Union[Text, bytes]]
    """Given the NotoColorEmoji.ttx file, parse it to generate a map from
    codepoint to filename (a la glyph0****.png)
    """
    result = {}  # type: Dict[int, Union[Text, bytes]]
    xml = ET.parse(ttx)
    cmap = xml.find("*cmap_format_12")
    assert cmap is not None
    for elem in cmap:
        code_point = int(elem.attrib["code"], 16)
        fname = elem.attrib["name"]
        result[code_point] = fname
    return result


def main():
    # type: () -> None
    # ttx is in the fonttools pacakge, the -z option is only on master
    # https://github.com/behdad/fonttools/

    # NotoColorEmoji.tff is from
    # https://android.googlesource.com/platform/external/noto-fonts/+/master/other/NotoColorEmoji.ttf
    # Commit ID: 1e75a5582b3fb386725aaa944f32fba71f155588

    # this is so we don't accidently leave ttx files from previous
    # runs of this script lying around
    for fname in glob.glob(EMOJI_DUMP_PATH("*ttx*")):
        os.remove(fname)

    # check if directory `var/emoji_dump` exists
    subprocess.check_call(['mkdir', '-p', EMOJI_DUMP_DIR_PATH])
    success_stamp = get_success_stamp()
    source_emoji_dump = dirname(success_stamp)

    if not os.path.exists(success_stamp):
        print("Dumping emojis ...")
        dump_emojis(source_emoji_dump)
        run(['touch', success_stamp])

    print("Using cached emojis from {}".format(source_emoji_dump))
    run(['rm', '-rf', TARGET_EMOJI_DUMP])
    try:
        os.symlink(
            source_emoji_dump,
            TARGET_EMOJI_DUMP
        )
    except OSError:
        print("Error: Unable to create symlinks. Make sure you have permission to create symbolic links.")

def get_success_stamp():
    # type: () -> str
    sha = hashlib.sha1()

    filenames = ['NotoColorEmoji.ttf', 'emoji_map.json', 'AndroidEmoji.ttf',
                 'build_emoji', 'emoji_setup_utils.py']

    for filename in filenames:
        with open(filename, 'rb') as reader:
            sha.update(reader.read())

    # Take into account the version of `emoji-datasource` package while generating success stamp.
    PACKAGE_FILE_PATH = os.path.join(ZULIP_PATH, 'package.json')
    with open(PACKAGE_FILE_PATH, 'r') as fp:
        parsed_package_file = ujson.load(fp)
        dependency_data = parsed_package_file['dependencies']
        emoji_datasource_version = dependency_data['emoji-datasource'].encode('utf-8')
    sha.update(emoji_datasource_version)

    return os.path.join(EMOJI_CACHE_PATH, sha.hexdigest(), 'emoji', '.success-stamp')

def dump_emojis(cache_path):
    # type: (str) -> None
    subprocess.call('ttx -v -z extfile -d {} NotoColorEmoji.ttf'.format(EMOJI_DUMP_DIR_PATH), shell=True)

    with open('emoji_map.json') as emoji_map_file:
        emoji_map = ujson.load(emoji_map_file)

    code_point_to_fname_map = code_point_to_file_name_map(EMOJI_DUMP_PATH("NotoColorEmoji.ttx"))

    EMOJI_DATA_FILE_PATH = os.path.join(EMOJI_DATA_PATH, 'emoji.json')
    with open(EMOJI_DATA_FILE_PATH) as emoji_data_file:
        emoji_data = ujson.load(emoji_data_file)
    emoji_catalog = generate_emoji_catalog(emoji_data)

    UNIFIED_REACTIONS_PATH = os.path.join(ZULIP_PATH, 'zerver', 'management', 'data', 'unified_reactions.json')
    with open(UNIFIED_REACTIONS_PATH) as unified_reactions_file:
        unified_reactions_data = ujson.load(unified_reactions_file)

    os.chdir(EMOJI_DUMP_DIR_PATH)

    try:
        shutil.rmtree('out')
    except OSError:
        pass

    os.mkdir('out')
    os.mkdir('out/unicode')

    failed = False
    for name, code_point in emoji_map.items():
        try:
            color_font(name, code_point, code_point_to_fname_map)
        except MissingGlyphError:
            print("Warning: Missing color glyph for %s; using black/white." % (name,))
            try:
                bw_font(name, code_point)
            except Exception as e:
                print(e)
                print('Missing {}, {}'.format(name, code_point))
                failed = True
                continue

        try:
            os.symlink(
                'unicode/{}.png'.format(code_point),
                'out/{}.png'.format(name)
            )
        except OSError:
            print("Error: Unable to create symlinks. Make sure you have permission to create symbolic links.")
            failed = True
            # probably should not try to create additional links
            break

    if failed:
        print("Errors dumping emoji!")
        sys.exit(1)

    cache_emoji = os.path.join(cache_path, 'images', 'emoji')
    cache_emoji_unicode = os.path.join(cache_path, 'images', 'emoji', 'unicode')
    run(['rm', '-rf', cache_path])
    run(['mkdir', '-p', cache_emoji])
    run(['mv', 'out/*', cache_emoji], shell=True)
    assets = "{}/static/assets/zulip-emoji/*".format(ZULIP_PATH)
    run(['cp', '-RPp', assets, cache_emoji_unicode], shell=True)

    for fn in [os.path.basename(file_name) for file_name in glob.glob(assets)]:
        os.symlink(
            os.path.join(cache_emoji_unicode, fn),
            os.path.join(cache_emoji, fn)
        )

    # Spritesheet CSS generation code.
    emoji_positions = ""
    for emoji in emoji_data:
        if emoji["has_img_google"]:
            emoji_positions += EMOJI_POS_INFO_TEMPLATE % {
                'codepoint': emoji['unified'].lower(),
                'pos_x': (emoji["sheet_x"] * 100) / 48,
                'pos_y': (emoji["sheet_y"] * 100) / 48,
            }
            # Remove the code below once the migration to iamcal's dataset is complete.
            emoji_name = emoji['short_name']
            codepoint = emoji['unified'].lower()
            if emoji_name in emoji_map and codepoint != emoji_map[emoji_name]:
                emoji_positions += EMOJI_POS_INFO_TEMPLATE % {
                    'codepoint': emoji_map[emoji_name],
                    'pos_x': (emoji["sheet_x"] * 100) / 48,
                    'pos_y': (emoji["sheet_y"] * 100) / 48,
                }

    for emojiset in EMOJISETS:
        input_sprite_sheet = os.path.join(EMOJI_DATA_PATH, 'img', emojiset, 'sheets', '32.png')
        output_sprite_sheet = os.path.join(cache_path, 'sheet_%s_32.png' % (emojiset,))
        run(['cp', input_sprite_sheet, output_sprite_sheet], shell=True)
        SPRITE_CSS_PATH = os.path.join(cache_path, '%s_sprite.css' % (emojiset,))
        sprite_css_file = open(SPRITE_CSS_PATH, 'w')
        sprite_css_file.write(SPRITE_CSS_FILE_TEMPLATE % {'emojiset': emojiset,
                                                          'emoji_positions': emoji_positions,
                                                          })
        sprite_css_file.close()

    EMOJI_CODES_PATH = os.path.join(cache_path, 'emoji_codes.js')
    emoji_codes_file = open(EMOJI_CODES_PATH, 'w')

    # put thumbs_up before thumbs_down
    names = emoji_names_for_picker(emoji_map)
    down_index = names.index('thumbs_down')
    up_index = names.index('thumbs_up')
    names[down_index], names[up_index] = ('thumbs_up', 'thumbs_down')

    # Patch CSS classes of flag emojis.
    patched_css_classes = {}
    for emoji in emoji_data:
        if emoji['category'] == 'Flags':
            for name in emoji['short_names']:
                if name in emoji_map:
                    patched_css_classes[str(name)] = str(emoji['unified'].lower())

    codepoint_to_name = generate_codepoint_to_name_map(names, unified_reactions_data)

    emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % {
        'names': names,
        'codepoints': sorted([str(code_point) for code_point in set(emoji_map.values())]),
        'name_to_codepoint': {str(key): str(emoji_map[key]) for key in emoji_map},
        'codepoint_to_name': codepoint_to_name,
        'emoji_catalog': emoji_catalog,
        'patched_css_classes': patched_css_classes
    })
    emoji_codes_file.close()

    NAME_TO_CODEPOINT_PATH = os.path.join(cache_path, 'name_to_codepoint.json')
    name_to_codepoint_file = open(NAME_TO_CODEPOINT_PATH, 'w')

    name_to_codepoint_file.write(ujson.dumps(emoji_map))
    name_to_codepoint_file.close()

    CODEPOINT_TO_NAME_PATH = os.path.join(cache_path, 'codepoint_to_name.json')
    codepoint_to_name_file = open(CODEPOINT_TO_NAME_PATH, 'w')

    codepoint_to_name_file.write(ujson.dumps(codepoint_to_name))
    codepoint_to_name_file.close()

if __name__ == "__main__":
    main()