#!/usr/bin/env python # # See docs/emoji.md for a high-level explanation of how this system # works. from __future__ import division, print_function import os import glob import shutil import subprocess import ujson import sys import hashlib import xml.etree.ElementTree as ET from six import unichr from typing import Dict, Text, Union from os.path import dirname from PIL import Image, ImageDraw, ImageFont from emoji_setup_utils import generate_emoji_catalog, emoji_names_for_picker, \ EMOJISETS ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import run AA_SCALE = 8 SIZE = (136, 136) SPRITE_SIZE = (50, 50) BIG_SIZE = tuple([x * AA_SCALE for x in SIZE]) EMOJI_DUMP_DIR_PATH = os.path.join(ZULIP_PATH, 'var', 'emoji_dump') EMOJI_DUMP_PATH = lambda p: os.path.join(EMOJI_DUMP_DIR_PATH, p) TARGET_EMOJI_DUMP = os.path.join(ZULIP_PATH, 'static', 'generated', 'emoji') EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache" EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'emoji') EMOJI_DATA_PATH = os.path.join(ZULIP_PATH, 'static', 'third', 'emoji-data') EMOJI_CODES_FILE_TEMPLATE = """\ var emoji_codes = (function () { var exports = {}; exports.names = %(names)s; exports.codepoints = %(codepoints)s; exports.name_to_codepoint = %(name_to_codepoint)s; exports.emoji_catalog = %(emoji_catalog)s; return exports; }()); if (typeof module !== 'undefined') { module.exports = emoji_codes; } """ SPRITE_CSS_FILE_TEMPLATE = """\ div.emoji, span.emoji { display: inline-block; background-image: url('sheet_%(emojiset)s_32.png'); -webkit-background-size: 1025px 1025px; -moz-background-size: 1025px 1025px; background-size: 1025px 1025px; background-repeat: no-repeat; /* Hide the text. */ text-indent: 100%%; white-space: nowrap; overflow: hidden; } %(emoji_positions)s """ EMOJI_POS_INFO_TEMPLATE = """\ .emoji-%(codepoint)s { background-position: %(pos_x)s%% %(pos_y)s%%; } """ # change directory os.chdir(EMOJI_SCRIPT_DIR_PATH) if 'TRAVIS' in os.environ: # In Travis CI, we don't have root access EMOJI_CACHE_PATH = "/home/travis/zulip-emoji-cache" class MissingGlyphError(Exception): pass def color_font(name, code_point, code_point_to_fname_map): # type: (str, str, Dict[int, Union[Text, bytes]]) -> None glyph_name = code_point_to_fname_map[int(code_point, 16)] in_name = 'bitmaps/strike0/{}.png'.format(glyph_name) out_name = 'out/unicode/{}.png'.format(code_point) out_sprite_name = 'out/sprite/{}.png'.format(name) # These emojis are colored white and need to be recolored white_emojis = ['eight', 'five', 'four', 'hash', 'nine', 'one', 'seven', 'six', 'three', 'two', 'zero'] try: if name in white_emojis: white_emoji_image = Image.open(in_name).convert('RGBA') # Paste blue onto the image using the number as a mask white_emoji_image.paste('#40C0E7', white_emoji_image) white_emoji_image.save(in_name) shutil.copyfile(in_name, out_name) image = Image.new('RGBA', SIZE) image.paste(Image.open(out_name), (0, 2)) image.resize(SPRITE_SIZE, Image.ANTIALIAS).save(out_sprite_name, 'PNG') except IOError: raise MissingGlyphError('code_point: %r' % (code_point)) def bw_font(name, code_point): # type: (str, str) -> None char = unichr(int(code_point, 16)) # AndroidEmoji.ttf is from # https://android.googlesource.com/platform/frameworks/base.git/+/master/data/fonts/AndroidEmoji.ttf # commit 07912f876c8639f811b06831465c14c4a3b17663 font = ImageFont.truetype('AndroidEmoji.ttf', 65 * AA_SCALE) image = Image.new('RGBA', BIG_SIZE) draw = ImageDraw.Draw(image) draw.text((0, 0), char, font=font, fill='black') image.resize(SIZE, Image.ANTIALIAS).save( 'out/unicode/{}.png'.format(code_point), 'PNG' ) image.resize(SPRITE_SIZE, Image.ANTIALIAS).save( 'out/sprite/{}.png'.format(name), 'PNG' ) def code_point_to_file_name_map(ttx): # type: (str) -> Dict[int, Union[Text, bytes]] """Given the NotoColorEmoji.ttx file, parse it to generate a map from codepoint to filename (a la glyph0****.png) """ result = {} # type: Dict[int, Union[Text, bytes]] xml = ET.parse(ttx) for elem in xml.find("*cmap_format_12"): code_point = int(elem.attrib["code"], 16) fname = elem.attrib["name"] result[code_point] = fname return result def main(): # type: () -> None # ttx is in the fonttools pacakge, the -z option is only on master # https://github.com/behdad/fonttools/ # NotoColorEmoji.tff is from # https://android.googlesource.com/platform/external/noto-fonts/+/master/other/NotoColorEmoji.ttf # Commit ID: 1e75a5582b3fb386725aaa944f32fba71f155588 # this is so we don't accidently leave ttx files from previous # runs of this script lying around for fname in glob.glob(EMOJI_DUMP_PATH("*ttx*")): os.remove(fname) # check if directory `var/emoji_dump` exists subprocess.check_call(['mkdir', '-p', EMOJI_DUMP_DIR_PATH]) success_stamp = get_success_stamp() source_emoji_dump = dirname(success_stamp) if not os.path.exists(success_stamp): print("Dumping emojis ...") dump_emojis(source_emoji_dump) run(['touch', success_stamp]) print("Using cached emojis from {}".format(source_emoji_dump)) run(['rm', '-rf', TARGET_EMOJI_DUMP]) try: os.symlink( source_emoji_dump, TARGET_EMOJI_DUMP ) except OSError: print("Error: Unable to create symlinks. Make sure you have permission to create symbolic links.") def get_success_stamp(): # type: () -> str sha = hashlib.sha1() filenames = ['NotoColorEmoji.ttf', 'emoji_map.json', 'AndroidEmoji.ttf', 'build_emoji', 'emoji_setup_utils.py'] for filename in filenames: with open(filename, 'rb') as reader: sha.update(reader.read()) return os.path.join(EMOJI_CACHE_PATH, sha.hexdigest(), 'emoji', '.success-stamp') def dump_emojis(cache_path): # type: (str) -> None subprocess.call('ttx -v -z extfile -d {} NotoColorEmoji.ttf'.format(EMOJI_DUMP_DIR_PATH), shell=True) emoji_map = ujson.load(open('emoji_map.json')) code_point_to_fname_map = code_point_to_file_name_map(EMOJI_DUMP_PATH("NotoColorEmoji.ttx")) emoji_data = ujson.load(open(os.path.join(EMOJI_DATA_PATH, 'emoji.json'))) emoji_catalog = generate_emoji_catalog(emoji_data) os.chdir(EMOJI_DUMP_DIR_PATH) try: shutil.rmtree('out') except OSError: pass for fname in glob.glob("sprite*"): os.remove(fname) os.mkdir('out') os.mkdir('out/sprite') os.mkdir('out/unicode') failed = False for name, code_point in emoji_map.items(): try: color_font(name, code_point, code_point_to_fname_map) except MissingGlyphError: print("Warning: Missing color glyph for %s; using black/white." % (name,)) try: bw_font(name, code_point) except Exception as e: print(e) print('Missing {}, {}'.format(name, code_point)) failed = True continue try: os.symlink( 'unicode/{}.png'.format(code_point), 'out/{}.png'.format(name) ) except OSError: print("Error: Unable to create symlinks. Make sure you have permission to create symbolic links.") failed = True # probably should not try to create additional links break if failed: print("Errors dumping emoji!") sys.exit(1) subprocess.call('glue --quiet out/sprite . --margin=1 --namespace=emoji --sprite-namespace= --retina', shell=True) cache_emoji = os.path.join(cache_path, 'images', 'emoji') cache_emoji_unicode = os.path.join(cache_path, 'images', 'emoji', 'unicode') run(['rm', '-rf', cache_path]) run(['mkdir', '-p', cache_emoji]) run(['mv', 'out/*', cache_emoji], shell=True) run(['mv', 'sprite*', cache_path], shell=True) assets = "{}/static/assets/zulip-emoji/*".format(ZULIP_PATH) run(['cp', '-RPp', assets, cache_emoji_unicode], shell=True) for fn in [os.path.basename(file_name) for file_name in glob.glob(assets)]: os.symlink( os.path.join(cache_emoji_unicode, fn), os.path.join(cache_emoji, fn) ) # Spritesheet CSS generation code. emoji_positions = "" for emoji in emoji_data: if emoji["has_img_google"]: emoji_positions += EMOJI_POS_INFO_TEMPLATE % {'codepoint': emoji['unified'], 'pos_x': (emoji["sheet_x"] * 100) / 40, 'pos_y': (emoji["sheet_y"] * 100) / 40, } for emojiset in EMOJISETS: sprite_sheet = os.path.join(EMOJI_DATA_PATH, 'sheet_%s_32.png' % (emojiset,)) run(['cp', sprite_sheet, cache_path], shell=True) SPRITE_CSS_PATH = os.path.join(cache_path, '%s_sprite.css' % (emojiset,)) sprite_css_file = open(SPRITE_CSS_PATH, 'w') sprite_css_file.write(SPRITE_CSS_FILE_TEMPLATE % {'emojiset': emojiset, 'emoji_positions': emoji_positions, }) sprite_css_file.close() # Add zulip emoji to `emoji_map` so that we can avoid # adding unnecessary exceptions to the rendering logic. emoji_map['zulip'] = 'zulip' EMOJI_CODES_PATH = os.path.join(cache_path, 'emoji_codes.js') emoji_codes_file = open(EMOJI_CODES_PATH, 'w') # put thumbs_up before thumbs_down names = emoji_names_for_picker(emoji_map) down_index = names.index('thumbs_down') up_index = names.index('thumbs_up') names[down_index], names[up_index] = ('thumbs_up', 'thumbs_down') emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % { 'names': names, 'codepoints': sorted([str(code_point) for code_point in set(emoji_map.values())]), 'name_to_codepoint': {str(key): str(emoji_map[key]) for key in emoji_map}, 'emoji_catalog': emoji_catalog }) emoji_codes_file.close() NAME_TO_CODEPOINT_PATH = os.path.join(cache_path, 'name_to_codepoint.json') name_to_codepoint_file = open(NAME_TO_CODEPOINT_PATH, 'w') name_to_codepoint_file.write(ujson.dumps(emoji_map)) name_to_codepoint_file.close() if __name__ == "__main__": main()