Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tools: Remove empty wrapper script build_emoji. 

This renames the old `emoji_dump.py` to `build_emoji`, removing the
old shell essentially empty shell script.  `emoji_dump.py` was always
a weird name, and this makes it a bit easier to read the code for this
system.
This commit is contained in:
Prabhakar Gupta 2016-10-21 00:11:39 +05:30 committed by Tim Abbott
parent 2083ffa7a6
commit bc6421fbfb
5 changed files with 195 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/install-generic-unix-dev.md
View File

@ -283,7 +283,7 @@ Now run these commands:
```
./tools/install-mypy
./tools/setup/download-zxcvbn
./tools/setup/emoji_dump/build_emoji
python ./tools/setup/emoji_dump/build_emoji
./scripts/setup/generate_secrets.py -d
if [ $(uname) = "OpenBSD" ]; then sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/; else sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/9.*/tsearch_data/; fi
./scripts/setup/configure-rabbitmq

2
tools/provision.py
View File

@ -170,7 +170,7 @@ def main():
run(["mkdir", "-p", NODE_TEST_COVERAGE_DIR_PATH])
run(["tools/setup/download-zxcvbn"])
run(["tools/setup/emoji_dump/build_emoji"])
run(["python", "tools/setup/emoji_dump/build_emoji"])
run(["scripts/setup/generate_secrets.py", "--development"])
if TRAVIS and not PRODUCTION_TRAVIS:
run(["sudo", "service", "rabbitmq-server", "restart"])

196
tools/setup/emoji_dump/build_emoji
View File

@ -1,5 +1,193 @@
#!/usr/bin/env bash
set -e
set -x
#!/usr/bin/env python
from __future__ import print_function
import os
import glob
import shutil
import subprocess
import json
import sys
import hashlib
import xml.etree.ElementTree as ET
from six import unichr, text_type
from typing import Union
from os.path import dirname
from PIL import Image, ImageDraw, ImageFont
(cd tools/setup/emoji_dump && python ./emoji_dump.py)
ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run
AA_SCALE = 8
SIZE = (136, 136)
SPRITE_SIZE = (50, 50)
BIG_SIZE = tuple([x * AA_SCALE for x in SIZE])
EMOJI_DUMP_DIR_PATH = os.path.join(ZULIP_PATH, 'var', 'emoji_dump')
EMOJI_DUMP_PATH = lambda p: os.path.join(EMOJI_DUMP_DIR_PATH, p)
TARGET_EMOJI_DUMP = os.path.join(ZULIP_PATH, 'static', 'third', 'gemoji')
EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'emoji_dump')
# change directory
os.chdir(EMOJI_SCRIPT_DIR_PATH)
if 'TRAVIS' in os.environ:
# In Travis CI, we don't have root access
EMOJI_CACHE_PATH = "/home/travis/zulip-emoji-cache"
class MissingGlyphError(Exception):
pass
def color_font(name, code_point, code_point_to_fname_map):
# type: (str, str, Dict[int, Union[text_type, bytes]]) -> None
glyph_name = code_point_to_fname_map[int(code_point, 16)]
in_name = 'bitmaps/strike0/{}.png'.format(glyph_name)
out_name = 'out/unicode/{}.png'.format(code_point)
out_sprite_name = 'out/sprite/{}.png'.format(name)
try:
shutil.copyfile(in_name, out_name)
image = Image.new('RGBA', SIZE)
image.paste(Image.open(out_name), (0, 2))
image.resize(SPRITE_SIZE, Image.ANTIALIAS).save(out_sprite_name, 'PNG')
except IOError:
raise MissingGlyphError('code_point: %r' % (code_point))
def bw_font(name, code_point):
# type: (str, str) -> None
char = unichr(int(code_point, 16))
# AndroidEmoji.ttf is from
# https://android.googlesource.com/platform/frameworks/base.git/+/master/data/fonts/AndroidEmoji.ttf
# commit 07912f876c8639f811b06831465c14c4a3b17663
font = ImageFont.truetype('AndroidEmoji.ttf', 65 * AA_SCALE)
image = Image.new('RGBA', BIG_SIZE)
draw = ImageDraw.Draw(image)
draw.text((0, 0), char, font=font, fill='black')
image.resize(SIZE, Image.ANTIALIAS).save(
'out/unicode/{}.png'.format(code_point), 'PNG'
)
image.resize(SPRITE_SIZE, Image.ANTIALIAS).save(
'out/sprite/{}.png'.format(name), 'PNG'
)
def code_point_to_file_name_map(ttx):
# type: (str) -> Dict[int, Union[text_type, bytes]]
"""Given the NotoColorEmoji.ttx file, parse it to generate a map from
codepoint to filename (a la glyph0****.png)
"""
result = {} # type: Dict[int, Union[text_type, bytes]]
xml = ET.parse(ttx)
for elem in xml.find("*cmap_format_12"):
code_point = int(elem.attrib["code"], 16)
fname = elem.attrib["name"]
result[code_point] = fname
return result
def main():
# type: () -> None
# ttx is in the fonttools pacakge, the -z option is only on master
# https://github.com/behdad/fonttools/
# NotoColorEmoji.tff is from
# https://android.googlesource.com/platform/external/noto-fonts/+/marshmallow-release/other/NotoColorEmoji.ttf
# note that you have to run it though base64 -D before being able
# to use it, since it downloads from that page base64 encoded
# this is so we don't accidently leave ttx files from previous
# runs of this script lying around
for fname in glob.glob(EMOJI_DUMP_PATH("*ttx*")):
os.remove(fname)
# check if directory `var/emoji_dump` exists
subprocess.check_call(['mkdir', '-p', EMOJI_DUMP_DIR_PATH])
success_stamp = get_success_stamp()
source_emoji_dump = dirname(success_stamp)
if not os.path.exists(success_stamp):
print("Dumping emojis ...")
dump_emojis(source_emoji_dump)
run(['touch', success_stamp])
print("Using cached emojis from {}".format(source_emoji_dump))
run(['rm', '-rf', TARGET_EMOJI_DUMP])
run(['ln', '-nsf', source_emoji_dump, TARGET_EMOJI_DUMP])
def get_success_stamp():
# type: () -> str
sha = hashlib.sha1()
filenames = ['NotoColorEmoji.ttf', 'emoji_map.json', 'AndroidEmoji.ttf',
'build_emoji']
for filename in filenames:
with open(filename, 'rb') as reader:
sha.update(reader.read())
return os.path.join(EMOJI_CACHE_PATH, sha.hexdigest(), 'gemoji', '.success-stamp')
def dump_emojis(cache_path):
# type: (str) -> None
subprocess.call('ttx -v -z extfile -d {} NotoColorEmoji.ttf'.format(EMOJI_DUMP_DIR_PATH), shell=True)
emoji_map = json.load(open('emoji_map.json'))
# Fix data problem with red/blue cars being inaccurate.
emoji_map['blue_car'] = emoji_map['red_car']
emoji_map['red_car'] = emoji_map['oncoming_automobile']
code_point_to_fname_map = code_point_to_file_name_map(EMOJI_DUMP_PATH("NotoColorEmoji.ttx"))
os.chdir(EMOJI_DUMP_DIR_PATH)
try:
shutil.rmtree('out')
except OSError:
pass
for fname in glob.glob("sprite*"):
os.remove(fname)
os.mkdir('out')
os.mkdir('out/sprite')
os.mkdir('out/unicode')
failed = False
for name, code_point in emoji_map.items():
try:
color_font(name, code_point, code_point_to_fname_map)
except MissingGlyphError:
print("Warning: Missing color glyph for %s; using black/white." % (name,))
try:
bw_font(name, code_point)
except Exception as e:
print(e)
print('Missing {}, {}'.format(name, code_point))
failed = True
continue
os.symlink(
'unicode/{}.png'.format(code_point),
'out/{}.png'.format(name)
)
if failed:
print("Errors dumping emoji!")
sys.exit(1)
subprocess.call('glue --quiet out/sprite . --namespace=emoji --sprite-namespace= --retina',
shell=True)
cache_emoji = os.path.join(cache_path, 'images', 'emoji')
run(['sudo', 'rm', '-rf', cache_path])
run(['sudo', 'mkdir', '-p', cache_emoji])
run(["sudo", "chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), cache_path])
run(['mv', 'out/*', cache_emoji], shell=True)
run(['mv', 'sprite*', cache_path], shell=True)
assets = "{}/static/assets/zulip-emoji/*".format(ZULIP_PATH)
run(['cp', '-RPp', assets, cache_emoji], shell=True)
if __name__ == "__main__":
main()

189
tools/setup/emoji_dump/emoji_dump.py
View File

@ -1,189 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import glob
import shutil
import subprocess
import json
import sys
import hashlib
import xml.etree.ElementTree as ET
from six import unichr, text_type
from typing import Union
from os.path import dirname
from PIL import Image, ImageDraw, ImageFont
ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
sys.path.append(ZULIP_PATH)
from scripts.lib.zulip_tools import run
AA_SCALE = 8
SIZE = (136, 136)
SPRITE_SIZE = (50, 50)
BIG_SIZE = tuple([x * AA_SCALE for x in SIZE])
EMOJI_DUMP_DIR_PATH = os.path.join(ZULIP_PATH, 'var', 'emoji_dump')
EMOJI_DUMP_PATH = lambda p: os.path.join(EMOJI_DUMP_DIR_PATH, p)
TARGET_EMOJI_DUMP = os.path.join(ZULIP_PATH, 'static', 'third', 'gemoji')
EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
if 'TRAVIS' in os.environ:
# In Travis CI, we don't have root access
EMOJI_CACHE_PATH = "/home/travis/zulip-emoji-cache"
class MissingGlyphError(Exception):
pass
def color_font(name, code_point, code_point_to_fname_map):
# type: (str, str, Dict[int, Union[text_type, bytes]]) -> None
glyph_name = code_point_to_fname_map[int(code_point, 16)]
in_name = 'bitmaps/strike0/{}.png'.format(glyph_name)
out_name = 'out/unicode/{}.png'.format(code_point)
out_sprite_name = 'out/sprite/{}.png'.format(name)
try:
shutil.copyfile(in_name, out_name)
image = Image.new('RGBA', SIZE)
image.paste(Image.open(out_name), (0, 2))
image.resize(SPRITE_SIZE, Image.ANTIALIAS).save(out_sprite_name, 'PNG')
except IOError:
raise MissingGlyphError('code_point: %r' % (code_point))
def bw_font(name, code_point):
# type: (str, str) -> None
char = unichr(int(code_point, 16))
# AndroidEmoji.ttf is from
# https://android.googlesource.com/platform/frameworks/base.git/+/master/data/fonts/AndroidEmoji.ttf
# commit 07912f876c8639f811b06831465c14c4a3b17663
font = ImageFont.truetype('AndroidEmoji.ttf', 65 * AA_SCALE)
image = Image.new('RGBA', BIG_SIZE)
draw = ImageDraw.Draw(image)
draw.text((0, 0), char, font=font, fill='black')
image.resize(SIZE, Image.ANTIALIAS).save(
'out/unicode/{}.png'.format(code_point), 'PNG'
)
image.resize(SPRITE_SIZE, Image.ANTIALIAS).save(
'out/sprite/{}.png'.format(name), 'PNG'
)
def code_point_to_file_name_map(ttx):
# type: (str) -> Dict[int, Union[text_type, bytes]]
"""Given the NotoColorEmoji.ttx file, parse it to generate a map from
codepoint to filename (a la glyph0****.png)
"""
result = {} # type: Dict[int, Union[text_type, bytes]]
xml = ET.parse(ttx)
for elem in xml.find("*cmap_format_12"):
code_point = int(elem.attrib["code"], 16)
fname = elem.attrib["name"]
result[code_point] = fname
return result
def main():
# type: () -> None
# ttx is in the fonttools pacakge, the -z option is only on master
# https://github.com/behdad/fonttools/
# NotoColorEmoji.tff is from
# https://android.googlesource.com/platform/external/noto-fonts/+/marshmallow-release/other/NotoColorEmoji.ttf
# note that you have to run it though base64 -D before being able
# to use it, since it downloads from that page base64 encoded
# this is so we don't accidently leave ttx files from previous
# runs of this script lying around
for fname in glob.glob(EMOJI_DUMP_PATH("*ttx*")):
os.remove(fname)
# check if directory `var/emoji_dump` exists
subprocess.check_call(['mkdir', '-p', EMOJI_DUMP_DIR_PATH])
success_stamp = get_success_stamp()
source_emoji_dump = dirname(success_stamp)
if not os.path.exists(success_stamp):
print("Dumping emojis ...")
dump_emojis(source_emoji_dump)
run(['touch', success_stamp])
print("Using cached emojis from {}".format(source_emoji_dump))
run(['rm', '-rf', TARGET_EMOJI_DUMP])
run(['ln', '-nsf', source_emoji_dump, TARGET_EMOJI_DUMP])
def get_success_stamp():
# type: () -> str
sha = hashlib.sha1()
filenames = ['NotoColorEmoji.ttf', 'emoji_map.json', 'AndroidEmoji.ttf',
'build_emoji', 'emoji_dump.py']
for filename in filenames:
with open(filename, 'rb') as reader:
sha.update(reader.read())
return os.path.join(EMOJI_CACHE_PATH, sha.hexdigest(), 'gemoji', '.success-stamp')
def dump_emojis(cache_path):
# type: (str) -> None
subprocess.call('ttx -v -z extfile -d {} NotoColorEmoji.ttf'.format(EMOJI_DUMP_DIR_PATH), shell=True)
emoji_map = json.load(open('emoji_map.json'))
# Fix data problem with red/blue cars being inaccurate.
emoji_map['blue_car'] = emoji_map['red_car']
emoji_map['red_car'] = emoji_map['oncoming_automobile']
code_point_to_fname_map = code_point_to_file_name_map(EMOJI_DUMP_PATH("NotoColorEmoji.ttx"))
os.chdir(EMOJI_DUMP_DIR_PATH)
try:
shutil.rmtree('out')
except OSError:
pass
for fname in glob.glob("sprite*"):
os.remove(fname)
os.mkdir('out')
os.mkdir('out/sprite')
os.mkdir('out/unicode')
failed = False
for name, code_point in emoji_map.items():
try:
color_font(name, code_point, code_point_to_fname_map)
except MissingGlyphError:
print("Warning: Missing color glyph for %s; using black/white." % (name,))
try:
bw_font(name, code_point)
except Exception as e:
print(e)
print('Missing {}, {}'.format(name, code_point))
failed = True
continue
os.symlink(
'unicode/{}.png'.format(code_point),
'out/{}.png'.format(name)
)
if failed:
print("Errors dumping emoji!")
sys.exit(1)
subprocess.call('glue --quiet out/sprite . --namespace=emoji --sprite-namespace= --retina',
shell=True)
cache_emoji = os.path.join(cache_path, 'images', 'emoji')
run(['sudo', 'rm', '-rf', cache_path])
run(['sudo', 'mkdir', '-p', cache_emoji])
run(["sudo", "chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), cache_path])
run(['mv', 'out/*', cache_emoji], shell=True)
run(['mv', 'sprite*', cache_path], shell=True)
assets = "{}/static/assets/zulip-emoji/*".format(ZULIP_PATH)
run(['cp', '-RPp', assets, cache_emoji], shell=True)
if __name__ == "__main__":
main()

2
tools/update-prod-static
View File

@ -37,7 +37,7 @@ subprocess.check_call(['python', 'tools/minify-js']
stdout=fp, stderr=fp)
# Build emoji
subprocess.check_call(['bash', '-ex', 'tools/setup/emoji_dump/build_emoji'],
subprocess.check_call(['python', 'tools/setup/emoji_dump/build_emoji'],
stdout=fp, stderr=fp)
# Download and include zxcvbn.js