From 390a1fec928a2f4fb1944b036a6d9187f8969e9d Mon Sep 17 00:00:00 2001 From: Eeshan Garg Date: Sat, 16 Sep 2017 21:44:56 -0230 Subject: [PATCH] zulip_bots: Generate static files during provisioning. This commit implements support for copying over static files for all bots in the zulip_bots package to static/generated/bots/ during provisioning. This directory isn't tracked by Git. This allows us to have access to files stored in an arbitrary zulip_bots package directory somewhere on the system. For now, logo.* and doc.md files are copied over. This commit should act as a starting point for extending our macro-based Markdown framework to our bots/API packages' documentation and eventually rendering these static files alongside our webhooks' documentation. --- docs/dev-setup-non-vagrant.md | 1 + static/.gitignore | 3 ++ tools/lib/provision.py | 3 ++ tools/setup/generate_zulip_bots_static_files | 42 ++++++++++++++++++++ tools/update-prod-static | 4 ++ version.py | 2 +- zerver/tests/test_templates.py | 8 +++- zproject/settings.py | 5 ++- 8 files changed, 65 insertions(+), 3 deletions(-) create mode 100755 tools/setup/generate_zulip_bots_static_files diff --git a/docs/dev-setup-non-vagrant.md b/docs/dev-setup-non-vagrant.md index e528ffc926..4652ac177b 100644 --- a/docs/dev-setup-non-vagrant.md +++ b/docs/dev-setup-non-vagrant.md @@ -324,6 +324,7 @@ sudo chown -R `whoami`:`whoami` /srv/zulip-emoji-cache ./tools/setup/emoji/build_emoji ./tools/inline-email-css ./tools/setup/build_pygments_data.py +./tools/setup/generate_zulip_bots_static_files ./scripts/setup/generate_secrets.py --development if [ $(uname) = "OpenBSD" ]; then sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/ diff --git a/static/.gitignore b/static/.gitignore index d5a4e2636d..0e7bbabed5 100644 --- a/static/.gitignore +++ b/static/.gitignore @@ -11,3 +11,6 @@ /locale/language_options.json /locale/language_name_map.json /third/emoji-data + +# Static files from the zulip_bots package +/generated/bots/ diff --git a/tools/lib/provision.py b/tools/lib/provision.py index bf977cf1b6..ac615017df 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -277,6 +277,9 @@ def main(options): run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH]) run(["tools/setup/emoji/build_emoji"]) + # copy over static files from the zulip_bots package + run(["tools/setup/generate_zulip_bots_static_files"]) + run(["tools/setup/build_pygments_data.py"]) run(["scripts/setup/generate_secrets.py", "--development"]) run(["tools/update-authors-json", "--use-fixture"]) diff --git a/tools/setup/generate_zulip_bots_static_files b/tools/setup/generate_zulip_bots_static_files new file mode 100755 index 0000000000..5c3a2b2624 --- /dev/null +++ b/tools/setup/generate_zulip_bots_static_files @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import glob +import os +import sys +import shutil + +ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if ZULIP_PATH not in sys.path: + sys.path.append(ZULIP_PATH) + +from typing import List, Text +from zproject import settings +from zulip_bots.lib import get_bots_directory_path + + +bots_dir = os.path.join(settings.STATIC_ROOT, 'generated/bots') +if not os.path.isdir(bots_dir): + os.makedirs(bots_dir) + +def copyfiles(paths): + # type: (List[Text]) -> None + for src_path in paths: + bot_name = os.path.basename(os.path.dirname(src_path)) + + bot_dir = os.path.join(bots_dir, bot_name) + if not os.path.isdir(bot_dir): + os.mkdir(bot_dir) + + dst_path = os.path.join(bot_dir, os.path.basename(src_path)) + if not os.path.isfile(dst_path): + shutil.copyfile(src_path, dst_path) + +package_bots_dir = get_bots_directory_path() + +logo_glob_pattern = os.path.join(package_bots_dir, '*/logo.*') +logos = glob.glob(logo_glob_pattern) +copyfiles(logos) + +doc_glob_pattern = os.path.join(package_bots_dir, '*/doc.md') +docs = glob.glob(doc_glob_pattern) +copyfiles(docs) diff --git a/tools/update-prod-static b/tools/update-prod-static index 43e0c12c7e..2c686be308 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -45,6 +45,10 @@ subprocess.check_call(['./tools/setup/emoji/build_emoji'], subprocess.check_call(['./tools/inline-email-css'], stdout=fp, stderr=fp) +# Copy over static files from the zulip_bots package +subprocess.check_call(['./tools/setup/generate_zulip_bots_static_files'], + stdout=fp, stderr=fp) + # Build pygment data subprocess.check_call(['./tools/setup/build_pygments_data.py'], stdout=fp, stderr=fp) diff --git a/version.py b/version.py index 66d1c8d808..ef49b677e0 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.6.0+git" -PROVISION_VERSION = '9.12' +PROVISION_VERSION = '9.13' diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index d67117d420..06ed6dab1f 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -122,10 +122,16 @@ class TemplateTestCase(ZulipTestCase): integrations_regexp = re.compile('zerver/integrations/.*.html') + # Since static/generated/bots/ is searched by Jinja2 for templates, + # it mistakes logo files under that directory for templates. + bot_logos_regexp = re.compile('\w+\/logo\.(svg|png)$') + skip = covered + defer + logged_out + logged_in + unusual + ['tests/test_markdown.html', 'zerver/terms.html', 'zerver/privacy.html'] - templates = [t for t in get_all_templates() if not (t in skip or integrations_regexp.match(t))] + + templates = [t for t in get_all_templates() if not ( + t in skip or integrations_regexp.match(t) or bot_logos_regexp.match(t))] self.render_templates(templates, self.get_context()) # Test the deferred templates with updated context. diff --git a/zproject/settings.py b/zproject/settings.py index c46a7e146b..ad24e8b002 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -965,8 +965,12 @@ default_template_engine_settings = deepcopy(base_template_engine_settings) default_template_engine_settings.update({ 'NAME': 'Jinja2', 'DIRS': [ + # The main templates directory os.path.join(DEPLOY_ROOT, 'templates'), + # The webhook integration templates os.path.join(DEPLOY_ROOT, 'zerver', 'webhooks'), + # The python-zulip-api:zulip_bots package templates + os.path.join(STATIC_ROOT, 'generated', 'bots'), ], 'APP_DIRS': True, }) @@ -989,7 +993,6 @@ TEMPLATES = [ default_template_engine_settings, non_html_template_engine_settings, ] - ######################################################################## # LOGGING SETTINGS ########################################################################