zulip_bots: Generate static files during provisioning.

This commit implements support for copying over static files
for all bots in the zulip_bots package to
static/generated/bots/ during provisioning. This directory
isn't tracked by Git. This allows us to have access to files
stored in an arbitrary zulip_bots package directory somewhere
on the system. For now, logo.* and doc.md files are copied over.

This commit should act as a starting point for extending our
macro-based Markdown framework to our bots/API packages'
documentation and eventually rendering these static files
alongside our webhooks' documentation.

docs/dev-setup-non-vagrant.md

@@ -324,6 +324,7 @@ sudo chown -R `whoami`:`whoami` /srv/zulip-emoji-cache
 ./tools/setup/emoji/build_emoji
 ./tools/inline-email-css
 ./tools/setup/build_pygments_data.py
+./tools/setup/generate_zulip_bots_static_files
 ./scripts/setup/generate_secrets.py --development
 if [ $(uname) = "OpenBSD" ]; then
     sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/

static/.gitignore

@@ -11,3 +11,6 @@
 /locale/language_options.json
 /locale/language_name_map.json
 /third/emoji-data
+
+# Static files from the zulip_bots package
+/generated/bots/

tools/lib/provision.py

@@ -277,6 +277,9 @@ def main(options):
     run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
     run(["tools/setup/emoji/build_emoji"])
 
+    # copy over static files from the zulip_bots package
+    run(["tools/setup/generate_zulip_bots_static_files"])
+
     run(["tools/setup/build_pygments_data.py"])
     run(["scripts/setup/generate_secrets.py", "--development"])
     run(["tools/update-authors-json", "--use-fixture"])

tools/setup/generate_zulip_bots_static_files

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+import sys
+import shutil
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+
+from typing import List, Text
+from zproject import settings
+from zulip_bots.lib import get_bots_directory_path
+
+
+bots_dir = os.path.join(settings.STATIC_ROOT, 'generated/bots')
+if not os.path.isdir(bots_dir):
+    os.makedirs(bots_dir)
+
+def copyfiles(paths):
+    # type: (List[Text]) -> None
+    for src_path in paths:
+        bot_name = os.path.basename(os.path.dirname(src_path))
+
+        bot_dir = os.path.join(bots_dir, bot_name)
+        if not os.path.isdir(bot_dir):
+            os.mkdir(bot_dir)
+
+        dst_path = os.path.join(bot_dir, os.path.basename(src_path))
+        if not os.path.isfile(dst_path):
+            shutil.copyfile(src_path, dst_path)
+
+package_bots_dir = get_bots_directory_path()
+
+logo_glob_pattern = os.path.join(package_bots_dir, '*/logo.*')
+logos = glob.glob(logo_glob_pattern)
+copyfiles(logos)
+
+doc_glob_pattern = os.path.join(package_bots_dir, '*/doc.md')
+docs = glob.glob(doc_glob_pattern)
+copyfiles(docs)

tools/update-prod-static

@@ -45,6 +45,10 @@ subprocess.check_call(['./tools/setup/emoji/build_emoji'],
 subprocess.check_call(['./tools/inline-email-css'],
                       stdout=fp, stderr=fp)
 
+# Copy over static files from the zulip_bots package
+subprocess.check_call(['./tools/setup/generate_zulip_bots_static_files'],
+                      stdout=fp, stderr=fp)
+
 # Build pygment data
 subprocess.check_call(['./tools/setup/build_pygments_data.py'],
                       stdout=fp, stderr=fp)

version.py

@@ -1,2 +1,2 @@
 ZULIP_VERSION = "1.6.0+git"
-PROVISION_VERSION = '9.12'
+PROVISION_VERSION = '9.13'

zerver/tests/test_templates.py

@@ -122,10 +122,16 @@ class TemplateTestCase(ZulipTestCase):
 
         integrations_regexp = re.compile('zerver/integrations/.*.html')
 
+        # Since static/generated/bots/ is searched by Jinja2 for templates,
+        # it mistakes logo files under that directory for templates.
+        bot_logos_regexp = re.compile('\w+\/logo\.(svg|png)$')
+
         skip = covered + defer + logged_out + logged_in + unusual + ['tests/test_markdown.html',
                                                                      'zerver/terms.html',
                                                                      'zerver/privacy.html']
-        templates = [t for t in get_all_templates() if not (t in skip or integrations_regexp.match(t))]
+
+        templates = [t for t in get_all_templates() if not (
+            t in skip or integrations_regexp.match(t) or bot_logos_regexp.match(t))]
         self.render_templates(templates, self.get_context())
 
         # Test the deferred templates with updated context.

zproject/settings.py

@@ -965,8 +965,12 @@ default_template_engine_settings = deepcopy(base_template_engine_settings)
 default_template_engine_settings.update({
     'NAME': 'Jinja2',
     'DIRS': [
+        # The main templates directory
         os.path.join(DEPLOY_ROOT, 'templates'),
+        # The webhook integration templates
         os.path.join(DEPLOY_ROOT, 'zerver', 'webhooks'),
+        # The python-zulip-api:zulip_bots package templates
+        os.path.join(STATIC_ROOT, 'generated', 'bots'),
     ],
     'APP_DIRS': True,
 })
@@ -989,7 +993,6 @@ TEMPLATES = [
     default_template_engine_settings,
     non_html_template_engine_settings,
 ]
-
 ########################################################################
 # LOGGING SETTINGS
 ########################################################################