[manual] Minify JavaScript and CSS in production

Manual deployment steps: The same Nginx reload as for "Get rid of the
static-access-control mechanism".  If deploying both commits at once,
just do it once.

(imported from commit dd8dbbf14b95fce0a4b6f66f462fa0a6b50bfb8c)

.gitignore

@@ -4,4 +4,6 @@
 /all_messages_log.*
 /event_log/*
 /server.log
+/update-prod-static.log
+/prod-static
 *.sw[po]

humbug/settings.py

@@ -172,12 +172,71 @@ INSTALLED_APPS = (
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.sites',
+    'django.contrib.staticfiles',
     'south',
     'jstemplate',
     'confirmation',
+    'pipeline',
     'zephyr',
 )
 
+
+# Static files and minification
+
+STATIC_URL = '/static/'
+
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+)
+
+STATICFILES_STORAGE = 'pipeline.storage.PipelineStorage'
+
+STATIC_ROOT = 'prod-static/collected'
+
+PIPELINE_CSS = {
+    'app': {
+        'source_filenames': (
+            'styles/zephyr.css',
+            'styles/pygments.css',
+        ),
+        'output_filename': 'min/app.css'
+    },
+}
+
+PIPELINE_JS = {
+    'app': {
+        'source_filenames': (
+            'js/util.js',
+            'js/setup.js',
+            'js/rows.js',
+            'js/narrow.js',
+            'js/reload.js',
+            'js/compose.js',
+            'js/subs.js',
+            'js/ui.js',
+            'js/typeahead_helper.js',
+            'js/search.js',
+            'js/composebox_typeahead.js',
+            'js/hotkey.js',
+            'js/notifications.js',
+            'js/hashchange.js',
+            'js/invite.js',
+            'js/zephyr.js',
+        ),
+        'output_filename': 'min/app.js'
+    },
+}
+
+PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'
+PIPELINE_JS_COMPRESSOR  = 'pipeline.compressors.yui.YUICompressor'
+PIPELINE_YUI_BINARY     = '/usr/bin/env yui-compressor'
+
+# Disable stuffing the entire JavaScript codebase inside an anonymous function.
+# We need modules to be externally visible, so that methods can be called from
+# event handlers defined in HTML.
+PIPELINE_DISABLE_WRAPPER = True
+
+
 USING_RABBITMQ = DEPLOYED
 # This password also appears in servers/configure-rabbitmq
 RABBITMQ_PASSWORD = 'xxxxxxxxxxxxxxxx'

humbug/urls.py

@@ -85,7 +85,4 @@ urlpatterns = patterns('',
     url(r'^notify_pointer_update$',         'zephyr.tornadoviews.notify_pointer_update'),
 )
 
-if not settings.DEPLOYED:
-    urlpatterns += patterns('',
-        url(r'^static/(?P<path>.*)$', 'django.views.static.serve',
-            {'document_root': os.path.join(settings.SITE_ROOT, '../zephyr/static')}))
+# Static file serving in dev is now handled by django.contrib.staticfiles

servers/puppet/files/nginx/humbug-include/app

@@ -9,7 +9,7 @@ error_page 502 503 504 /static/html/5xx.html;
 
 # Serve static files directly
 location /static/ {
-    alias /home/humbug/humbug/zephyr/static/;
+    alias /home/humbug/humbug/prod-static/serve/;
     error_page 404 /static/html/404.html;
 }
 

templates/zephyr/index.html

@@ -4,6 +4,7 @@
 {# Includes some other templates as tabs. #}
 
 {% load jstemplate %}
+{% load compressed %}
 
 {% block customhead %}
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -29,8 +30,9 @@
 {% rawjstemplate "timeinfo_popover_content" %}
 </script>
 
-<link href="/static/styles/zephyr.css?dummy_time={% now "U" %}" rel="stylesheet">
-<link href="/static/styles/pygments.css" rel="stylesheet">
+{% compressed_css 'app' %}
+<link rel="stylesheet" href="/static/third/spectrum/spectrum.css" />
+
 <script type="text/javascript" src="/static/third/jquery/jquery.form.js"></script>
 <script type="text/javascript" src="/static/third/jquery/jquery.highlight.js"></script>
 <script type="text/javascript" src="/static/third/xdate/xdate.js"></script>
@@ -40,24 +42,8 @@
 <script type="text/javascript" src="/static/third/jquery-throttle-debounce/jquery.ba-throttle-debounce.min.js"></script>
 <script type="text/javascript" src="/static/third/jquery.idle/jquery.idle.js"></script>
 <script type="text/javascript" src="/static/third/jquery-autosize/jquery.autosize.js"></script>
-<script type="text/javascript" src="/static/js/util.js"></script>
 <script type="text/javascript" src="/static/third/spectrum/spectrum.js"></script>
-<link rel="stylesheet" href="/static/third/spectrum/spectrum.css" />
-<script type="text/javascript" src="/static/js/setup.js"></script>
-<script type="text/javascript" src="/static/js/rows.js"></script>
-<script type="text/javascript" src="/static/js/narrow.js"></script>
-<script type="text/javascript" src="/static/js/reload.js"></script>
-<script type="text/javascript" src="/static/js/compose.js"></script>
-<script type="text/javascript" src="/static/js/subs.js"></script>
-<script type="text/javascript" src="/static/js/ui.js"></script>
-<script type="text/javascript" src="/static/js/typeahead_helper.js"></script>
-<script type="text/javascript" src="/static/js/search.js"></script>
-<script type="text/javascript" src="/static/js/composebox_typeahead.js"></script>
-<script type="text/javascript" src="/static/js/hotkey.js"></script>
-<script type="text/javascript" src="/static/js/notifications.js"></script>
-<script type="text/javascript" src="/static/js/hashchange.js"></script>
-<script type="text/javascript" src="/static/js/invite.js"></script>
-<script type="text/javascript" src="/static/js/zephyr.js"></script>
+{% compressed_js 'app' %}
 
 {% if debug %}
 <script type="text/javascript" src="/static/js/debug.js"></script>

tools/restart-server

@@ -27,6 +27,9 @@ except:
 # Delete all .pyc files to avoid old module files hanging around
 subprocess.check_call(["find", ".", "-name", "*.pyc", "-delete"], stdout=open('/dev/null', 'w'))
 
+# Update static files
+subprocess.check_call(["./tools/update-prod-static"])
+
 # Restart the FastCGI process, which is running in a shell loop in screen.
 # TODO: real daemonization
 for cmd in ('runfcgi', 'runtornado', "process_user_activity"):

tools/update-prod-static

@@ -0,0 +1,28 @@
+#!/bin/bash -e
+
+# Update static files in production.
+
+cd "$(dirname "$0")"/..
+
+# Redirect output to a log file (most recent run only)
+exec >update-prod-static.log
+
+# Collect both original and minified files
+rm -rf prod-static/collected
+./manage.py collectstatic --noinput
+
+# Copy the files we want to expose
+rm -rf prod-static/serve-new
+mkdir -p prod-static/serve-new/{js,styles}
+cp -r prod-static/collected/{favicon.ico,robots.txt,html,third,min} prod-static/serve-new/
+cp prod-static/collected/js/{common,debug,signup}.js prod-static/serve-new/js/
+cp prod-static/collected/styles/{activity,portico}.css prod-static/serve-new/styles/
+
+# Sync the new directory to the one nginx actually serves.
+# Hopefully this doesn't race too badly for clients loading
+# right as a deploy happens.
+mkdir -p prod-static/serve
+rsync -ar --delete prod-static/serve-new/* prod-static/serve/
+
+# Clean up
+rm -rf prod-static/{collected,serve-new}