Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Reuse minified JS from previous deploys 

This is a big change affecting lots of areas:

* Pipeline no longer deals with JS (though it still minifies CSS)
* A new script, tools/minify-js (called from update-prod-static),
  minifies JavaScripts
* A command-line argument --prev-deploy, if passed to minify-js or
  update-prod-static, is used to copy minified JS from a previous
  deploy (i.e., a previous git checkout), if the source files have
  not changed
* update-deployment passes --prev-deploy
* Scripts are now included with the minified_js template tag, rather
  than Pipeline's compressed_js

Also, as a side benefit of this commit, our Handlebars templates will
no longer be copied into prod-static/ and accessible in production.

Unminification is probably broken, but, per Zev and Trac ticket #1377,
it wasn't working perfectly before this change either.

(Based on code review, this commit has been revised to:
 * Warn if git returns an error in minify-js
 * Add missing output redirects in update-prod-static
 * Use DEPLOY_ROOT instead of manually constructing that directory
 * Use old style formatting)

(imported from commit e67722ea252756db8519d5c0bd6a421d59374185)
This commit is contained in:
Scott Feeney 2013-07-03 16:42:25 -04:00
parent 88d070e182
commit 2c33320746
16 changed files with 232 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
humbug/settings.py
View File

@ -13,7 +13,8 @@ DEPLOYED = (('humbughq.com' in platform.node())
STAGING_DEPLOYED = (platform.node() == 'staging.humbughq.com') STAGING_DEPLOYED = (platform.node() == 'staging.humbughq.com')
TESTING_DEPLOYED = not not re.match(r'^test', platform.node()) TESTING_DEPLOYED = not not re.match(r'^test', platform.node())
DEBUG = not DEPLOYED # Uncomment end of next line to test JS/CSS minification.
DEBUG = not DEPLOYED # and platform.node() != 'your-machine'
TEMPLATE_DEBUG = DEBUG TEMPLATE_DEBUG = DEBUG
TEST_SUITE = False TEST_SUITE = False
@ -293,7 +294,7 @@ PIPELINE_CSS = {
}, },
} }
PIPELINE_JS = { JS_SPECS = {
'common': { 'common': {
'source_filenames': ( 'source_filenames': (
'third/jquery/jquery-1.7.2.js', 'third/jquery/jquery-1.7.2.js',
@ -389,23 +390,18 @@ PIPELINE_JS = {
}, },
} }
if PIPELINE: if not DEBUG:
# This file is generated by update-prod-static. # This file is generated by update-prod-static.
# In dev we fetch individual templates using Ajax. # In dev we fetch individual templates using Ajax.
PIPELINE_JS['app']['source_filenames'].append('templates/compiled.js') JS_SPECS['app']['source_filenames'].append('templates/compiled.js')
PIPELINE_JS = {} # Now handled in tools/minify-js
PIPELINE_JS_COMPRESSOR = None
PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor' PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'
PIPELINE_YUI_BINARY = '/usr/bin/env yui-compressor' PIPELINE_YUI_BINARY = '/usr/bin/env yui-compressor'
PIPELINE_JS_COMPRESSOR = 'zephyr.lib.minify.ClosureSourceMapCompressor'
PIPELINE_CLOSURE_BINARY = os.path.join(DEPLOY_ROOT, 'tools/closure-compiler/run')
PIPELINE_CLOSURE_SOURCE_MAP_DIR = 'prod-static/source-map'
# Disable stuffing the entire JavaScript codebase inside an anonymous function.
# We need modules to be externally visible, so that methods can be called from
# event handlers defined in HTML.
PIPELINE_DISABLE_WRAPPER = True
USING_RABBITMQ = DEPLOYED USING_RABBITMQ = DEPLOYED
# This password also appears in servers/configure-rabbitmq # This password also appears in servers/configure-rabbitmq

3
templates/zephyr/activity.html
View File

@ -1,12 +1,13 @@
{% extends "zephyr/base.html" %} {% extends "zephyr/base.html" %}
{% load compressed %} {% load compressed %}
{% load minified_js %}
{# User Activity. #} {# User Activity. #}
{% block customhead %} {% block customhead %}
{{ block.super }} {{ block.super }}
{% compressed_js 'activity' %} {% minified_js 'activity' %}
{% compressed_css 'activity' %} {% compressed_css 'activity' %}
{% endblock %} {% endblock %}

4
templates/zephyr/api_endpoints.html
View File

@ -2,11 +2,11 @@
{# API information page #} {# API information page #}
{% load compressed %} {% load minified_js %}
{% block customhead %} {% block customhead %}
{{ block.super }} {{ block.super }}
{% compressed_js 'api' %} {% minified_js 'api' %}
{% endblock %} {% endblock %}

3
templates/zephyr/base.html
View File

@ -3,6 +3,7 @@
{# Base template for the whole site. #} {# Base template for the whole site. #}
{% load compressed %} {% load compressed %}
{% load minified_js %}
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
@ -40,7 +41,7 @@ mixpanel.init("{{ mixpanel_token }}", {track_pageview: {{ enable_metrics }}});
<script type="text/javascript"> <script type="text/javascript">
page_params.enable_metrics = {{ enable_metrics }}; page_params.enable_metrics = {{ enable_metrics }};
</script> </script>
{% compressed_js 'common' %} {% minified_js 'common' %}
{% block customhead %} {% block customhead %}
{% endblock %} {% endblock %}
</head> </head>

5
templates/zephyr/index.html
View File

@ -4,6 +4,7 @@
{# Includes some other templates as tabs. #} {# Includes some other templates as tabs. #}
{% load compressed %} {% load compressed %}
{% load minified_js %}
{% block page_params %} {% block page_params %}
{# Insert parameters, which have been encoded with JSONEncoderForHTML. #} {# Insert parameters, which have been encoded with JSONEncoderForHTML. #}
@ -23,10 +24,10 @@ var page_params = {{ page_params }};
{% else %} {% else %}
{% compressed_css 'app' %} {% compressed_css 'app' %}
{% endif %} {% endif %}
{% compressed_js 'app' %} {% minified_js 'app' %}
{% if debug %} {% if debug %}
{% compressed_js 'app_debug' %} {% minified_js 'app_debug' %}
{% endif %} {% endif %}
{% endblock %} {% endblock %}

4
templates/zephyr/portico_signup.html
View File

@ -1,10 +1,10 @@
{% extends "zephyr/portico.html" %} {% extends "zephyr/portico.html" %}
{% load compressed %} {% load minified_js %}
{# Portico page with signup code #} {# Portico page with signup code #}
{% block customhead %} {% block customhead %}
{{ block.super }} {{ block.super }}
{% compressed_js 'signup' %} {% minified_js 'signup' %}
{% endblock %} {% endblock %}

3
templates/zephyr/signup.html
View File

@ -1,9 +1,10 @@
{% extends "zephyr/portico.html" %} {% extends "zephyr/portico.html" %}
{% load compressed %} {% load compressed %}
{% load minified_js %}
{% block customhead %} {% block customhead %}
{% compressed_css 'portico' %} {% compressed_css 'portico' %}
{% compressed_js 'landing-page' %} {% minified_js 'landing-page' %}
{% endblock %} {% endblock %}
{% block inner_content %} {% block inner_content %}

117
tools/minify-js Executable file
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python
# Minifies JavaScripts, creating source maps
from __future__ import absolute_import
import os
from glob import glob
import subprocess
import optparse
import sys
parser = optparse.OptionParser()
parser.add_option('--prev-deploy', nargs=1, metavar='DIR',
help='A previous deploy from which to reuse files if possible')
(options, args) = parser.parse_args()
prev_deploy = options.prev_deploy
# We have to pull out JS_SPECS, defined in our settings file, so we know what
# JavaScript source files to minify (and what output files to create).
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'humbug.settings'
from django.conf import settings
os.chdir(settings.DEPLOY_ROOT)
STATIC_PATH = 'zephyr/static/'
with open(settings.STATIC_HEADER_FILE, 'r') as fp:
static_header = fp.read()
# Compile Handlebars templates
subprocess.check_call(['tools/node', 'node_modules/.bin/handlebars']
+ glob(os.path.join(STATIC_PATH, 'templates/*.handlebars'))
+ ['--output', os.path.join(STATIC_PATH, 'templates/compiled.js'),
'--known', 'if,unless,each,with'])
def get_changed_source_files(other_checkout):
""" Get list of changed static files since other_checkout.
If git fails to return a reasonable looking list, this returns None,
in which case it should be assumed no files can be reused from
other_checkout. """
try:
git_dir = os.path.join(other_checkout, '.git')
old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
env={'GIT_DIR': git_dir})
old_commit_sha1 = old_commit_sha1.rstrip()
git_diff = subprocess.check_output(['git', 'diff', '--name-only',
old_commit_sha1])
except subprocess.CalledProcessError:
# If git returned an error, assume we can't reuse any files, and
# regenerate everything.
print "Warning: git returned an error when comparing to the previous"
print ("deploy in %s. Will re-minify JavaScript instead of reusing"
% other_checkout)
return None
changed = set()
for filename in git_diff.split('\n'):
if not filename.startswith(STATIC_PATH):
continue # Ignore non-static files.
if filename.endswith('.handlebars'):
# Since Handlebars templates are compiled, treat this as if the
# compiled .js file changed (which it did).
changed.add('templates/compiled.js')
continue
changed.add(filename)
return changed
changed_files = set()
if prev_deploy:
changed_files = get_changed_source_files(prev_deploy)
if changed_files is None:
prev_deploy = None
JS_SPECS = settings.JS_SPECS
CLOSURE_BINARY = 'tools/closure-compiler/run'
# Where to put minified JS and source maps
MIN_DIR = os.path.join(STATIC_PATH, 'min/')
MAP_DIR = os.path.join(STATIC_PATH, 'source-map/')
subprocess.check_call(['mkdir', '-p', MIN_DIR, MAP_DIR])
for js_group, filespec in JS_SPECS.iteritems():
in_files = [os.path.join(STATIC_PATH, filename)
for filename in filespec['source_filenames']]
out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename']))
map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename'])
+ '.map')
if (prev_deploy and len(set(in_files) & changed_files) == 0):
# Try to reuse the output file from previous deploy
try:
for dest in [out_file, map_file]:
src = os.path.join(prev_deploy, dest)
os.path.getsize(src) # Just to throw error if it doesn't exist.
if os.path.abspath(src) != os.path.abspath(dest):
subprocess.check_call(['cp', src, dest])
continue # Copy succeeded, so go on to next file.
except (subprocess.CalledProcessError, OSError):
pass # Copy failed, so fall through to minification instead.
# No previous deploy, or a source file has changed, or copying was
# supposed to work but failed. Thus, minify the JS anew.
cmd = '%s --language_in ECMASCRIPT5 --create_source_map %s %s' % (
CLOSURE_BINARY, map_file, ' '.join(in_files))
js = subprocess.check_output(cmd, shell=True)
# Write out the JS with static header prepended
with open(out_file, 'w') as fp:
fp.write(static_header)
fp.write(js)

3
tools/update-deployment
View File

@ -52,7 +52,8 @@ subprocess.check_call(["find", ".", "-name", "*.pyc", "-delete"], stdout=open('/
# Update static files # Update static files
logging.info("Updating static files") logging.info("Updating static files")
subprocess.check_call(["./tools/update-prod-static"]) subprocess.check_call(["./tools/update-prod-static", "--prev-deploy",
os.path.join(DEPLOYMENTS_DIR, 'current')])
logging.info("Restarting server...") logging.info("Restarting server...")
subprocess.check_call(["./tools/restart-server"]) subprocess.check_call(["./tools/restart-server"])

54
tools/update-prod-static
View File

@ -1,25 +1,47 @@
#!/usr/bin/env python #!/usr/bin/env python
# Update static files in production. # Updates static files for production.
from os import chdir, path, open, close, O_WRONLY, O_CREAT from __future__ import absolute_import
from glob import glob
import os
import subprocess import subprocess
import optparse
import sys
chdir(path.join(path.dirname(__file__), '..')) # We need settings so we can figure out where the prod-static directory is.
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'humbug.settings'
from django.conf import settings
# Redirect child processes' output to a log file (most recent run only) parser = optparse.OptionParser()
close(1) parser.add_option('--prev-deploy', nargs=1, metavar='DIR',
fp = open("update-prod-static.log", O_WRONLY|O_CREAT) # Will open on 1, stdout. help='A previous deploy from which to reuse files if possible')
(options, args) = parser.parse_args()
prev_deploy = options.prev_deploy
# Compile Handlebars templates os.chdir(settings.DEPLOY_ROOT)
subprocess.check_call(['tools/node', 'node_modules/.bin/handlebars']
+ glob('zephyr/static/templates/*.handlebars')
+ ['--output', 'zephyr/static/templates/compiled.js',
'--known', 'if,unless,each,with'])
# Collect the files that we're going to serve # Redirect child processes' output to a log file (most recent run only).
subprocess.check_call(['rm', '-r', 'prod-static/source-map']) fp = open('update-prod-static.log', 'w')
subprocess.check_call(['mkdir', '-p', 'prod-static/source-map'])
subprocess.check_call(['python', './manage.py', 'collectstatic', '--noinput'])
# Compile Handlebars templates and minify JavaScripts.
subprocess.check_call(['python', 'tools/minify-js']
+ (['--prev-deploy', prev_deploy] if prev_deploy else []),
stdout=fp, stderr=fp)
# Collect the files that we're going to serve.
subprocess.check_call(['python', './manage.py', 'collectstatic', '--noinput'],
stdout=fp, stderr=fp)
# Move the source maps out of the serve/ directory and into their
# proper place.
subprocess.check_call(['rm', '-rf', 'prod-static/source-map'],
stdout=fp, stderr=fp)
subprocess.check_call(['mkdir', '-p', 'prod-static'], # Needed if DEPLOYED
stdout=fp, stderr=fp)
subprocess.check_call(['mv', os.path.join(settings.STATIC_ROOT, 'source-map'),
'prod-static/source-map'],
stdout=fp, stderr=fp)
fp.close()

23
zephyr/finders.py
View File

@ -1,19 +1,24 @@
from django.conf import settings import re
from django.contrib.staticfiles.finders import AppDirectoriesFinder from django.contrib.staticfiles.finders import AppDirectoriesFinder
class ExcludeMinifiedMixin(object): class ExcludeUnminifiedMixin(object):
""" Excludes unminified copies of our JavaScript code, templates
and stylesheets, so that these sources don't end up getting served
in production. """
def list(self, ignore_patterns): def list(self, ignore_patterns):
# We can't use ignore_patterns because the patterns are # We can't use ignore_patterns because the patterns are
# applied to just the file part, not the entire path # applied to just the file part, not the entire path
to_exclude = set() excluded = '^(js|styles|templates)/'
for collection in (settings.PIPELINE_CSS, settings.PIPELINE_JS):
for key in collection:
to_exclude.update(collection[key]['source_filenames'])
super_class = super(ExcludeMinifiedMixin, self) # source-map/ should also not be included.
# However, we work around that by moving it later,
# in tools/update-prod-static.
super_class = super(ExcludeUnminifiedMixin, self)
for path, storage in super_class.list(ignore_patterns): for path, storage in super_class.list(ignore_patterns):
if not path in to_exclude: if not re.search(excluded, path):
yield path, storage yield path, storage
class HumbugFinder(ExcludeMinifiedMixin, AppDirectoriesFinder): class HumbugFinder(ExcludeUnminifiedMixin, AppDirectoriesFinder):
pass pass

2
zephyr/lib/unminify.py
View File

@ -19,7 +19,7 @@ class LineToFile(object):
self._cumulative_counts = [] self._cumulative_counts = []
total = 0 total = 0
for filename in settings.PIPELINE_JS['app']['source_filenames']: for filename in settings.JS_SPECS['app']['source_filenames']:
self._names.append(filename) self._names.append(filename)
self._cumulative_counts.append(total) self._cumulative_counts.append(total)
with open(path.join('zephyr/static', filename), 'r') as fil: with open(path.join('zephyr/static', filename), 'r') as fil:

2
zephyr/static/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
min/
source-map/

9
zephyr/storage.py
View File

@ -20,14 +20,7 @@ class AddHeaderMixin(object):
for name in paths: for name in paths:
storage, path = paths[name] storage, path = paths[name]
# Find the top-level directory for the file if not path.startswith('min/') or not path.endswith('.css'):
head, _ = os.path.split(path)
top_dir = head
while head != '':
top_dir = head
head, _ = os.path.split(head)
if top_dir != 'min':
ret_dict[path] = (path, path, False) ret_dict[path] = (path, path, False)
continue continue

0
zephyr/templatetags/__init__.py Normal file
View File

35
zephyr/templatetags/minified_js.py Normal file
View File

@ -0,0 +1,35 @@
from django.template import Node, Library, TemplateSyntaxError
from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
register = Library()
class MinifiedJSNode(Node):
def __init__(self, sourcefile):
self.sourcefile = sourcefile
def render(self, context):
if settings.DEBUG:
scripts = settings.JS_SPECS[self.sourcefile]['source_filenames']
else:
scripts = [settings.JS_SPECS[self.sourcefile]['output_filename']]
script_urls = [staticfiles_storage.url(script) for script in scripts]
script_tags = ['<script type="text/javascript" src="%s" charset="utf-8"></script>'
% url for url in script_urls]
return '\n'.join(script_tags)
@register.tag
def minified_js(parser, token):
try:
tag_name, sourcefile = token.split_contents()
except ValueError:
raise TemplateSyntaxError("%s tag requires an argument" % tag_name)
if not (sourcefile[0] == sourcefile[-1] and sourcefile[0] in ('"', "'")):
raise TemplateSyntaxError("%s tag should be quoted" % tag_name)
sourcefile = sourcefile[1:-1]
if sourcefile not in settings.JS_SPECS:
raise TemplateSyntaxError("%s tag invalid argument: no JS file %s"
% (tag_name, sourcefile))
return MinifiedJSNode(sourcefile)