mirror of https://github.com/zulip/zulip.git
Reuse minified JS from previous deploys
This is a big change affecting lots of areas: * Pipeline no longer deals with JS (though it still minifies CSS) * A new script, tools/minify-js (called from update-prod-static), minifies JavaScripts * A command-line argument --prev-deploy, if passed to minify-js or update-prod-static, is used to copy minified JS from a previous deploy (i.e., a previous git checkout), if the source files have not changed * update-deployment passes --prev-deploy * Scripts are now included with the minified_js template tag, rather than Pipeline's compressed_js Also, as a side benefit of this commit, our Handlebars templates will no longer be copied into prod-static/ and accessible in production. Unminification is probably broken, but, per Zev and Trac ticket #1377, it wasn't working perfectly before this change either. (Based on code review, this commit has been revised to: * Warn if git returns an error in minify-js * Add missing output redirects in update-prod-static * Use DEPLOY_ROOT instead of manually constructing that directory * Use old style formatting) (imported from commit e67722ea252756db8519d5c0bd6a421d59374185)
This commit is contained in:
Scott Feeney
parent
88d070e182
commit
2c33320746
|
|
@ -13,7 +13,8 @@ DEPLOYED = (('humbughq.com' in platform.node())
|
|||
STAGING_DEPLOYED = (platform.node() == 'staging.humbughq.com')
|
||||
TESTING_DEPLOYED = not not re.match(r'^test', platform.node())
|
||||
|
||||
DEBUG = not DEPLOYED
|
||||
# Uncomment end of next line to test JS/CSS minification.
|
||||
DEBUG = not DEPLOYED # and platform.node() != 'your-machine'
|
||||
TEMPLATE_DEBUG = DEBUG
|
||||
TEST_SUITE = False
|
||||
|
||||
|
|
@ -293,7 +294,7 @@ PIPELINE_CSS = {
|
|||
},
|
||||
}
|
||||
|
||||
PIPELINE_JS = {
|
||||
JS_SPECS = {
|
||||
'common': {
|
||||
'source_filenames': (
|
||||
'third/jquery/jquery-1.7.2.js',
|
||||
|
|
@ -389,23 +390,18 @@ PIPELINE_JS = {
|
|||
},
|
||||
}
|
||||
|
||||
if PIPELINE:
|
||||
if not DEBUG:
|
||||
# This file is generated by update-prod-static.
|
||||
# In dev we fetch individual templates using Ajax.
|
||||
PIPELINE_JS['app']['source_filenames'].append('templates/compiled.js')
|
||||
JS_SPECS['app']['source_filenames'].append('templates/compiled.js')
|
||||
|
||||
|
||||
PIPELINE_JS = {} # Now handled in tools/minify-js
|
||||
PIPELINE_JS_COMPRESSOR = None
|
||||
|
||||
PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'
|
||||
PIPELINE_YUI_BINARY = '/usr/bin/env yui-compressor'
|
||||
|
||||
PIPELINE_JS_COMPRESSOR = 'zephyr.lib.minify.ClosureSourceMapCompressor'
|
||||
PIPELINE_CLOSURE_BINARY = os.path.join(DEPLOY_ROOT, 'tools/closure-compiler/run')
|
||||
PIPELINE_CLOSURE_SOURCE_MAP_DIR = 'prod-static/source-map'
|
||||
|
||||
# Disable stuffing the entire JavaScript codebase inside an anonymous function.
|
||||
# We need modules to be externally visible, so that methods can be called from
|
||||
# event handlers defined in HTML.
|
||||
PIPELINE_DISABLE_WRAPPER = True
|
||||
|
||||
|
||||
USING_RABBITMQ = DEPLOYED
|
||||
# This password also appears in servers/configure-rabbitmq
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
{% extends "zephyr/base.html" %}
|
||||
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
{# User Activity. #}
|
||||
|
||||
{% block customhead %}
|
||||
{{ block.super }}
|
||||
{% compressed_js 'activity' %}
|
||||
{% minified_js 'activity' %}
|
||||
{% compressed_css 'activity' %}
|
||||
{% endblock %}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
|
||||
{# API information page #}
|
||||
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
{% block customhead %}
|
||||
{{ block.super }}
|
||||
{% compressed_js 'api' %}
|
||||
{% minified_js 'api' %}
|
||||
{% endblock %}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
{# Base template for the whole site. #}
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
|
|
@ -40,7 +41,7 @@ mixpanel.init("{{ mixpanel_token }}", {track_pageview: {{ enable_metrics }}});
|
|||
<script type="text/javascript">
|
||||
page_params.enable_metrics = {{ enable_metrics }};
|
||||
</script>
|
||||
{% compressed_js 'common' %}
|
||||
{% minified_js 'common' %}
|
||||
{% block customhead %}
|
||||
{% endblock %}
|
||||
</head>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
{# Includes some other templates as tabs. #}
|
||||
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
{% block page_params %}
|
||||
{# Insert parameters, which have been encoded with JSONEncoderForHTML. #}
|
||||
|
|
@ -23,10 +24,10 @@ var page_params = {{ page_params }};
|
|||
{% else %}
|
||||
{% compressed_css 'app' %}
|
||||
{% endif %}
|
||||
{% compressed_js 'app' %}
|
||||
{% minified_js 'app' %}
|
||||
|
||||
{% if debug %}
|
||||
{% compressed_js 'app_debug' %}
|
||||
{% minified_js 'app_debug' %}
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
{% extends "zephyr/portico.html" %}
|
||||
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
{# Portico page with signup code #}
|
||||
|
||||
{% block customhead %}
|
||||
{{ block.super }}
|
||||
{% compressed_js 'signup' %}
|
||||
{% minified_js 'signup' %}
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
{% extends "zephyr/portico.html" %}
|
||||
{% load compressed %}
|
||||
{% load minified_js %}
|
||||
|
||||
{% block customhead %}
|
||||
{% compressed_css 'portico' %}
|
||||
{% compressed_js 'landing-page' %}
|
||||
{% minified_js 'landing-page' %}
|
||||
{% endblock %}
|
||||
|
||||
{% block inner_content %}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Minifies JavaScripts, creating source maps
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
from glob import glob
|
||||
import subprocess
|
||||
import optparse
|
||||
import sys
|
||||
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option('--prev-deploy', nargs=1, metavar='DIR',
|
||||
help='A previous deploy from which to reuse files if possible')
|
||||
(options, args) = parser.parse_args()
|
||||
prev_deploy = options.prev_deploy
|
||||
|
||||
# We have to pull out JS_SPECS, defined in our settings file, so we know what
|
||||
# JavaScript source files to minify (and what output files to create).
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'humbug.settings'
|
||||
from django.conf import settings
|
||||
|
||||
os.chdir(settings.DEPLOY_ROOT)
|
||||
|
||||
STATIC_PATH = 'zephyr/static/'
|
||||
with open(settings.STATIC_HEADER_FILE, 'r') as fp:
|
||||
static_header = fp.read()
|
||||
|
||||
# Compile Handlebars templates
|
||||
subprocess.check_call(['tools/node', 'node_modules/.bin/handlebars']
|
||||
+ glob(os.path.join(STATIC_PATH, 'templates/*.handlebars'))
|
||||
+ ['--output', os.path.join(STATIC_PATH, 'templates/compiled.js'),
|
||||
'--known', 'if,unless,each,with'])
|
||||
|
||||
def get_changed_source_files(other_checkout):
|
||||
""" Get list of changed static files since other_checkout.
|
||||
If git fails to return a reasonable looking list, this returns None,
|
||||
in which case it should be assumed no files can be reused from
|
||||
other_checkout. """
|
||||
|
||||
try:
|
||||
git_dir = os.path.join(other_checkout, '.git')
|
||||
old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
|
||||
env={'GIT_DIR': git_dir})
|
||||
old_commit_sha1 = old_commit_sha1.rstrip()
|
||||
|
||||
git_diff = subprocess.check_output(['git', 'diff', '--name-only',
|
||||
old_commit_sha1])
|
||||
except subprocess.CalledProcessError:
|
||||
# If git returned an error, assume we can't reuse any files, and
|
||||
# regenerate everything.
|
||||
print "Warning: git returned an error when comparing to the previous"
|
||||
print ("deploy in %s. Will re-minify JavaScript instead of reusing"
|
||||
% other_checkout)
|
||||
return None
|
||||
|
||||
changed = set()
|
||||
for filename in git_diff.split('\n'):
|
||||
if not filename.startswith(STATIC_PATH):
|
||||
continue # Ignore non-static files.
|
||||
|
||||
if filename.endswith('.handlebars'):
|
||||
# Since Handlebars templates are compiled, treat this as if the
|
||||
# compiled .js file changed (which it did).
|
||||
changed.add('templates/compiled.js')
|
||||
continue
|
||||
|
||||
changed.add(filename)
|
||||
|
||||
return changed
|
||||
|
||||
|
||||
changed_files = set()
|
||||
if prev_deploy:
|
||||
changed_files = get_changed_source_files(prev_deploy)
|
||||
if changed_files is None:
|
||||
prev_deploy = None
|
||||
|
||||
JS_SPECS = settings.JS_SPECS
|
||||
CLOSURE_BINARY = 'tools/closure-compiler/run'
|
||||
|
||||
# Where to put minified JS and source maps
|
||||
MIN_DIR = os.path.join(STATIC_PATH, 'min/')
|
||||
MAP_DIR = os.path.join(STATIC_PATH, 'source-map/')
|
||||
subprocess.check_call(['mkdir', '-p', MIN_DIR, MAP_DIR])
|
||||
|
||||
for js_group, filespec in JS_SPECS.iteritems():
|
||||
in_files = [os.path.join(STATIC_PATH, filename)
|
||||
for filename in filespec['source_filenames']]
|
||||
out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename']))
|
||||
map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename'])
|
||||
+ '.map')
|
||||
|
||||
if (prev_deploy and len(set(in_files) & changed_files) == 0):
|
||||
# Try to reuse the output file from previous deploy
|
||||
try:
|
||||
for dest in [out_file, map_file]:
|
||||
src = os.path.join(prev_deploy, dest)
|
||||
os.path.getsize(src) # Just to throw error if it doesn't exist.
|
||||
if os.path.abspath(src) != os.path.abspath(dest):
|
||||
subprocess.check_call(['cp', src, dest])
|
||||
continue # Copy succeeded, so go on to next file.
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
pass # Copy failed, so fall through to minification instead.
|
||||
|
||||
# No previous deploy, or a source file has changed, or copying was
|
||||
# supposed to work but failed. Thus, minify the JS anew.
|
||||
cmd = '%s --language_in ECMASCRIPT5 --create_source_map %s %s' % (
|
||||
CLOSURE_BINARY, map_file, ' '.join(in_files))
|
||||
js = subprocess.check_output(cmd, shell=True)
|
||||
|
||||
# Write out the JS with static header prepended
|
||||
with open(out_file, 'w') as fp:
|
||||
fp.write(static_header)
|
||||
fp.write(js)
|
||||
|
|
@ -52,7 +52,8 @@ subprocess.check_call(["find", ".", "-name", "*.pyc", "-delete"], stdout=open('/
|
|||
|
||||
# Update static files
|
||||
logging.info("Updating static files")
|
||||
subprocess.check_call(["./tools/update-prod-static"])
|
||||
subprocess.check_call(["./tools/update-prod-static", "--prev-deploy",
|
||||
os.path.join(DEPLOYMENTS_DIR, 'current')])
|
||||
|
||||
logging.info("Restarting server...")
|
||||
subprocess.check_call(["./tools/restart-server"])
|
||||
|
|
|
|||
|
|
@ -1,25 +1,47 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Update static files in production.
|
||||
# Updates static files for production.
|
||||
|
||||
from os import chdir, path, open, close, O_WRONLY, O_CREAT
|
||||
from glob import glob
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import optparse
|
||||
import sys
|
||||
|
||||
chdir(path.join(path.dirname(__file__), '..'))
|
||||
# We need settings so we can figure out where the prod-static directory is.
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'humbug.settings'
|
||||
from django.conf import settings
|
||||
|
||||
# Redirect child processes' output to a log file (most recent run only)
|
||||
close(1)
|
||||
fp = open("update-prod-static.log", O_WRONLY|O_CREAT) # Will open on 1, stdout.
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option('--prev-deploy', nargs=1, metavar='DIR',
|
||||
help='A previous deploy from which to reuse files if possible')
|
||||
(options, args) = parser.parse_args()
|
||||
prev_deploy = options.prev_deploy
|
||||
|
||||
# Compile Handlebars templates
|
||||
subprocess.check_call(['tools/node', 'node_modules/.bin/handlebars']
|
||||
+ glob('zephyr/static/templates/*.handlebars')
|
||||
+ ['--output', 'zephyr/static/templates/compiled.js',
|
||||
'--known', 'if,unless,each,with'])
|
||||
os.chdir(settings.DEPLOY_ROOT)
|
||||
|
||||
# Collect the files that we're going to serve
|
||||
subprocess.check_call(['rm', '-r', 'prod-static/source-map'])
|
||||
subprocess.check_call(['mkdir', '-p', 'prod-static/source-map'])
|
||||
subprocess.check_call(['python', './manage.py', 'collectstatic', '--noinput'])
|
||||
# Redirect child processes' output to a log file (most recent run only).
|
||||
fp = open('update-prod-static.log', 'w')
|
||||
|
||||
# Compile Handlebars templates and minify JavaScripts.
|
||||
subprocess.check_call(['python', 'tools/minify-js']
|
||||
+ (['--prev-deploy', prev_deploy] if prev_deploy else []),
|
||||
stdout=fp, stderr=fp)
|
||||
|
||||
# Collect the files that we're going to serve.
|
||||
subprocess.check_call(['python', './manage.py', 'collectstatic', '--noinput'],
|
||||
stdout=fp, stderr=fp)
|
||||
|
||||
# Move the source maps out of the serve/ directory and into their
|
||||
# proper place.
|
||||
subprocess.check_call(['rm', '-rf', 'prod-static/source-map'],
|
||||
stdout=fp, stderr=fp)
|
||||
subprocess.check_call(['mkdir', '-p', 'prod-static'], # Needed if DEPLOYED
|
||||
stdout=fp, stderr=fp)
|
||||
subprocess.check_call(['mv', os.path.join(settings.STATIC_ROOT, 'source-map'),
|
||||
'prod-static/source-map'],
|
||||
stdout=fp, stderr=fp)
|
||||
|
||||
fp.close()
|
||||
|
|
|
|||
|
|
@ -1,19 +1,24 @@
|
|||
from django.conf import settings
|
||||
import re
|
||||
from django.contrib.staticfiles.finders import AppDirectoriesFinder
|
||||
|
||||
class ExcludeMinifiedMixin(object):
|
||||
class ExcludeUnminifiedMixin(object):
|
||||
""" Excludes unminified copies of our JavaScript code, templates
|
||||
and stylesheets, so that these sources don't end up getting served
|
||||
in production. """
|
||||
|
||||
def list(self, ignore_patterns):
|
||||
# We can't use ignore_patterns because the patterns are
|
||||
# applied to just the file part, not the entire path
|
||||
to_exclude = set()
|
||||
for collection in (settings.PIPELINE_CSS, settings.PIPELINE_JS):
|
||||
for key in collection:
|
||||
to_exclude.update(collection[key]['source_filenames'])
|
||||
excluded = '^(js|styles|templates)/'
|
||||
|
||||
super_class = super(ExcludeMinifiedMixin, self)
|
||||
# source-map/ should also not be included.
|
||||
# However, we work around that by moving it later,
|
||||
# in tools/update-prod-static.
|
||||
|
||||
super_class = super(ExcludeUnminifiedMixin, self)
|
||||
for path, storage in super_class.list(ignore_patterns):
|
||||
if not path in to_exclude:
|
||||
if not re.search(excluded, path):
|
||||
yield path, storage
|
||||
|
||||
class HumbugFinder(ExcludeMinifiedMixin, AppDirectoriesFinder):
|
||||
class HumbugFinder(ExcludeUnminifiedMixin, AppDirectoriesFinder):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class LineToFile(object):
|
|||
self._cumulative_counts = []
|
||||
|
||||
total = 0
|
||||
for filename in settings.PIPELINE_JS['app']['source_filenames']:
|
||||
for filename in settings.JS_SPECS['app']['source_filenames']:
|
||||
self._names.append(filename)
|
||||
self._cumulative_counts.append(total)
|
||||
with open(path.join('zephyr/static', filename), 'r') as fil:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
min/
|
||||
source-map/
|
||||
|
|
@ -20,14 +20,7 @@ class AddHeaderMixin(object):
|
|||
for name in paths:
|
||||
storage, path = paths[name]
|
||||
|
||||
# Find the top-level directory for the file
|
||||
head, _ = os.path.split(path)
|
||||
top_dir = head
|
||||
while head != '':
|
||||
top_dir = head
|
||||
head, _ = os.path.split(head)
|
||||
|
||||
if top_dir != 'min':
|
||||
if not path.startswith('min/') or not path.endswith('.css'):
|
||||
ret_dict[path] = (path, path, False)
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
from django.template import Node, Library, TemplateSyntaxError
|
||||
from django.conf import settings
|
||||
from django.contrib.staticfiles.storage import staticfiles_storage
|
||||
|
||||
register = Library()
|
||||
|
||||
class MinifiedJSNode(Node):
|
||||
def __init__(self, sourcefile):
|
||||
self.sourcefile = sourcefile
|
||||
|
||||
def render(self, context):
|
||||
if settings.DEBUG:
|
||||
scripts = settings.JS_SPECS[self.sourcefile]['source_filenames']
|
||||
else:
|
||||
scripts = [settings.JS_SPECS[self.sourcefile]['output_filename']]
|
||||
script_urls = [staticfiles_storage.url(script) for script in scripts]
|
||||
script_tags = ['<script type="text/javascript" src="%s" charset="utf-8"></script>'
|
||||
% url for url in script_urls]
|
||||
return '\n'.join(script_tags)
|
||||
|
||||
|
||||
@register.tag
|
||||
def minified_js(parser, token):
|
||||
try:
|
||||
tag_name, sourcefile = token.split_contents()
|
||||
except ValueError:
|
||||
raise TemplateSyntaxError("%s tag requires an argument" % tag_name)
|
||||
if not (sourcefile[0] == sourcefile[-1] and sourcefile[0] in ('"', "'")):
|
||||
raise TemplateSyntaxError("%s tag should be quoted" % tag_name)
|
||||
|
||||
sourcefile = sourcefile[1:-1]
|
||||
if sourcefile not in settings.JS_SPECS:
|
||||
raise TemplateSyntaxError("%s tag invalid argument: no JS file %s"
|
||||
% (tag_name, sourcefile))
|
||||
return MinifiedJSNode(sourcefile)
|
||||
Loading…
Reference in New Issue