#!/usr/bin/env python3 # Minifies JavaScripts, creating source maps import os import subprocess import argparse import sys import shutil parser = argparse.ArgumentParser() parser.add_argument('--prev-deploy', metavar='DIR', help='a previous deploy from which to reuse files if possible') args = parser.parse_args() prev_deploy = args.prev_deploy # We have to pull out JS_SPECS, defined in our settings file, so we know what # JavaScript source files to minify (and what output files to create). sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) import scripts.lib.setup_path_on_import from typing import Any, Dict, Optional, Set os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' from django.conf import settings os.chdir(settings.DEPLOY_ROOT) STATIC_PATH = 'static/' # Compile Handlebars templates subprocess.check_call(['tools/compile-handlebars-templates']) # Create webpack bundle subprocess.check_call(['tools/webpack']) def get_changed_source_files(other_checkout): # type: (str) -> Optional[Set[str]] """ Get list of changed static files since other_checkout. If git fails to return a reasonable looking list, this returns None, in which case it should be assumed no files can be reused from other_checkout. """ try: git_dir = os.path.join(other_checkout, '.git') old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'], env={'GIT_DIR': git_dir}, universal_newlines=True) old_commit_sha1 = old_commit_sha1.rstrip() git_diff = subprocess.check_output(['git', 'diff', '--name-only', old_commit_sha1], universal_newlines=True) except subprocess.CalledProcessError: # If git returned an error, assume we can't reuse any files, and # regenerate everything. print("Warning: git returned an error when comparing to the previous") print("deploy in %s. Will re-minify JavaScript instead of reusing" % (other_checkout,)) return None changed = set() # type: Set[str] for filename in git_diff.split('\n'): if filename in ["package.json", "zproject/settings.py", "tools/minify-js"]: print("Changed a core JS pipeline file; not reusing cached minification results") return None if not filename.startswith(STATIC_PATH): continue # Ignore non-static files. if filename.endswith('.handlebars'): continue changed.add(filename) return changed changed_files = set() # type: Set[str] if prev_deploy: changed_files_tmp = get_changed_source_files(prev_deploy) if changed_files_tmp is None: prev_deploy = None else: changed_files = changed_files_tmp # Always use the newly compiled handlebars templates and webpack bundle. if prev_deploy: changed_files.add(os.path.join(STATIC_PATH, 'templates/compiled.js')) JS_SPECS = settings.JS_SPECS CLOSURE_BINARY = '/usr/bin/closure-compiler' if not os.path.exists(CLOSURE_BINARY): CLOSURE_BINARY = 'tools/closure-compiler/run' if not os.path.exists(CLOSURE_BINARY): print("closure-compiler not installed; the Vagrant tools/provision installs it via apt " "or you can manually unpack http://dl.google.com/closure-compiler/compiler-latest.zip to " "tools/closure-compiler") sys.exit(1) # Where to put minified JS and source maps MIN_DIR = os.path.join(STATIC_PATH, 'min/') MAP_DIR = os.path.join(STATIC_PATH, 'source-map/') os.makedirs(MIN_DIR, exist_ok=True) os.makedirs(MAP_DIR, exist_ok=True) for js_group_filespec_pair in JS_SPECS.items(): # JS_SPECS is not typed, so forcefully type keys and values being read from JS_SPECS js_group = js_group_filespec_pair[0] # type: str filespec = js_group_filespec_pair[1] # type: Dict[str, Any] # JS_SPECS look like 'js/foobar.js'. # changed_files look like 'static/js/foobar.js'. # So we prepend 'static/' to the JS_SPECS so these match up. in_files = [os.path.join(STATIC_PATH, filename) for filename in filespec['source_filenames']] min_files = [os.path.join(STATIC_PATH, filename) for filename in filespec.get('minifed_source_filenames', [])] out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename'])) map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename']) + '.map') if ('force_minify' not in filespec) and \ (prev_deploy and len(set(in_files) & changed_files) == 0): # Try to reuse the output file from previous deploy try: for dest in [out_file, map_file]: src = os.path.join(prev_deploy, dest) os.path.getsize(src) # Just to throw error if it doesn't exist. if os.path.abspath(src) != os.path.abspath(dest): shutil.copyfile(src, dest) continue # Copy succeeded, so go on to next file. except (subprocess.CalledProcessError, OSError): pass # Copy failed, so fall through to minification instead. # No previous deploy, or a source file has changed, or copying was # supposed to work but failed. Thus, minify the JS anew. # (N.B. we include STATIC_HEADER_FILE before the JavaScripts. # This way it doesn't throw off the source map.) cmd = [ CLOSURE_BINARY, '--language_in', 'ECMASCRIPT5', '--create_source_map', map_file, settings.STATIC_HEADER_FILE] + in_files js = subprocess.check_output(cmd) # Write out the JS with open(out_file, 'wb') as fp: # Minified source files (most likely libraries) should be loaded # first to prevent any dependency errors. for file in min_files: with open(file, 'rb') as f: fp.write(f.read()) fp.write(js)