minify-js: Remove; everything has been migrated to Webpack.

min/sockjs-0.3.4.min.js is not used.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

docs/production/maintain-secure-upgrade.md

@@ -215,8 +215,8 @@ git_repo_url = https://github.com/zulip/zulip.git
 **Systems with limited RAM**: If you are running a minimal Zulip
   server with 2GB of RAM or less, the upgrade can fail due to the
   system running out of RAM running both the Zulip server and Zulip's
-  static asset build process (`tools/minify-js`, which calls
-  `webpack`, is usually the step that fails).  If you encounter this,
+  static asset build process (`tools/webpack`
+  is usually the step that fails).  If you encounter this,
   you can run `supervisorctl stop all` to shut down the Zulip server
   while you run the upgrade (this will, of course, add some downtime,
   which is part of we already recommend more RAM for organizations of

docs/subsystems/front-end-build-process.md

@@ -78,10 +78,6 @@ If you want to test minified files in development, look for the
 `PIPELINE_ENABLED =` line in `zproject/settings.py` and set it to `True`
 -- or just set `DEBUG = False`.
 
-Note that `static/html/5xx.html` will only render properly if
-minification is enabled, since they, by nature, hardcode the path
-`static/min/portico.css`.
-
 ## How it works in production
 
 You can learn a lot from reading about django-pipeline, but a few

puppet/zulip/manifests/static_asset_compiler.pp

@@ -2,10 +2,7 @@ class zulip::static_asset_compiler {
   include zulip::common
   case $::osfamily {
     'debian': {
-      $closure_compiler_package = 'closure-compiler'
       $static_asset_compiler_packages = [
-        # Needed for minify-js
-        $closure_compiler_package,
         'yui-compressor',
         # Used by makemessages i18n
         'gettext',
@@ -13,7 +10,6 @@ class zulip::static_asset_compiler {
     }
     'redhat': {
       $static_asset_compiler_packages = [
-        # TODO CentOS doesn't have closure-compiler
         'yuicompressor',
         'gettext',
       ]

static/.gitignore

@@ -1,6 +1,4 @@
 # Code
-/min/
-/source-map/
 /webpack-bundles
 
 # Generated static files

tools/circleci/Dockerfile.template

@@ -89,7 +89,7 @@ RUN DOCKERIZE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cac
 # Extra packages used by Zulip.
 RUN apt-get update \
   && apt-get install --no-install-recommends \
-    closure-compiler memcached rabbitmq-server redis-server \
+    memcached rabbitmq-server redis-server \
     hunspell-en-us supervisor libssl-dev yui-compressor puppet \
     gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
     libldap2-dev libmemcached-dev python-dev python-pip \

tools/lib/provision.py

@@ -143,7 +143,6 @@ POSTGRES_VERSION_MAP = {
 POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename]
 
 COMMON_DEPENDENCIES = [
-    "closure-compiler",
     "memcached",
     "rabbitmq-server",
     "supervisor",

tools/minify-js

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-
-# Minifies JavaScripts, creating source maps
-
-
-import os
-import subprocess
-import argparse
-import sys
-import shutil
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--prev-deploy', metavar='DIR',
-                    help='a previous deploy from which to reuse files if possible')
-args = parser.parse_args()
-prev_deploy = args.prev_deploy
-
-# We have to pull out JS_SPECS, defined in our settings file, so we know what
-# JavaScript source files to minify (and what output files to create).
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-import scripts.lib.setup_path_on_import
-from typing import Any, Dict, Optional, Set
-
-os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings'
-from django.conf import settings
-
-os.chdir(settings.DEPLOY_ROOT)
-
-STATIC_PATH = 'static/'
-
-# Create webpack bundle
-subprocess.check_call(['tools/webpack'])
-
-def get_changed_source_files(other_checkout):
-    # type: (str) -> Optional[Set[str]]
-    """ Get list of changed static files since other_checkout.
-    If git fails to return a reasonable looking list, this returns None,
-    in which case it should be assumed no files can be reused from
-    other_checkout. """
-
-    try:
-        git_dir = os.path.join(other_checkout, '.git')
-        old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
-                                                  env={'GIT_DIR': git_dir}, universal_newlines=True)
-        old_commit_sha1 = old_commit_sha1.rstrip()
-
-        git_diff = subprocess.check_output(['git', 'diff', '--name-only',
-                                            old_commit_sha1], universal_newlines=True)
-    except subprocess.CalledProcessError:
-        # If git returned an error, assume we can't reuse any files, and
-        # regenerate everything.
-        print("Warning: git returned an error when comparing to the previous")
-        print("deploy in %s. Will re-minify JavaScript instead of reusing"
-              % (other_checkout,))
-        return None
-
-    changed = set()  # type: Set[str]
-    for filename in git_diff.split('\n'):
-        if filename in ["package.json", "zproject/settings.py", "tools/minify-js"]:
-            print("Changed a core JS pipeline file; not reusing cached minification results")
-            return None
-        if not filename.startswith(STATIC_PATH):
-            continue  # Ignore non-static files.
-
-        if filename.endswith('.handlebars'):
-            continue
-
-        changed.add(filename)
-
-    return changed
-
-
-changed_files = set()  # type: Set[str]
-if prev_deploy:
-    changed_files_tmp = get_changed_source_files(prev_deploy)
-    if changed_files_tmp is None:
-        prev_deploy = None
-    else:
-        changed_files = changed_files_tmp
-
-JS_SPECS = settings.JS_SPECS
-CLOSURE_BINARY = '/usr/bin/closure-compiler'
-if not os.path.exists(CLOSURE_BINARY):
-    CLOSURE_BINARY = 'tools/closure-compiler/run'
-    if not os.path.exists(CLOSURE_BINARY):
-        print("closure-compiler not installed; the Vagrant tools/provision installs it via apt "
-              "or you can manually unpack http://dl.google.com/closure-compiler/compiler-latest.zip to "
-              "tools/closure-compiler")
-        sys.exit(1)
-
-# Where to put minified JS and source maps
-MIN_DIR = os.path.join(STATIC_PATH, 'min/')
-MAP_DIR = os.path.join(STATIC_PATH, 'source-map/')
-os.makedirs(MIN_DIR, exist_ok=True)
-os.makedirs(MAP_DIR, exist_ok=True)
-
-for js_group_filespec_pair in JS_SPECS.items():
-    # JS_SPECS is not typed, so forcefully type keys and values being read from JS_SPECS
-    js_group = js_group_filespec_pair[0]  # type: str
-    filespec = js_group_filespec_pair[1]  # type: Dict[str, Any]
-    #      JS_SPECS look like        'js/foobar.js'.
-    # changed_files look like 'static/js/foobar.js'.
-    # So we prepend 'static/' to the JS_SPECS so these match up.
-    in_files = [os.path.join(STATIC_PATH, filename)
-                for filename in filespec['source_filenames']]
-
-    min_files = [os.path.join(STATIC_PATH, filename)
-                 for filename in filespec.get('minifed_source_filenames', [])]
-
-    out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename']))
-    map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename']) +
-                            '.map')
-
-    if ('force_minify' not in filespec) and \
-       (prev_deploy and len(set(in_files) & changed_files) == 0):
-        # Try to reuse the output file from previous deploy
-        try:
-            for dest in [out_file, map_file]:
-                src = os.path.join(prev_deploy, dest)
-                os.path.getsize(src)  # Just to throw error if it doesn't exist.
-                if os.path.abspath(src) != os.path.abspath(dest):
-                    shutil.copyfile(src, dest)
-            continue  # Copy succeeded, so go on to next file.
-        except (subprocess.CalledProcessError, OSError):
-            pass  # Copy failed, so fall through to minification instead.
-
-    # No previous deploy, or a source file has changed, or copying was
-    # supposed to work but failed.  Thus, minify the JS anew.
-    # (N.B. we include STATIC_HEADER_FILE before the JavaScripts.
-    #  This way it doesn't throw off the source map.)
-    cmd = [
-        CLOSURE_BINARY, '--language_in', 'ECMASCRIPT5',
-        '--create_source_map', map_file,
-        settings.STATIC_HEADER_FILE] + in_files
-    js = subprocess.check_output(cmd)
-
-    # Write out the JS
-    with open(out_file, 'wb') as fp:
-        # Minified source files (most likely libraries) should be loaded
-        # first to prevent any dependency errors.
-        for file in min_files:
-            with open(file, 'rb') as f:
-                fp.write(f.read())
-
-        fp.write(js)

tools/test-install/prepare-base

@@ -47,7 +47,7 @@ run apt-get install -y --no-install-recommends \
   xvfb parallel netcat unzip zip jq python3-pip wget curl eatmydata \
   git crudini openssl ssl-cert \
   build-essential python3-dev \
-  closure-compiler memcached rabbitmq-server redis-server \
+  memcached rabbitmq-server redis-server \
   hunspell-en-us supervisor libssl-dev yui-compressor puppet \
   gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
   libldap2-dev libmemcached-dev python-dev python-pip \

tools/update-prod-static

@@ -52,9 +52,8 @@ run(['./tools/setup/generate-custom-icon-webfont'], stdout=fp, stderr=fp)
 # Build pygment data
 run(['./tools/setup/build_pygments_data'], stdout=fp, stderr=fp)
 
-# Compile Handlebars templates and minify JavaScript.
-run(['./tools/minify-js'] + (['--prev-deploy', prev_deploy] if prev_deploy else []),
-    stdout=fp, stderr=fp)
+# Create webpack bundle
+run(['./tools/webpack'], stdout=fp, stderr=fp)
 
 # Copy the KaTeX files outside node_modules
 os.makedirs(os.path.join(settings.STATIC_ROOT, 'node_modules/katex/dist/'),
@@ -90,16 +89,9 @@ if not settings.PRODUCTION:
 # Compile translation strings to generate `.mo` files.
 run(['./manage.py', 'compilemessages'], stdout=fp, stderr=fp)
 
-# Move the source maps out of the serve/ directory and into their
-# proper place.
-if os.path.exists('prod-static/source-map'):
-    shutil.rmtree('prod-static/source-map')
-
 # Needed if PRODUCTION
 os.makedirs('prod-static', exist_ok=True)
 
-shutil.move(os.path.join(settings.STATIC_ROOT, 'source-map'), 'prod-static/source-map')
-
 # Generate /team page markdown for authors
 authors_cmd = ['./tools/update-authors-json']
 if os.environ.get("TRAVIS"):

version.py

@@ -21,4 +21,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea
 # Typically, adding a dependency only requires a minor version bump, and
 # removing a dependency requires a major version bump.
 
-PROVISION_VERSION = '36.1'
+PROVISION_VERSION = '37.0'

zerver/lib/unminify.py

@@ -30,7 +30,7 @@ class SourceMap:
         out = ''  # type: str
         for ln in stacktrace.splitlines():
             out += ln + '\n'
-            match = re.search(r'/static/(?:webpack-bundles|min)/(.+)(\.[\.0-9a-f]+\.js):(\d+):(\d+)', ln)
+            match = re.search(r'/static/webpack-bundles/(.+)(\.[\.0-9a-f]+\.js):(\d+):(\d+)', ln)
             if match:
                 # Get the appropriate source map for the minified file.
                 minified_src = match.groups()[0] + match.groups()[1]

zerver/views/report.py

@@ -28,7 +28,6 @@ def get_js_source_map() -> Optional[SourceMap]:
     global js_source_map
     if not js_source_map and not (settings.DEVELOPMENT or settings.TEST_SUITE):
         js_source_map = SourceMap([
-            os.path.join(settings.DEPLOY_ROOT, 'prod-static/source-map'),
             os.path.join(settings.STATIC_ROOT, 'webpack-bundles')
         ])
     return js_source_map

zproject/settings.py

@@ -914,20 +914,6 @@ PIPELINE = {
     'JAVASCRIPT': {},
 }
 
-# Useful reading on how this works is in
-# https://zulip.readthedocs.io/en/latest/subsystems/front-end-build-process.html
-JS_SPECS = {
-    # One of the main reason we are treating the following bundles separately
-    # from webpack is we want to reduce the webpack compile time since These
-    # files are very large in size and are already minified or being minified
-    # in the pipeline itself
-    # We also want to minify sockjs separately for the sockjs iframe transport
-    'sockjs': {
-        'source_filenames': ['third/sockjs/sockjs-0.3.4.js'],
-        'output_filename': 'min/sockjs-0.3.4.min.js'
-    },
-}
-
 if DEVELOPMENT:
     WEBPACK_STATS_FILE = os.path.join('var', 'webpack-stats-dev.json')
 else: