diff --git a/docs/production/maintain-secure-upgrade.md b/docs/production/maintain-secure-upgrade.md index 41ad07a701..7537dd5550 100644 --- a/docs/production/maintain-secure-upgrade.md +++ b/docs/production/maintain-secure-upgrade.md @@ -215,8 +215,8 @@ git_repo_url = https://github.com/zulip/zulip.git **Systems with limited RAM**: If you are running a minimal Zulip server with 2GB of RAM or less, the upgrade can fail due to the system running out of RAM running both the Zulip server and Zulip's - static asset build process (`tools/minify-js`, which calls - `webpack`, is usually the step that fails). If you encounter this, + static asset build process (`tools/webpack` + is usually the step that fails). If you encounter this, you can run `supervisorctl stop all` to shut down the Zulip server while you run the upgrade (this will, of course, add some downtime, which is part of we already recommend more RAM for organizations of diff --git a/docs/subsystems/front-end-build-process.md b/docs/subsystems/front-end-build-process.md index 96e4436203..13660f0021 100644 --- a/docs/subsystems/front-end-build-process.md +++ b/docs/subsystems/front-end-build-process.md @@ -78,10 +78,6 @@ If you want to test minified files in development, look for the `PIPELINE_ENABLED =` line in `zproject/settings.py` and set it to `True` -- or just set `DEBUG = False`. -Note that `static/html/5xx.html` will only render properly if -minification is enabled, since they, by nature, hardcode the path -`static/min/portico.css`. - ## How it works in production You can learn a lot from reading about django-pipeline, but a few diff --git a/puppet/zulip/manifests/static_asset_compiler.pp b/puppet/zulip/manifests/static_asset_compiler.pp index 6563dc92a3..52c6fcdd2e 100644 --- a/puppet/zulip/manifests/static_asset_compiler.pp +++ b/puppet/zulip/manifests/static_asset_compiler.pp @@ -2,10 +2,7 @@ class zulip::static_asset_compiler { include zulip::common case $::osfamily { 'debian': { - $closure_compiler_package = 'closure-compiler' $static_asset_compiler_packages = [ - # Needed for minify-js - $closure_compiler_package, 'yui-compressor', # Used by makemessages i18n 'gettext', @@ -13,7 +10,6 @@ class zulip::static_asset_compiler { } 'redhat': { $static_asset_compiler_packages = [ - # TODO CentOS doesn't have closure-compiler 'yuicompressor', 'gettext', ] diff --git a/static/.gitignore b/static/.gitignore index 428366b85b..ac1fc3bd87 100644 --- a/static/.gitignore +++ b/static/.gitignore @@ -1,6 +1,4 @@ # Code -/min/ -/source-map/ /webpack-bundles # Generated static files diff --git a/tools/circleci/Dockerfile.template b/tools/circleci/Dockerfile.template index 1f6bd55f07..7526947a52 100644 --- a/tools/circleci/Dockerfile.template +++ b/tools/circleci/Dockerfile.template @@ -89,7 +89,7 @@ RUN DOCKERIZE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cac # Extra packages used by Zulip. RUN apt-get update \ && apt-get install --no-install-recommends \ - closure-compiler memcached rabbitmq-server redis-server \ + memcached rabbitmq-server redis-server \ hunspell-en-us supervisor libssl-dev yui-compressor puppet \ gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \ libldap2-dev libmemcached-dev python-dev python-pip \ diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 169981cc6c..bccd3f10d7 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -143,7 +143,6 @@ POSTGRES_VERSION_MAP = { POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename] COMMON_DEPENDENCIES = [ - "closure-compiler", "memcached", "rabbitmq-server", "supervisor", diff --git a/tools/minify-js b/tools/minify-js deleted file mode 100755 index 379eb5100f..0000000000 --- a/tools/minify-js +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 - -# Minifies JavaScripts, creating source maps - - -import os -import subprocess -import argparse -import sys -import shutil - -parser = argparse.ArgumentParser() -parser.add_argument('--prev-deploy', metavar='DIR', - help='a previous deploy from which to reuse files if possible') -args = parser.parse_args() -prev_deploy = args.prev_deploy - -# We have to pull out JS_SPECS, defined in our settings file, so we know what -# JavaScript source files to minify (and what output files to create). -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -import scripts.lib.setup_path_on_import -from typing import Any, Dict, Optional, Set - -os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' -from django.conf import settings - -os.chdir(settings.DEPLOY_ROOT) - -STATIC_PATH = 'static/' - -# Create webpack bundle -subprocess.check_call(['tools/webpack']) - -def get_changed_source_files(other_checkout): - # type: (str) -> Optional[Set[str]] - """ Get list of changed static files since other_checkout. - If git fails to return a reasonable looking list, this returns None, - in which case it should be assumed no files can be reused from - other_checkout. """ - - try: - git_dir = os.path.join(other_checkout, '.git') - old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - env={'GIT_DIR': git_dir}, universal_newlines=True) - old_commit_sha1 = old_commit_sha1.rstrip() - - git_diff = subprocess.check_output(['git', 'diff', '--name-only', - old_commit_sha1], universal_newlines=True) - except subprocess.CalledProcessError: - # If git returned an error, assume we can't reuse any files, and - # regenerate everything. - print("Warning: git returned an error when comparing to the previous") - print("deploy in %s. Will re-minify JavaScript instead of reusing" - % (other_checkout,)) - return None - - changed = set() # type: Set[str] - for filename in git_diff.split('\n'): - if filename in ["package.json", "zproject/settings.py", "tools/minify-js"]: - print("Changed a core JS pipeline file; not reusing cached minification results") - return None - if not filename.startswith(STATIC_PATH): - continue # Ignore non-static files. - - if filename.endswith('.handlebars'): - continue - - changed.add(filename) - - return changed - - -changed_files = set() # type: Set[str] -if prev_deploy: - changed_files_tmp = get_changed_source_files(prev_deploy) - if changed_files_tmp is None: - prev_deploy = None - else: - changed_files = changed_files_tmp - -JS_SPECS = settings.JS_SPECS -CLOSURE_BINARY = '/usr/bin/closure-compiler' -if not os.path.exists(CLOSURE_BINARY): - CLOSURE_BINARY = 'tools/closure-compiler/run' - if not os.path.exists(CLOSURE_BINARY): - print("closure-compiler not installed; the Vagrant tools/provision installs it via apt " - "or you can manually unpack http://dl.google.com/closure-compiler/compiler-latest.zip to " - "tools/closure-compiler") - sys.exit(1) - -# Where to put minified JS and source maps -MIN_DIR = os.path.join(STATIC_PATH, 'min/') -MAP_DIR = os.path.join(STATIC_PATH, 'source-map/') -os.makedirs(MIN_DIR, exist_ok=True) -os.makedirs(MAP_DIR, exist_ok=True) - -for js_group_filespec_pair in JS_SPECS.items(): - # JS_SPECS is not typed, so forcefully type keys and values being read from JS_SPECS - js_group = js_group_filespec_pair[0] # type: str - filespec = js_group_filespec_pair[1] # type: Dict[str, Any] - # JS_SPECS look like 'js/foobar.js'. - # changed_files look like 'static/js/foobar.js'. - # So we prepend 'static/' to the JS_SPECS so these match up. - in_files = [os.path.join(STATIC_PATH, filename) - for filename in filespec['source_filenames']] - - min_files = [os.path.join(STATIC_PATH, filename) - for filename in filespec.get('minifed_source_filenames', [])] - - out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename'])) - map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename']) + - '.map') - - if ('force_minify' not in filespec) and \ - (prev_deploy and len(set(in_files) & changed_files) == 0): - # Try to reuse the output file from previous deploy - try: - for dest in [out_file, map_file]: - src = os.path.join(prev_deploy, dest) - os.path.getsize(src) # Just to throw error if it doesn't exist. - if os.path.abspath(src) != os.path.abspath(dest): - shutil.copyfile(src, dest) - continue # Copy succeeded, so go on to next file. - except (subprocess.CalledProcessError, OSError): - pass # Copy failed, so fall through to minification instead. - - # No previous deploy, or a source file has changed, or copying was - # supposed to work but failed. Thus, minify the JS anew. - # (N.B. we include STATIC_HEADER_FILE before the JavaScripts. - # This way it doesn't throw off the source map.) - cmd = [ - CLOSURE_BINARY, '--language_in', 'ECMASCRIPT5', - '--create_source_map', map_file, - settings.STATIC_HEADER_FILE] + in_files - js = subprocess.check_output(cmd) - - # Write out the JS - with open(out_file, 'wb') as fp: - # Minified source files (most likely libraries) should be loaded - # first to prevent any dependency errors. - for file in min_files: - with open(file, 'rb') as f: - fp.write(f.read()) - - fp.write(js) diff --git a/tools/test-install/prepare-base b/tools/test-install/prepare-base index 8f572aebb4..7d928bf014 100755 --- a/tools/test-install/prepare-base +++ b/tools/test-install/prepare-base @@ -47,7 +47,7 @@ run apt-get install -y --no-install-recommends \ xvfb parallel netcat unzip zip jq python3-pip wget curl eatmydata \ git crudini openssl ssl-cert \ build-essential python3-dev \ - closure-compiler memcached rabbitmq-server redis-server \ + memcached rabbitmq-server redis-server \ hunspell-en-us supervisor libssl-dev yui-compressor puppet \ gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \ libldap2-dev libmemcached-dev python-dev python-pip \ diff --git a/tools/update-prod-static b/tools/update-prod-static index 3600bece7d..218d9de4e6 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -52,9 +52,8 @@ run(['./tools/setup/generate-custom-icon-webfont'], stdout=fp, stderr=fp) # Build pygment data run(['./tools/setup/build_pygments_data'], stdout=fp, stderr=fp) -# Compile Handlebars templates and minify JavaScript. -run(['./tools/minify-js'] + (['--prev-deploy', prev_deploy] if prev_deploy else []), - stdout=fp, stderr=fp) +# Create webpack bundle +run(['./tools/webpack'], stdout=fp, stderr=fp) # Copy the KaTeX files outside node_modules os.makedirs(os.path.join(settings.STATIC_ROOT, 'node_modules/katex/dist/'), @@ -90,16 +89,9 @@ if not settings.PRODUCTION: # Compile translation strings to generate `.mo` files. run(['./manage.py', 'compilemessages'], stdout=fp, stderr=fp) -# Move the source maps out of the serve/ directory and into their -# proper place. -if os.path.exists('prod-static/source-map'): - shutil.rmtree('prod-static/source-map') - # Needed if PRODUCTION os.makedirs('prod-static', exist_ok=True) -shutil.move(os.path.join(settings.STATIC_ROOT, 'source-map'), 'prod-static/source-map') - # Generate /team page markdown for authors authors_cmd = ['./tools/update-authors-json'] if os.environ.get("TRAVIS"): diff --git a/version.py b/version.py index 94ea05b7ef..cefc354923 100644 --- a/version.py +++ b/version.py @@ -21,4 +21,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea # Typically, adding a dependency only requires a minor version bump, and # removing a dependency requires a major version bump. -PROVISION_VERSION = '36.1' +PROVISION_VERSION = '37.0' diff --git a/zerver/lib/unminify.py b/zerver/lib/unminify.py index 420961a5a6..ffcd72aaa1 100644 --- a/zerver/lib/unminify.py +++ b/zerver/lib/unminify.py @@ -30,7 +30,7 @@ class SourceMap: out = '' # type: str for ln in stacktrace.splitlines(): out += ln + '\n' - match = re.search(r'/static/(?:webpack-bundles|min)/(.+)(\.[\.0-9a-f]+\.js):(\d+):(\d+)', ln) + match = re.search(r'/static/webpack-bundles/(.+)(\.[\.0-9a-f]+\.js):(\d+):(\d+)', ln) if match: # Get the appropriate source map for the minified file. minified_src = match.groups()[0] + match.groups()[1] diff --git a/zerver/views/report.py b/zerver/views/report.py index c02444f6ff..f17cff556f 100644 --- a/zerver/views/report.py +++ b/zerver/views/report.py @@ -28,7 +28,6 @@ def get_js_source_map() -> Optional[SourceMap]: global js_source_map if not js_source_map and not (settings.DEVELOPMENT or settings.TEST_SUITE): js_source_map = SourceMap([ - os.path.join(settings.DEPLOY_ROOT, 'prod-static/source-map'), os.path.join(settings.STATIC_ROOT, 'webpack-bundles') ]) return js_source_map diff --git a/zproject/settings.py b/zproject/settings.py index 192a498787..73cab0f17a 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -914,20 +914,6 @@ PIPELINE = { 'JAVASCRIPT': {}, } -# Useful reading on how this works is in -# https://zulip.readthedocs.io/en/latest/subsystems/front-end-build-process.html -JS_SPECS = { - # One of the main reason we are treating the following bundles separately - # from webpack is we want to reduce the webpack compile time since These - # files are very large in size and are already minified or being minified - # in the pipeline itself - # We also want to minify sockjs separately for the sockjs iframe transport - 'sockjs': { - 'source_filenames': ['third/sockjs/sockjs-0.3.4.js'], - 'output_filename': 'min/sockjs-0.3.4.min.js' - }, -} - if DEVELOPMENT: WEBPACK_STATS_FILE = os.path.join('var', 'webpack-stats-dev.json') else: