webpack: Upgrade to Webpack 5.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

package.json

@@ -22,12 +22,11 @@
     "babel-loader": "^8.0.6",
     "babel-plugin-formatjs": "^10.2.6",
     "blueimp-md5": "^2.10.0",
-    "cache-loader": "^4.0.0",
     "clean-css": "^5.1.0",
     "clipboard": "^2.0.4",
     "core-js": "^3.6.5",
-    "css-loader": "^5.0.0",
+    "css-loader": "^6.2.0",
-    "css-minimizer-webpack-plugin": "^1.3.0",
+    "css-minimizer-webpack-plugin": "^3.0.2",
     "css.escape": "^1.5.1",
     "date-fns": "^2.16.1",
     "date-fns-tz": "^1.1.1",
@@ -36,27 +35,26 @@
     "emoji-datasource-twitter": "^7.0.2",
     "error-stack-parser": "^2.0.2",
     "eslint-plugin-formatjs": "^2.16.0",
-    "expose-loader": "^1.0.0",
+    "expose-loader": "^3.0.0",
-    "file-loader": "^6.0.0",
     "flatpickr": "^4.5.7",
     "font-awesome": "^4.7.0",
     "font-subset-loader2": "^1.1.7",
     "ga-gtag": "^1.0.1",
     "handlebars": "^4.7.2",
     "handlebars-loader": "^1.7.1",
-    "html-webpack-plugin": "^4.0.0-beta.8",
+    "html-webpack-plugin": "^5.3.2",
     "jquery": "^3.4.1",
     "jquery-caret-plugin": "^1.5.2",
     "jquery-validation": "^1.19.0",
     "katex": "^0.13.2",
     "lodash": "^4.17.19",
-    "mini-css-extract-plugin": "^1.2.0",
+    "mini-css-extract-plugin": "^2.2.2",
     "plotly.js": "^2.0.0",
     "postcss": "^8.0.3",
     "postcss-calc": "^8.0.0",
     "postcss-extend-rule": "^3.0.0",
     "postcss-import": "^14.0.2",
-    "postcss-loader": "^4.0.2",
+    "postcss-loader": "^6.1.1",
     "postcss-media-minmax": "https://github.com/andersk/postcss-media-minmax.git#01239f17f4a9872ace1dd133cee526a7de4ac9f5",
     "postcss-nested": "^5.0.0",
     "postcss-prefixwrap": "^1.24.0",
@@ -70,14 +68,13 @@
     "source-sans": "^3.28.0",
     "spectrum-colorpicker": "^1.8.1",
     "stacktrace-gps": "^3.0.4",
-    "style-loader": "^2.0.0",
+    "style-loader": "^3.2.1",
-    "terser-webpack-plugin": "^4.1.0",
     "text-field-edit": "^3.1.0",
     "tippy.js": "^6.2.7",
     "turndown": "^7.0.0",
     "url-loader": "^4.1.1",
     "webfonts-loader": "^7.0.1",
-    "webpack": "^4.33.0",
+    "webpack": "^5.52.1",
     "webpack-bundle-tracker": "^1.2.0",
     "webpack-cli": "^4.6.0",
     "winchan": "^0.2.1",
@@ -87,14 +84,12 @@
     "@babel/eslint-parser": "^7.11.3",
     "@formatjs/cli": "^4.2.6",
     "@types/clean-css": "^4.2.2",
-    "@types/css-minimizer-webpack-plugin": "~1.1.3",
+    "@types/css-minimizer-webpack-plugin": "^3.0.2",
     "@types/jquery": "^3.3.31",
     "@types/lodash": "^4.14.172",
-    "@types/mini-css-extract-plugin": "1.4.0",
+    "@types/mini-css-extract-plugin": "^2.2.0",
     "@types/node": "^14.0.11",
     "@types/node-fetch": "^2.5.8",
-    "@types/terser-webpack-plugin": "^4.1.0",
-    "@types/webpack": "^4.41.31",
     "@types/webpack-dev-server": "^4.1.0",
     "@types/zxcvbn": "^4.4.1",
     "@typescript-eslint/eslint-plugin": "^4.0.1",
@@ -103,6 +98,7 @@
     "callsites": "^3.1.0",
     "diff": "^5.0.0",
     "difflib": "^0.2.4",
+    "enhanced-resolve": "^5.8.2",
     "es-check": "^6.0.0",
     "eslint": "^7.2.0",
     "eslint-config-prettier": "^8.0.0",
@@ -135,12 +131,6 @@
   "resolutions": {
     "/source-map": "https://codeload.github.com/benthemonkey/source-map/tar.gz/d95423f77edef6cbb9e21d2d6014c7de85ae220a"
   },
-  "scripts": {
-    "postinstall": "rm -rf ./var/webpack-cache",
-    "lint": "eslint --quiet --cache",
-    "lint-fix": "eslint --quiet --fix",
-    "lint-loud": "eslint static/js frontend_tests --cache"
-  },
   "repository": {
     "type": "git",
     "url": "https://github.com/zulip/zulip.git"

tools/debug-require-webpack-plugin.ts

@@ -4,94 +4,111 @@
 
 import path from "path";
 
-import type webpack from "webpack";
+import type {ResolveRequest} from "enhanced-resolve";
-import {Template} from "webpack";
+import type {Chunk, Compiler, WebpackPluginInstance} from "webpack";
+import {NormalModule, Template} from "webpack";
 
-export default class DebugRequirePlugin {
+export default class DebugRequirePlugin implements WebpackPluginInstance {
-    apply(compiler: webpack.Compiler): void {
+    apply(compiler: Compiler): void {
-        const resolved = new Map();
+        const resolved = new Map<string, Set<string>>();
         const nameSymbol = Symbol("DebugRequirePluginName");
-        let debugRequirePath: string | undefined;
+        type NamedRequest = ResolveRequest & {
+            [nameSymbol]?: string;
+        };
+        let debugRequirePath: string | false = false;
 
-        (compiler as any).resolverFactory.hooks.resolver
+        compiler.resolverFactory.hooks.resolver
             .for("normal")
-            .tap("DebugRequirePlugin", (resolver: any) => {
+            .tap("DebugRequirePlugin", (resolver) => {
-                resolver.getHook("beforeRawModule").tap("DebugRequirePlugin", (req: any) => {
+                resolver.getHook("beforeRawModule").tap("DebugRequirePlugin", (req) => {
-                    req[nameSymbol] = req[nameSymbol] || req.request;
+                    if (!(nameSymbol in req)) {
-                });
+                        (req as NamedRequest)[nameSymbol] = req.request;
-
-                resolver.getHook("beforeRelative").tap("DebugRequirePlugin", (req: any) => {
-                    const inPath = path.relative(compiler.context, req.path);
-                    if (!inPath.startsWith("../")) {
-                        req[nameSymbol] = req[nameSymbol] || "./" + inPath;
                     }
+                    return undefined!;
                 });
 
-                resolver.getHook("beforeResolved").tap("DebugRequirePlugin", (req: any) => {
+                resolver.getHook("beforeRelative").tap("DebugRequirePlugin", (req) => {
-                    if (req[nameSymbol]) {
+                    if (req.path !== false) {
-                        const names = resolved.get(req.path);
+                        const inPath = path.relative(compiler.context, req.path);
-                        if (names) {
+                        if (!inPath.startsWith("../") && !(nameSymbol in req)) {
-                            names.add(req[nameSymbol]);
+                            (req as NamedRequest)[nameSymbol] = "./" + inPath;
-                        } else {
-                            resolved.set(req.path, new Set([req[nameSymbol]]));
                         }
                     }
+                    return undefined!;
                 });
+
+                resolver
+                    .getHook("beforeResolved")
+                    .tap("DebugRequirePlugin", (req: ResolveRequest) => {
+                        const name = (req as NamedRequest)[nameSymbol];
+                        if (name !== undefined && req.path !== false) {
+                            const names = resolved.get(req.path);
+                            if (names) {
+                                names.add(name);
+                            } else {
+                                resolved.set(req.path, new Set([name]));
+                            }
+                        }
+                        return undefined!;
+                    });
             });
 
         compiler.hooks.beforeCompile.tapPromise(
             "DebugRequirePlugin",
-            async ({normalModuleFactory}: any) => {
+            async ({normalModuleFactory}) => {
                 const resolver = normalModuleFactory.getResolver("normal");
-                debugRequirePath = await new Promise((resolve, reject) =>
+                debugRequirePath = await new Promise((resolve) =>
                     resolver.resolve(
                         {},
                         __dirname,
                         "./debug-require",
                         {},
-                        (err?: Error, result?: string) => (err ? reject(err) : resolve(result)),
+                        (err?: Error | null, result?: string | false) =>
+                            resolve(err ? false : result!),
                     ),
                 );
             },
         );
 
-        compiler.hooks.compilation.tap("DebugRequirePlugin", (compilation: any) => {
+        compiler.hooks.compilation.tap("DebugRequirePlugin", (compilation) => {
-            compilation.mainTemplate.hooks.beforeStartup.tap(
+            compilation.mainTemplate.hooks.bootstrap.tap(
                 "DebugRequirePlugin",
-                (source: string, chunk: webpack.compilation.Chunk) => {
+                (source: string, chunk: Chunk) => {
                     const ids: [string, string | number][] = [];
-                    let debugRequireId;
+                    let hasDebugRequire = false;
-                    chunk.hasModuleInGraph(
+                    compilation.chunkGraph.hasModuleInGraph(
-                        ({resource, rawRequest, id}: any) => {
+                        chunk,
-                            if (resource === debugRequirePath) {
+                        (m) => {
-                                debugRequireId = id;
+                            if (m instanceof NormalModule) {
-                            }
+                                const id = compilation.chunkGraph.getModuleId(m);
-                            for (const name of resolved.get(resource) || []) {
+                                if (m.resource === debugRequirePath) {
-                                ids.push([
+                                    hasDebugRequire = true;
-                                    rawRequest.slice(0, rawRequest.lastIndexOf("!") + 1) + name,
+                                }
-                                    id,
+                                for (const name of resolved.get(m.resource) ?? []) {
-                                ]);
+                                    ids.push([
+                                        m.rawRequest.slice(0, m.rawRequest.lastIndexOf("!") + 1) +
+                                            name,
+                                        id,
+                                    ]);
+                                }
                             }
                             return false;
                         },
                         () => true,
                     );
 
-                    if (debugRequireId === undefined) {
+                    if (!hasDebugRequire) {
                         return source;
                     }
 
                     ids.sort();
-                    const {requireFn} = compilation.mainTemplate;
                     return Template.asString([
                         source,
-                        `${requireFn}(${JSON.stringify(
+                        `__webpack_require__.debugRequireIds = ${JSON.stringify(
-                            debugRequireId,
-                        )}).initialize(${JSON.stringify(
                             Object.fromEntries(ids),
                             null,
                             "\t",
-                        )}, modules);`,
+                        )};`,
                     ]);
                 },
             );

tools/debug-require.js

@@ -2,22 +2,18 @@
 
 /* global __webpack_require__ */
 
-var webpackModules;
-
 function debugRequire(request) {
     if (!Object.prototype.hasOwnProperty.call(debugRequire.ids, request)) {
         throw new Error("Cannot find module '" + request + "'");
     }
     var moduleId = debugRequire.ids[request];
-    if (!Object.prototype.hasOwnProperty.call(webpackModules, moduleId)) {
+    if (!Object.prototype.hasOwnProperty.call(__webpack_require__.m, moduleId)) {
         throw new Error("Module '" + request + "' has not been loaded yet");
     }
     return __webpack_require__(moduleId);
 }
 
-debugRequire.initialize = function (ids, modules) {
+debugRequire.r = __webpack_require__;
-    debugRequire.ids = ids;
+debugRequire.ids = __webpack_require__.debugRequireIds;
-    webpackModules = modules;
-};
 
 module.exports = debugRequire;

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 97
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "159.2"
+PROVISION_VERSION = "160.0"

webpack.config.ts

@@ -5,7 +5,6 @@ import path from "path";
 import CssMinimizerPlugin from "css-minimizer-webpack-plugin";
 import HtmlWebpackPlugin from "html-webpack-plugin";
 import MiniCssExtractPlugin from "mini-css-extract-plugin";
-import TerserPlugin from "terser-webpack-plugin";
 import webpack from "webpack";
 import BundleTracker from "webpack-bundle-tracker";
 
@@ -13,20 +12,26 @@ import DebugRequirePlugin from "./tools/debug-require-webpack-plugin";
 import assets from "./tools/webpack.assets.json";
 import dev_assets from "./tools/webpack.dev-assets.json";
 
-const cacheLoader: webpack.RuleSetUseItem = {
-    loader: "cache-loader",
-    options: {
-        cacheDirectory: path.resolve(__dirname, "var/webpack-cache"),
-    },
-};
-
 export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.Configuration[] => {
     const production: boolean = argv.mode === "production";
 
-    const config: webpack.Configuration = {
+    const baseConfig: webpack.Configuration = {
-        name: "frontend",
         mode: production ? "production" : "development",
         context: __dirname,
+        cache: {
+            type: "filesystem",
+            buildDependencies: {
+                config: [__filename],
+            },
+        },
+        snapshot: {
+            immutablePaths: ["/srv/zulip-npm-cache"],
+        },
+    };
+
+    const frontendConfig: webpack.Configuration = {
+        ...baseConfig,
+        name: "frontend",
         entry: production
             ? assets
             : Object.fromEntries(
@@ -81,7 +86,7 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
                         path.resolve(__dirname, "static/shared/js"),
                         path.resolve(__dirname, "static/js"),
                     ],
-                    use: [cacheLoader, "babel-loader"],
+                    loader: "babel-loader",
                 },
                 // regular css files
                 {
@@ -89,7 +94,6 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
                     exclude: path.resolve(__dirname, "static/styles"),
                     use: [
                         MiniCssExtractPlugin.loader,
-                        cacheLoader,
                         {
                             loader: "css-loader",
                             options: {
@@ -104,7 +108,6 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
                     include: path.resolve(__dirname, "static/styles"),
                     use: [
                         MiniCssExtractPlugin.loader,
-                        cacheLoader,
                         {
                             loader: "css-loader",
                             options: {
@@ -122,54 +125,48 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
                 },
                 {
                     test: /\.hbs$/,
-                    use: [
+                    loader: "handlebars-loader",
-                        cacheLoader,
+                    options: {
-                        {
+                        ignoreHelpers: true,
-                            loader: "handlebars-loader",
+                        // Tell webpack not to explicitly require these.
-                            options: {
+                        knownHelpers: [
-                                ignoreHelpers: true,
+                            "if",
-                                // Tell webpack not to explicitly require these.
+                            "unless",
-                                knownHelpers: [
+                            "each",
-                                    "if",
+                            "with",
-                                    "unless",
+                            // The ones below are defined in static/js/templates.js
-                                    "each",
+                            "plural",
-                                    "with",
+                            "eq",
-                                    // The ones below are defined in static/js/templates.js
+                            "and",
-                                    "plural",
+                            "or",
-                                    "eq",
+                            "not",
-                                    "and",
+                            "t",
-                                    "or",
+                            "tr",
-                                    "not",
+                            "rendered_markdown",
-                                    "t",
+                        ],
-                                    "tr",
+                        preventIndent: true,
-                                    "rendered_markdown",
+                    },
-                                ],
-                                preventIndent: true,
-                            },
-                        },
-                    ],
                 },
                 // load fonts and files
                 {
                     test: /\.(eot|jpg|svg|ttf|otf|png|woff2?)$/,
-                    use: [
+                    type: "asset/resource",
-                        {
-                            loader: "file-loader",
-                            options: {
-                                name: production ? "[name].[hash].[ext]" : "[path][name].[ext]",
-                                outputPath: "files/",
-                            },
-                        },
-                    ],
                 },
             ],
         },
         output: {
             path: path.resolve(__dirname, "static/webpack-bundles"),
+            publicPath: "",
             filename: production ? "[name].[contenthash].js" : "[name].js",
+            assetModuleFilename: production
+                ? "files/[name].[hash][ext][query]"
+                : // Avoid directory traversal bug that upstream won't fix
+                  // (https://github.com/webpack/webpack/issues/11937)
+                  (pathData) => "files" + path.join("/", pathData.filename!),
             chunkFilename: production ? "[contenthash].js" : "[id].js",
         },
         resolve: {
+            ...baseConfig.resolve,
             extensions: [".ts", ".js"],
         },
         // We prefer cheap-module-source-map over any eval-* options
@@ -180,27 +177,9 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
             minimize: env.minimize ?? production,
             minimizer: [
                 new CssMinimizerPlugin({
-                    sourceMap: true,
+                    minify: CssMinimizerPlugin.cleanCssMinify,
-                    minify: (data: Record<string, string>, sourceMap) => {
-                        // css-minimizer-webpack-plugin needs this require
-                        // inside the function.
-                        // eslint-disable-next-line @typescript-eslint/consistent-type-imports, @typescript-eslint/no-var-requires
-                        const CleanCSS: typeof import("clean-css") = require("clean-css");
-                        const [[filename, styles]] = Object.entries(data);
-                        const out = new CleanCSS({sourceMap: true}).minify({
-                            [filename]: {styles, sourceMap},
-                        });
-                        return {
-                            css: out.styles,
-                            map: out.sourceMap.toString(),
-                            warnings: out.warnings,
-                        };
-                    },
-                }),
-                new TerserPlugin({
-                    cache: true,
-                    parallel: true,
                 }),
+                "...",
             ],
             splitChunks: {
                 chunks: "all",
@@ -215,13 +194,10 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
                 filename: production
                     ? "webpack-stats-production.json"
                     : "var/webpack-stats-dev.json",
-                relativePath: true,
             }),
             ...(production
                 ? []
                 : [
-                      // Better logging from console for hot reload
-                      new webpack.NamedModulesPlugin(),
                       // script-loader should load sourceURL in dev
                       new webpack.LoaderOptionsPlugin({debug: true}),
                   ]),
@@ -259,18 +235,16 @@ export default (env: {minimize?: boolean} = {}, argv: {mode?: string}): webpack.
     };
 
     const serverConfig: webpack.Configuration = {
+        ...baseConfig,
         name: "server",
-        mode: production ? "production" : "development",
         target: "node",
-        context: __dirname,
         entry: {
             "katex-cli": "shebang-loader!katex/cli",
         },
         output: {
             path: path.resolve(__dirname, "static/webpack-bundles"),
-            filename: "[name].js",
         },
     };
 
-    return [config, serverConfig];
+    return [frontendConfig, serverConfig];
 };