webpack: Transition app.js to be compiled by webpack.

This commit moves all files previously under the 'app' bundle in
the Django pipeline to being compiled by webpack under the 'app'
entry point. In the process, it moves assets under the app entry
to a file called app.js that consumes all relevant css and js files.

This commit also edits the webpack config to be able to expose certain
variables for third party libraries that are currently required by
some modules. This is bad coding form and should be refactored to
requiring whatever dependencies a module may have; we're just
deferring that to the future to simplify the series of transitions we
need to do here. The variable exposure is done using expose-loader in
webpack.

The app/index.html template is edited to override the newly introduced
'commonjs' block in the base template. This is done as a temporary
measure so as not to disrupt other pages on the app during the transition.

It also fixes the value of the 'this' context that was being inferred
as window by third party libraries. This is done using imports-loader
in the webpack config.  This is also messy and probably isn't how we
want things to work long term.

package.json

@@ -22,6 +22,7 @@
     "font-awesome": "4.7.0",
     "handlebars": "4.0.11",
     "i18next": "3.4.4",
+    "imports-loader": "0.8.0",
     "jquery": "3.3.1",
     "jquery-validation": "1.17.0",
     "katex": "0.8.3",

static/js/bundles/app.js

@@ -0,0 +1,220 @@
+import "js/bundles/commons.js";
+
+// Import Third party libraries
+import "third/bootstrap-notify/js/bootstrap-notify.js";
+import "third/html5-formdata/formdata.js";
+import "third/jquery-filedrop/jquery.filedrop.js";
+import "third/jquery-caret/jquery.caret.1.5.2.js";
+import "third/jquery-idle/jquery.idle.js";
+import "third/jquery-autosize/jquery.autosize.js";
+import "third/spectrum/spectrum.js";
+import "third/sockjs/sockjs-0.3.4.js";
+import "third/marked/lib/marked.js";
+import "node_modules/xdate/src/xdate.js";
+import "node_modules/jquery-validation/dist/jquery.validate.js";
+import "node_modules/blueimp-md5/js/md5.js";
+import "node_modules/clipboard/dist/clipboard.js";
+import "node_modules/string.prototype.codepointat/codepointat.js";
+import "node_modules/winchan/winchan.js";
+import "node_modules/handlebars/dist/handlebars.runtime.js";
+import "node_modules/to-markdown/dist/to-markdown.js";
+import "node_modules/flatpickr/dist/flatpickr.js";
+import "node_modules/flatpickr/dist/plugins/confirmDate/confirmDate.js";
+import "node_modules/perfect-scrollbar/dist/perfect-scrollbar.js";
+import "node_modules/error-stack-parser/dist/error-stack-parser.min.js";
+import "node_modules/sortablejs/Sortable.js";
+import "generated/emoji/emoji_codes.js";
+import "generated/pygments_data.js";
+
+// Import App JS
+import "templates/compiled.js";
+import "js/translations.js";
+import "js/feature_flags.js";
+import "js/loading.js";
+import 'js/schema.js';
+import "js/util.js";
+import "js/keydown_util.js";
+import "js/lightbox_canvas.js";
+import "js/rtl.js";
+import "js/dict.js";
+import "js/scroll_util.js";
+import "js/components.js";
+import "js/localstorage.js";
+import "js/drafts.js";
+import "js/input_pill.js";
+import "js/user_pill.js";
+import "js/compose_pm_pill.js";
+import "js/channel.js";
+import "js/setup.js";
+import "js/unread_ui.js";
+import "js/unread_ops.js";
+import "js/muting.js";
+import "js/muting_ui.js";
+import "js/message_viewport.js";
+import "js/rows.js";
+import "js/people.js";
+import "js/user_groups.js";
+import "js/unread.js";
+import "js/topic_list.js";
+import "js/pm_list.js";
+import "js/pm_conversations.js";
+import "js/recent_senders.js";
+import "js/stream_sort.js";
+import "js/topic_generator.js";
+import "js/top_left_corner.js";
+import "js/stream_list.js";
+import "js/filter.js";
+import 'js/voting_widget.js';
+import 'js/tictactoe_widget.js';
+import 'js/zform.js';
+import 'js/widgetize.js';
+import 'js/submessage.js';
+import "js/fetch_status.js";
+import "js/message_list_data.js";
+import "js/message_list_view.js";
+import "js/message_list.js";
+import "js/message_live_update.js";
+import "js/narrow_state.js";
+import "js/narrow.js";
+import "js/reload.js";
+import "js/compose_fade.js";
+import "js/fenced_code.js";
+import "js/markdown.js";
+import 'js/local_message.js';
+import "js/echo.js";
+import "js/socket.js";
+import "js/sent_messages.js";
+import "js/compose_state.js";
+import "js/compose_actions.js";
+import "js/transmit.js";
+import "js/zcommand.js";
+import "js/compose.js";
+import "js/upload.js";
+import "js/stream_color.js";
+import "js/stream_data.js";
+import "js/topic_data.js";
+import "js/stream_muting.js";
+import "js/stream_events.js";
+import "js/stream_create.js";
+import "js/stream_edit.js";
+import "js/subs.js";
+import "js/message_edit.js";
+import "js/condense.js";
+import "js/resize.js";
+import "js/list_render.js";
+import "js/floating_recipient_bar.js";
+import "js/lightbox.js";
+import "js/ui_report.js";
+import "js/message_scroll.js";
+import "js/info_overlay.js";
+import "js/ui.js";
+import "js/night_mode.js";
+import "js/ui_util.js";
+import "js/pointer.js";
+import "js/click_handlers.js";
+import "js/settings_panel_menu.js";
+import "js/settings_toggle.js";
+import "js/scroll_bar.js";
+import "js/gear_menu.js";
+import "js/copy_and_paste.js";
+import "js/stream_popover.js";
+import "js/popovers.js";
+import "js/overlays.js";
+import "js/typeahead_helper.js";
+import "js/search_suggestion.js";
+import "js/search.js";
+import "js/composebox_typeahead.js";
+import "js/navigate.js";
+import "js/list_util.js";
+import "js/hotkey.js";
+import "js/favicon.js";
+import "js/notifications.js";
+import "js/hash_util.js";
+import "js/hashchange.js";
+import "js/invite.js";
+import "js/message_flags.js";
+import "js/alert_words.js";
+import "js/alert_words_ui.js";
+import "js/attachments_ui.js";
+import "js/message_store.js";
+import "js/message_util.js";
+import "js/message_events.js";
+import "js/message_fetch.js";
+import "js/server_events.js";
+import "js/server_events_dispatch.js";
+import "js/zulip.js";
+import "js/presence.js";
+import "js/user_search.js";
+import "js/buddy_data.js";
+import "js/buddy_list.js";
+import "js/list_cursor.js";
+import "js/activity.js";
+import "js/user_events.js";
+import "js/colorspace.js";
+import "js/timerender.js";
+import "js/tutorial.js";
+import "js/hotspots.js";
+import "js/templates.js";
+import "js/upload_widget.js";
+import "js/avatar.js";
+import "js/realm_icon.js";
+import 'js/reminder.js';
+import "js/settings_account.js";
+import "js/settings_display.js";
+import "js/settings_notifications.js";
+import "js/settings_bots.js";
+import "js/settings_muting.js";
+import "js/settings_sections.js";
+import "js/settings_emoji.js";
+import "js/settings_org.js";
+import "js/settings_users.js";
+import "js/settings_streams.js";
+import "js/settings_filters.js";
+import "js/settings_invites.js";
+import "js/settings_user_groups.js";
+import "js/settings_profile_fields.js";
+import "js/settings.js";
+import "js/admin_sections.js";
+import "js/admin.js";
+import "js/tab_bar.js";
+import "js/emoji.js";
+import "js/bot_data.js";
+import "js/reactions.js";
+import "js/typing.js";
+import "js/typing_status.js";
+import "js/typing_data.js";
+import "js/typing_events.js";
+import "js/ui_init.js";
+import "js/emoji_picker.js";
+import "js/compose_ui.js";
+import "js/panels.js";
+import 'js/settings_ui.js';
+import 'js/search_pill.js';
+
+// Import Styles
+
+import "third/bootstrap-notify/css/bootstrap-notify.css";
+import "third/spectrum/spectrum.css";
+import "node_modules/katex/dist/katex.css";
+import "node_modules/flatpickr/dist/flatpickr.css";
+import "node_modules/flatpickr/dist/plugins/confirmDate/confirmDate.css";
+import "styles/components.scss";
+import "styles/app_components.scss";
+import "styles/zulip.scss";
+import "styles/alerts.scss";
+import "styles/settings.scss";
+import "styles/subscriptions.scss";
+import "styles/drafts.scss";
+import "styles/input_pill.scss";
+import "styles/informational-overlays.scss";
+import "styles/compose.scss";
+import "styles/reactions.scss";
+import "styles/left-sidebar.scss";
+import "styles/right-sidebar.scss";
+import "styles/lightbox.scss";
+import "styles/popovers.scss";
+import "styles/media.scss";
+import "styles/typing_notifications.scss";
+import "styles/hotspots.scss";
+import "styles/night_mode.scss";
+import "styles/widgets.scss";

static/js/bundles/commons.js

@@ -0,0 +1,20 @@
+import "string.prototype.endswith";
+import "string.prototype.startswith";
+import "string.prototype.codepointat";
+import "node_modules/jquery/dist/jquery.js";
+import "node_modules/underscore/underscore.js";
+import "js/blueslip.js";
+import "third/bootstrap/js/bootstrap.js";
+import "js/common.js";
+import "node_modules/moment/min/moment.min.js";
+import "node_modules/moment-timezone/builds/moment-timezone-with-data.min.js";
+import "node_modules/sortablejs/Sortable.js";
+import "third/bootstrap/css/bootstrap.css";
+import "third/bootstrap/css/bootstrap-btn.css";
+import "third/bootstrap/css/bootstrap-responsive.css";
+import "node_modules/perfect-scrollbar/css/perfect-scrollbar.css";
+import "node_modules/font-awesome/css/font-awesome.css";
+import "third/fontawesome-legacy.css";
+import "generated/icons/style.css";
+import "node_modules/source-sans-pro/source-sans-pro.css";
+import "styles/pygments.scss";

templates/zerver/app/index.html

@@ -11,6 +11,11 @@
 </script>
 {% endblock %}
 
+{% block commonjs %}
+{{ render_bundle('app', attrs='nonce="%s"' % (csp_nonce)) }}
+{{ render_bundle('katex', attrs='nonce="%s"' % (csp_nonce)) }}
+{% endblock %}
+
 {% block customhead %}
 <meta name="apple-mobile-web-app-capable" content="yes">
 <link href="/static/images/logo/apple-touch-icon-precomposed.png" rel="apple-touch-icon-precomposed">
@@ -28,10 +33,7 @@
 
 </style>
 
-{{ render_bundle('translations', attrs='nonce="%s"' % (csp_nonce)) }}
+
-{{ minified_js('app', csp_nonce)|safe }}
-{{ render_bundle('app-styles', attrs='nonce="%s"' % (csp_nonce)) }}
-{{ render_bundle('katex', attrs='nonce="%s"' % (csp_nonce)) }}
 {% include "zerver/app/topic_is_muted.html" %}
 {% endblock %}
 

templates/zerver/base.html

@@ -24,7 +24,12 @@
         var page_params = {debug_mode: false};
         </script>
         {% endblock %}
+        <!-- This is a temporary block to enable webpack transition
+        This allows pages requiring common files via webpack to override
+        this block -->
+        {% block commonjs %}
         {{ render_bundle('common', attrs='nonce="%s"' % (csp_nonce)) }}
+        {% endblock %}
         {% block customhead %}
         {% endblock %}
 

tools/webpack-helpers.ts

@@ -0,0 +1,72 @@
+import { basename } from 'path';
+
+/* Return imports-loader format to the config
+    For example:
+    [
+        // Adds 'imports-loader?this=>widndow'
+        {path: './foler/my_module.js', args: '?this=>window'},
+    ]
+*/
+interface importLoaderOptions {
+   path: string;
+   args: string;
+}
+function getImportLoaders( optionsArr:Array<importLoaderOptions> ) {
+    let importsLoaders = [];
+    for(var loaderEntry of optionsArr) {
+        importsLoaders.push({
+            test: require.resolve(loaderEntry.path),
+            use: "imports-loader?" + loaderEntry.args
+        });
+    }
+    return importsLoaders;
+}
+/* Return expose-loader format to the config
+    For example
+    [
+        // Exposes 'my_module' as the name
+        {path: './folder/my_module.js'},
+
+        // Exposes 'my_custom_name'
+        {path: './folder/my_module.js', name: 'my_custom_name'},
+
+        // Exposes 'name1' and 'name2'
+        {path: './folder/my_module.js', name: ['name1', 'name2']}
+    ]
+*/
+interface exportLoaderOptions {
+   path: string;
+   name?: string | Array<string>;
+}
+function getExposeLoaders( optionsArr:Array<exportLoaderOptions> ) {
+    let exposeLoaders = [];
+    for(var loaderEntry of optionsArr) {
+        let path = loaderEntry.path;
+        let name = "";
+        let useArr = [];
+        // If no name is provided, infer it
+        if(!loaderEntry.name) {
+            name = basename(path, '.js');
+            useArr.push({loader: 'expose-loader', options: name});
+        } else {
+            // If name is an array
+            if(Array.isArray(loaderEntry.name)) {
+                for(var exposeName of loaderEntry.name) {
+                    useArr.push({loader: 'expose-loader', options: exposeName})
+                }
+            // If name is a string
+            } else {
+                useArr.push({loader: 'expose-loader', options: loaderEntry.name});
+            }
+        }
+        exposeLoaders.push({
+            test: require.resolve(path),
+            use: useArr
+        });
+    }
+    return exposeLoaders;
+}
+export {
+    getExposeLoaders,
+    getImportLoaders
+}

tools/webpack.assets.json

@@ -56,33 +56,7 @@
     ],
     "translations": "./static/js/translations.js",
     "zxcvbn": "./node_modules/zxcvbn/dist/zxcvbn.js",
-    "app-styles": [
+    "app": "./static/js/bundles/app.js",
-        "./static/third/bootstrap-notify/css/bootstrap-notify.css",
-        "./static/third/spectrum/spectrum.css",
-        "./static/node_modules/katex/dist/katex.css",
-        "./node_modules/flatpickr/dist/flatpickr.css",
-        "./node_modules/flatpickr/dist/plugins/confirmDate/confirmDate.css",
-        "./static/styles/components.scss",
-        "./static/styles/app_components.scss",
-        "./static/styles/zulip.scss",
-        "./static/styles/alerts.scss",
-        "./static/styles/settings.scss",
-        "./static/styles/subscriptions.scss",
-        "./static/styles/drafts.scss",
-        "./static/styles/input_pill.scss",
-        "./static/styles/informational-overlays.scss",
-        "./static/styles/compose.scss",
-        "./static/styles/reactions.scss",
-        "./static/styles/left-sidebar.scss",
-        "./static/styles/right-sidebar.scss",
-        "./static/styles/lightbox.scss",
-        "./static/styles/popovers.scss",
-        "./static/styles/media.scss",
-        "./static/styles/typing_notifications.scss",
-        "./static/styles/hotspots.scss",
-        "./static/styles/night_mode.scss",
-        "./static/styles/widgets.scss"
-    ],
     "archive-styles": [
         "./node_modules/katex/dist/katex.css",
         "./static/styles/zulip.scss",

tools/webpack.config.ts

@@ -1,6 +1,7 @@
 import { resolve } from 'path';
 import * as BundleTracker from 'webpack-bundle-tracker';
 import * as webpack from 'webpack';
+import { getExposeLoaders, getImportLoaders } from './webpack-helpers';
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const assets = require('./webpack.assets.json');
@@ -14,6 +15,7 @@ function getHotCSS(bundle:any[], isProd:boolean) {
         'css-hot-loader',
     ].concat(bundle);
 }
+
 export default (env?: string) : webpack.Configuration => {
     const production: boolean = env === "production";
     let config: webpack.Configuration = {
@@ -35,43 +37,10 @@ export default (env?: string) : webpack.Configuration => {
                 // Currently the source maps don't work with these so use unminified files
                 // if debugging is required.
                 {
-                    test: /(min|zxcvbn)\.js/,
+                    // We dont want to match admin.js
+                    test: /(\.min|min\.|zxcvbn)\.js/,
                     use: [ 'script-loader' ],
                 },
-                // Expose Global variables to webpack
-                // Use the unminified versions of jquery and underscore so that
-                // Good error messages show up in production and development in the source maps
-                {
-                    test: require.resolve('../static/node_modules/jquery/dist/jquery.js'),
-                    use: [
-                        {loader: 'expose-loader', options: '$'},
-                        {loader: 'expose-loader', options: 'jQuery'},
-                    ],
-                },
-                {
-                    test: require.resolve('../node_modules/underscore/underscore.js'),
-                    use: [
-                        {loader: 'expose-loader', options: '_'},
-                    ],
-                },
-                {
-                    test: require.resolve('../static/js/debug.js'),
-                    use: [
-                        {loader: 'expose-loader', options: 'debug'},
-                    ],
-                },
-                {
-                    test: require.resolve('../static/js/blueslip.js'),
-                    use: [
-                        {loader: 'expose-loader', options: 'blueslip'},
-                    ],
-                },
-                {
-                    test: require.resolve('../static/js/common.js'),
-                    use: [
-                        {loader: 'expose-loader', options: 'common'},
-                    ],
-                },
                 // regular css files
                 {
                     test: /\.css$/,
@@ -81,9 +50,9 @@ export default (env?: string) : webpack.Configuration => {
                             loader: 'css-loader',
                             options: {
                                 sourceMap: true
-                            }
+                            },
                         },
-                    ], production),
+                    ], production)
                 },
                 // sass / scss loader
                 {
@@ -122,6 +91,11 @@ export default (env?: string) : webpack.Configuration => {
             filename: production ? '[name].[chunkhash].js' : '[name].js',
         },
         resolve: {
+            modules: [
+                resolve(__dirname, "../static"),
+                resolve(__dirname, "../node_modules"),
+                resolve(__dirname, "../"),
+            ],
             extensions: [".tsx", ".ts", ".js", ".json", ".scss", ".css"],
         },
         // We prefer cheap-module-source-map over any eval-** options
@@ -131,6 +105,38 @@ export default (env?: string) : webpack.Configuration => {
         // breakpoints in chrome.
         devtool: production ? 'source-map' : 'cheap-module-source-map',
     };
+
+    // Add variables within modules
+    // Because of code corecing the value of window within the libraries
+    var importsOptions = [
+        {
+            path: "../static/third/spectrum/spectrum.js",
+            args: "this=>window"
+        }
+    ];
+    config.module.rules.push(...getImportLoaders(importsOptions));
+
+    // Expose Global variables for third party libraries to webpack modules
+    // Use the unminified versions of jquery and underscore so that
+    // Good error messages show up in production and development in the source maps
+    var exposeOptions = [
+        { path: "../node_modules/blueimp-md5/js/md5.js" },
+        { path: "../node_modules/clipboard/dist/clipboard.js", name: "ClipboardJS" },
+        { path: "../node_modules/xdate/src/xdate.js", name: "XDate" },
+        { path: "../node_modules/perfect-scrollbar/dist/perfect-scrollbar.js", name: "PerfectScrollbar" },
+        { path: "../static/third/marked/lib/marked.js" },
+        { path: "../static/generated/emoji/emoji_codes.js" },
+        { path: "../static/generated/pygments_data.js" },
+        { path: "../static/js/debug.js" },
+        { path: "../static/js/blueslip.js" },
+        { path: "../static/js/common.js" },
+        { path: "../node_modules/jquery/dist/jquery.js", name: ['$', 'jQuery'] },
+        { path: "../node_modules/underscore/underscore.js", name: '_' },
+        { path: "../node_modules/handlebars/dist/handlebars.runtime.js", name: 'Handlebars' },
+        { path: "../node_modules/sortablejs/Sortable.js"}
+    ];
+    config.module.rules.push(...getExposeLoaders(exposeOptions));
+
     if (production) {
         config.plugins = [
             new BundleTracker({filename: 'webpack-stats-production.json'}),
@@ -165,7 +171,6 @@ export default (env?: string) : webpack.Configuration => {
                 chunkFilename: "[chunkhash].css"
             }),
         ];
-
         config.devServer = {
             clientLogLevel: "error",
             stats: "errors-only",

version.py

@@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git"
 # Typically, adding a dependency only requires a minor version bump, and
 # removing a dependency requires a major version bump.
 
-PROVISION_VERSION = '21.2'
+PROVISION_VERSION = '22.0'

yarn.lock

@@ -4808,6 +4808,13 @@ import-local@^1.0.0:
     pkg-dir "^2.0.0"
     resolve-cwd "^2.0.0"
 
+imports-loader@0.8.0:
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/imports-loader/-/imports-loader-0.8.0.tgz#030ea51b8ca05977c40a3abfd9b4088fe0be9a69"
+  dependencies:
+    loader-utils "^1.0.2"
+    source-map "^0.6.1"
+
 imurmurhash@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"

zerver/tests/test_home.py

@@ -39,7 +39,7 @@ class HomeTest(ZulipTestCase):
             'Search streams',
             'Welcome to Zulip',
             # Verify that the app styles get included
-            'app-styles-stubentry.js',
+            'app-stubentry.js',
             'var page_params',
         ]
 

zproject/settings.py

@@ -1127,7 +1127,6 @@ JS_SPECS = {
     },
 }
 
-app_srcs = JS_SPECS['app']['source_filenames']
 if DEVELOPMENT:
     WEBPACK_STATS_FILE = os.path.join('var', 'webpack-stats-dev.json')
 else: