diff --git a/tools/test-js-with-node b/tools/test-js-with-node
index 8bb2cc9c0e..f7b808dd1d 100755
--- a/tools/test-js-with-node
+++ b/tools/test-js-with-node
@@ -5,7 +5,7 @@ import os
 import pwd
 import subprocess
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Set
 
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.dirname(TOOLS_DIR))
@@ -33,163 +33,173 @@ USAGE = """
     tools/test-js-with-node --coverage           - to generate coverage report
     """
 
+
+def make_set(files: List[str]) -> Set[str]:
+    for i in range(1, len(files)):
+        if files[i - 1] > files[i]:
+            raise Exception(f"Please move {files[i]} so that names are sorted.")
+    return set(files)
+
+
 # We do not yet require 100% line coverage for these files:
-EXEMPT_FILES = {
-    "static/js/about_zulip.js",
-    "static/js/admin.js",
-    "static/js/alert_popup.ts",
-    "static/js/archive.js",
-    "static/js/attachments_ui.js",
-    "static/js/avatar.js",
-    "static/js/billing/helpers.js",
-    "static/js/billing/upgrade.js",
-    "static/js/blueslip.ts",
-    "static/js/blueslip_stacktrace.ts",
-    "static/js/click_handlers.js",
-    "static/js/compose_actions.js",
-    "static/js/compose_closed_ui.js",
-    "static/js/composebox_typeahead.js",
-    "static/js/compose_fade.js",
-    "static/js/compose_validate.js",
-    "static/js/compose.js",
-    "static/js/compose_ui.js",
-    "static/js/condense.js",
-    "static/js/confirm_dialog.js",
-    "static/js/dialog_widget.js",
-    "static/js/copy_and_paste.js",
-    "static/js/csrf.ts",
-    "static/js/css_variables.js",
-    "static/js/dark_theme.ts",
-    "static/js/debug.js",
-    "static/js/deprecated_feature_notice.js",
-    "static/js/desktop_integration.js",
-    "static/js/drafts.js",
-    "static/js/echo.js",
-    "static/js/emoji_picker.js",
-    "static/js/emojisets.js",
-    "static/js/favicon.js",
-    "static/js/feedback_widget.js",
-    "static/js/floating_recipient_bar.js",
-    "static/js/gear_menu.js",
-    "static/js/giphy.js",
-    "static/js/global.d.ts",
-    "static/js/hashchange.js",
-    "static/js/hbs.d.ts",
-    "static/js/hotkey.js",
-    "static/js/hotspots.js",
-    "static/js/info_overlay.js",
-    "static/js/invite.js",
-    "static/js/lightbox_canvas.js",
-    "static/js/lightbox.js",
-    "static/js/list_util.ts",
-    "static/js/loading.ts",
-    "static/js/local_message.js",
-    "static/js/localstorage.js",
-    "static/js/spectators.js",
-    "static/js/message_edit.js",
-    "static/js/message_edit_history.js",
-    "static/js/message_events.js",
-    "static/js/message_fetch.js",
-    "static/js/message_list_data.js",
-    "static/js/message_list.js",
-    "static/js/message_list_view.js",
-    "static/js/message_lists.js",
-    "static/js/message_live_update.js",
-    "static/js/message_scroll.js",
-    "static/js/message_util.js",
-    "static/js/message_viewport.js",
-    "static/js/muted_topics_ui.js",
-    "static/js/muted_users_ui.js",
-    "static/js/narrow.js",
-    "static/js/navigate.js",
-    "static/js/notifications.js",
-    "static/js/overlays.js",
-    "static/js/padded_widget.ts",
-    "static/js/page_params.ts",
-    "static/js/navbar_alerts.js",
-    "static/js/pm_list_dom.js",
-    "static/js/poll_widget.js",
-    "static/js/popovers.js",
-    "static/js/popover_menus.js",
-    "static/js/ready.ts",
-    "static/js/realm_icon.js",
-    "static/js/realm_logo.js",
-    "static/js/realm_playground.js",
-    "static/js/realm_user_settings_defaults.ts",
-    "static/js/recent_topics_ui.js",
-    "static/js/recent_topics_util.js",
-    "static/js/reload.js",
-    "static/js/reminder.js",
-    "static/js/resize.js",
-    "static/js/rows.js",
-    "static/js/scroll_bar.js",
-    "static/js/search_pill_widget.js",
-    "static/js/sent_messages.js",
-    "static/js/server_events.js",
-    "static/js/dropdown_list_widget.js",
-    "static/js/settings_account.js",
-    "static/js/settings_bots.js",
-    "static/js/settings_display.js",
-    "static/js/settings_emoji.js",
-    "static/js/settings_exports.js",
-    "static/js/settings_invites.js",
-    "static/js/settings.js",
-    "static/js/settings_linkifiers.js",
-    "static/js/settings_playgrounds.js",
-    "static/js/settings_muted_topics.js",
-    "static/js/settings_muted_users.js",
-    "static/js/settings_notifications.js",
-    "static/js/settings_org.js",
-    "static/js/settings_panel_menu.js",
-    "static/js/settings_profile_fields.js",
-    "static/js/settings_realm_user_settings_defaults.js",
-    "static/js/settings_sections.js",
-    "static/js/settings_streams.js",
-    "static/js/settings_toggle.js",
-    "static/js/settings_ui.js",
-    "static/js/settings_users.js",
-    "static/js/setup.js",
-    "static/js/spoilers.ts",
-    "static/js/starred_messages_ui.js",
-    "static/js/stream_bar.js",
-    "static/js/stream_color.js",
-    "static/js/stream_create.js",
-    "static/js/stream_edit.js",
-    "static/js/stream_list.js",
-    "static/js/stream_muting.js",
-    "static/js/stream_popover.js",
-    "static/js/stream_subscribers_ui.js",
-    "static/js/stream_ui_updates.js",
-    "static/js/submessage.js",
-    "static/js/stream_settings_ui.js",
-    "static/js/message_view_header.js",
-    "static/js/timerender.ts",
-    "static/js/tippyjs.js",
-    "static/js/todo_widget.js",
-    "static/js/topic_list.js",
-    "static/js/topic_zoom.js",
-    "static/js/tutorial.js",
-    "static/js/types.ts",
-    "static/js/typing_events.js",
-    "static/js/typing.js",
-    "static/js/ui_init.js",
-    "static/js/ui.js",
-    "static/js/ui_report.ts",
-    "static/js/ui_util.ts",
-    "static/js/unread_ops.js",
-    "static/js/unread_ui.js",
-    "static/js/upload_widget.ts",
-    "static/js/user_profile.js",
-    "static/js/user_settings.ts",
-    "static/js/user_status_ui.js",
-    "static/js/webpack_public_path.js",
-    "static/js/zcommand.js",
-    "static/js/zform.js",
-    "static/js/zulip.js",
-    "static/js/zulip_test.js",
-    "static/js/billing/event_status.js",
-    "static/shared/js/poll_data.js",
-}
+EXEMPT_FILES = make_set(
+    [
+        "static/js/about_zulip.js",
+        "static/js/admin.js",
+        "static/js/alert_popup.ts",
+        "static/js/archive.js",
+        "static/js/attachments_ui.js",
+        "static/js/avatar.js",
+        "static/js/billing/event_status.js",
+        "static/js/billing/helpers.js",
+        "static/js/billing/upgrade.js",
+        "static/js/blueslip.ts",
+        "static/js/blueslip_stacktrace.ts",
+        "static/js/click_handlers.js",
+        "static/js/compose.js",
+        "static/js/compose_actions.js",
+        "static/js/compose_closed_ui.js",
+        "static/js/compose_fade.js",
+        "static/js/compose_ui.js",
+        "static/js/compose_validate.js",
+        "static/js/composebox_typeahead.js",
+        "static/js/condense.js",
+        "static/js/confirm_dialog.js",
+        "static/js/copy_and_paste.js",
+        "static/js/csrf.ts",
+        "static/js/css_variables.js",
+        "static/js/dark_theme.ts",
+        "static/js/debug.js",
+        "static/js/deprecated_feature_notice.js",
+        "static/js/desktop_integration.js",
+        "static/js/dialog_widget.js",
+        "static/js/drafts.js",
+        "static/js/dropdown_list_widget.js",
+        "static/js/echo.js",
+        "static/js/emoji_picker.js",
+        "static/js/emojisets.js",
+        "static/js/favicon.js",
+        "static/js/feedback_widget.js",
+        "static/js/floating_recipient_bar.js",
+        "static/js/gear_menu.js",
+        "static/js/giphy.js",
+        "static/js/global.d.ts",
+        "static/js/hashchange.js",
+        "static/js/hbs.d.ts",
+        "static/js/hotkey.js",
+        "static/js/hotspots.js",
+        "static/js/info_overlay.js",
+        "static/js/invite.js",
+        "static/js/lightbox.js",
+        "static/js/lightbox_canvas.js",
+        "static/js/list_util.ts",
+        "static/js/loading.ts",
+        "static/js/local_message.js",
+        "static/js/localstorage.js",
+        "static/js/message_edit.js",
+        "static/js/message_edit_history.js",
+        "static/js/message_events.js",
+        "static/js/message_fetch.js",
+        "static/js/message_list.js",
+        "static/js/message_list_data.js",
+        "static/js/message_list_view.js",
+        "static/js/message_lists.js",
+        "static/js/message_live_update.js",
+        "static/js/message_scroll.js",
+        "static/js/message_util.js",
+        "static/js/message_view_header.js",
+        "static/js/message_viewport.js",
+        "static/js/muted_topics_ui.js",
+        "static/js/muted_users_ui.js",
+        "static/js/narrow.js",
+        "static/js/navbar_alerts.js",
+        "static/js/navigate.js",
+        "static/js/notifications.js",
+        "static/js/overlays.js",
+        "static/js/padded_widget.ts",
+        "static/js/page_params.ts",
+        "static/js/pm_list_dom.js",
+        "static/js/poll_widget.js",
+        "static/js/popover_menus.js",
+        "static/js/popovers.js",
+        "static/js/ready.ts",
+        "static/js/realm_icon.js",
+        "static/js/realm_logo.js",
+        "static/js/realm_playground.js",
+        "static/js/realm_user_settings_defaults.ts",
+        "static/js/recent_topics_ui.js",
+        "static/js/recent_topics_util.js",
+        "static/js/reload.js",
+        "static/js/reminder.js",
+        "static/js/resize.js",
+        "static/js/rows.js",
+        "static/js/scroll_bar.js",
+        "static/js/search_pill_widget.js",
+        "static/js/sent_messages.js",
+        "static/js/server_events.js",
+        "static/js/settings.js",
+        "static/js/settings_account.js",
+        "static/js/settings_bots.js",
+        "static/js/settings_display.js",
+        "static/js/settings_emoji.js",
+        "static/js/settings_exports.js",
+        "static/js/settings_invites.js",
+        "static/js/settings_linkifiers.js",
+        "static/js/settings_muted_topics.js",
+        "static/js/settings_muted_users.js",
+        "static/js/settings_notifications.js",
+        "static/js/settings_org.js",
+        "static/js/settings_panel_menu.js",
+        "static/js/settings_playgrounds.js",
+        "static/js/settings_profile_fields.js",
+        "static/js/settings_realm_user_settings_defaults.js",
+        "static/js/settings_sections.js",
+        "static/js/settings_streams.js",
+        "static/js/settings_toggle.js",
+        "static/js/settings_ui.js",
+        "static/js/settings_users.js",
+        "static/js/setup.js",
+        "static/js/spectators.js",
+        "static/js/spoilers.ts",
+        "static/js/starred_messages_ui.js",
+        "static/js/stream_bar.js",
+        "static/js/stream_color.js",
+        "static/js/stream_create.js",
+        "static/js/stream_edit.js",
+        "static/js/stream_list.js",
+        "static/js/stream_muting.js",
+        "static/js/stream_popover.js",
+        "static/js/stream_settings_ui.js",
+        "static/js/stream_subscribers_ui.js",
+        "static/js/stream_ui_updates.js",
+        "static/js/submessage.js",
+        "static/js/timerender.ts",
+        "static/js/tippyjs.js",
+        "static/js/todo_widget.js",
+        "static/js/topic_list.js",
+        "static/js/topic_zoom.js",
+        "static/js/tutorial.js",
+        "static/js/types.ts",
+        "static/js/typing.js",
+        "static/js/typing_events.js",
+        "static/js/ui.js",
+        "static/js/ui_init.js",
+        "static/js/ui_report.ts",
+        "static/js/ui_util.ts",
+        "static/js/unread_ops.js",
+        "static/js/unread_ui.js",
+        "static/js/upload_widget.ts",
+        "static/js/user_profile.js",
+        "static/js/user_settings.ts",
+        "static/js/user_status_ui.js",
+        "static/js/webpack_public_path.js",
+        "static/js/zcommand.js",
+        "static/js/zform.js",
+        "static/js/zulip.js",
+        "static/js/zulip_test.js",
+        "static/shared/js/poll_data.js",
+    ]
+)
 
 from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok