py3: Switch almost all shebang lines to use `python3`.
This causes `upgrade-zulip-from-git`, as well as a no-option run of
`tools/build-release-tarball`, to produce a Zulip install running
Python 3, rather than Python 2. In particular this means that the
virtualenv we create, in which all application code runs, is Python 3.
One shebang line, on `zulip-ec2-configure-interfaces`, explicitly
keeps Python 2, and at least one external ops script, `wal-e`, also
still runs on Python 2. See discussion on the respective previous
commits that made those explicit. There may also be some other
third-party scripts we use, outside of this source tree and running
outside our virtualenv, that still run on Python 2.
2017-08-02 23:15:16 +02:00
|
|
|
#!/usr/bin/env python3
|
2017-11-09 08:41:47 +01:00
|
|
|
import argparse
|
2020-02-09 14:47:22 +01:00
|
|
|
import glob
|
2017-04-05 17:47:13 +02:00
|
|
|
import os
|
2020-04-11 12:24:10 +02:00
|
|
|
import pwd
|
2017-04-05 17:47:13 +02:00
|
|
|
import subprocess
|
|
|
|
import sys
|
2021-12-18 21:49:09 +01:00
|
|
|
from typing import Any, Dict, List, Set
|
2017-04-05 17:47:13 +02:00
|
|
|
|
|
|
|
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
sys.path.insert(0, os.path.dirname(TOOLS_DIR))
|
|
|
|
ROOT_DIR = os.path.dirname(TOOLS_DIR)
|
|
|
|
|
|
|
|
# check for the venv
|
|
|
|
from tools.lib import sanity_check
|
2020-06-11 00:54:34 +02:00
|
|
|
|
2017-04-05 17:47:13 +02:00
|
|
|
sanity_check.check_venv(__file__)
|
|
|
|
|
2017-10-18 07:20:41 +02:00
|
|
|
# Import this after we do the sanity_check so it doesn't crash.
|
2020-08-07 01:09:47 +02:00
|
|
|
import orjson
|
2020-06-11 00:54:34 +02:00
|
|
|
from zulint.printer import BOLDRED, CYAN, ENDC, GREEN
|
2020-02-09 15:09:37 +01:00
|
|
|
|
2023-02-22 23:04:10 +01:00
|
|
|
INDEX_JS = os.path.join(ROOT_DIR, "web/tests/lib/index.js")
|
2021-02-12 08:20:45 +01:00
|
|
|
NODE_COVERAGE_PATH = os.path.join(ROOT_DIR, "var/node-coverage/coverage-final.json")
|
2020-12-15 21:16:10 +01:00
|
|
|
|
|
|
|
# Ideally, we wouldn't need this line, but it seems to be required to
|
|
|
|
# avoid problems finding node_modules when running `cd tools; ./test-js-with-node`.
|
|
|
|
os.chdir(ROOT_DIR)
|
2020-01-26 15:49:01 +01:00
|
|
|
|
2021-02-12 08:20:45 +01:00
|
|
|
USAGE = """
|
2023-02-22 23:04:10 +01:00
|
|
|
tools/test-js-with-node - to run all tests
|
|
|
|
tools/test-js-with-node util.test.js activity.test.js - to run just a couple tests
|
|
|
|
tools/test-js-with-node --coverage - to generate coverage report
|
2021-02-12 08:20:45 +01:00
|
|
|
"""
|
2017-04-05 17:47:13 +02:00
|
|
|
|
2021-12-18 21:49:09 +01:00
|
|
|
|
|
|
|
def make_set(files: List[str]) -> Set[str]:
|
|
|
|
for i in range(1, len(files)):
|
|
|
|
if files[i - 1] > files[i]:
|
|
|
|
raise Exception(f"Please move {files[i]} so that names are sorted.")
|
|
|
|
return set(files)
|
|
|
|
|
|
|
|
|
2020-02-09 14:47:22 +01:00
|
|
|
# We do not yet require 100% line coverage for these files:
|
2021-12-18 21:49:09 +01:00
|
|
|
EXEMPT_FILES = make_set(
|
|
|
|
[
|
2023-06-02 09:20:46 +02:00
|
|
|
"web/shared/src/poll_data.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/about_zulip.js",
|
2023-10-01 21:25:13 +02:00
|
|
|
"web/src/add_stream_options_popover.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/add_subscribers_pill.js",
|
|
|
|
"web/src/admin.js",
|
|
|
|
"web/src/alert_popup.ts",
|
2023-10-03 00:28:16 +02:00
|
|
|
"web/src/alert_words_ui.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/archive.js",
|
|
|
|
"web/src/assets.d.ts",
|
2023-08-30 14:08:17 +02:00
|
|
|
"web/src/attachments_ui.ts",
|
2023-10-07 00:09:47 +02:00
|
|
|
"web/src/audible_notifications.js",
|
2023-09-01 10:18:48 +02:00
|
|
|
"web/src/avatar.ts",
|
2023-02-26 11:38:01 +01:00
|
|
|
"web/src/billing/event_status.ts",
|
|
|
|
"web/src/billing/helpers.ts",
|
|
|
|
"web/src/billing/upgrade.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/blueslip.ts",
|
|
|
|
"web/src/blueslip_stacktrace.ts",
|
|
|
|
"web/src/click_handlers.js",
|
|
|
|
"web/src/compose.js",
|
|
|
|
"web/src/compose_actions.js",
|
|
|
|
"web/src/compose_banner.ts",
|
2023-10-05 21:02:54 +02:00
|
|
|
"web/src/compose_call_ui.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/compose_closed_ui.js",
|
|
|
|
"web/src/compose_fade.js",
|
2023-10-06 22:36:26 +02:00
|
|
|
"web/src/compose_notifications.js",
|
2023-10-03 19:32:19 +02:00
|
|
|
"web/src/compose_popovers.js",
|
2023-03-31 06:27:36 +02:00
|
|
|
"web/src/compose_recipient.js",
|
2023-10-06 01:43:50 +02:00
|
|
|
"web/src/compose_reply.js",
|
2023-10-05 21:02:57 +02:00
|
|
|
"web/src/compose_setup.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/compose_state.js",
|
2023-05-25 10:34:37 +02:00
|
|
|
"web/src/compose_textarea.ts",
|
2023-06-07 14:18:18 +02:00
|
|
|
"web/src/compose_tooltips.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/compose_ui.js",
|
|
|
|
"web/src/compose_validate.js",
|
|
|
|
"web/src/composebox_typeahead.js",
|
|
|
|
"web/src/condense.js",
|
2023-05-18 16:01:57 +02:00
|
|
|
"web/src/confirm_dialog.ts",
|
2023-10-03 00:06:31 +02:00
|
|
|
"web/src/copied_tooltip.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/copy_and_paste.js",
|
|
|
|
"web/src/csrf.ts",
|
2023-04-04 09:19:27 +02:00
|
|
|
"web/src/css_variables.js",
|
2023-09-25 13:16:40 +02:00
|
|
|
"web/src/custom_profile_fields_ui.js",
|
2023-03-25 23:50:45 +01:00
|
|
|
"web/src/dark_theme.ts",
|
2023-04-01 20:50:34 +02:00
|
|
|
"web/src/debug.ts",
|
2021-09-16 18:51:42 +02:00
|
|
|
"web/src/demo_organizations_ui.js",
|
2023-05-18 16:22:20 +02:00
|
|
|
"web/src/deprecated_feature_notice.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/desktop_integration.js",
|
2023-03-30 18:21:28 +02:00
|
|
|
"web/src/dialog_widget.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/drafts.js",
|
2023-10-06 01:19:21 +02:00
|
|
|
"web/src/drafts_overlay_ui.js",
|
2023-05-07 14:42:54 +02:00
|
|
|
"web/src/dropdown_widget.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/echo.js",
|
|
|
|
"web/src/emoji_picker.js",
|
2023-05-21 09:15:52 +02:00
|
|
|
"web/src/emojisets.ts",
|
2023-03-24 22:02:09 +01:00
|
|
|
"web/src/favicon.ts",
|
2023-05-04 12:58:14 +02:00
|
|
|
"web/src/feedback_widget.ts",
|
2023-09-22 19:42:16 +02:00
|
|
|
"web/src/flatpickr.ts",
|
2023-03-29 18:36:52 +02:00
|
|
|
"web/src/gear_menu.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/giphy.js",
|
|
|
|
"web/src/global.d.ts",
|
2023-08-15 04:19:07 +02:00
|
|
|
"web/src/hash_util.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/hashchange.js",
|
|
|
|
"web/src/hbs.d.ts",
|
|
|
|
"web/src/hotkey.js",
|
|
|
|
"web/src/hotspots.js",
|
2023-08-09 07:23:30 +02:00
|
|
|
"web/src/inbox_ui.js",
|
|
|
|
"web/src/inbox_util.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/info_overlay.js",
|
2023-09-25 19:13:40 +02:00
|
|
|
"web/src/integration_url_modal.js",
|
2023-09-22 10:32:06 +02:00
|
|
|
"web/src/invite.ts",
|
2023-10-03 19:29:53 +02:00
|
|
|
"web/src/left_sidebar_navigation_area_popovers.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/lightbox.js",
|
|
|
|
"web/src/list_util.ts",
|
2023-04-28 08:55:36 +02:00
|
|
|
"web/src/list_widget.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/loading.ts",
|
|
|
|
"web/src/local_message.js",
|
2023-03-25 16:54:49 +01:00
|
|
|
"web/src/localstorage.ts",
|
2023-10-03 11:48:40 +02:00
|
|
|
"web/src/message_actions_popover.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/message_edit.js",
|
|
|
|
"web/src/message_edit_history.js",
|
|
|
|
"web/src/message_events.js",
|
2023-05-10 18:06:57 +02:00
|
|
|
"web/src/message_feed_loading.ts",
|
2023-06-01 01:57:47 +02:00
|
|
|
"web/src/message_feed_top_notices.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/message_fetch.js",
|
|
|
|
"web/src/message_list.js",
|
|
|
|
"web/src/message_list_data.js",
|
2023-07-01 20:17:24 +02:00
|
|
|
"web/src/message_list_hover.js",
|
2023-06-01 02:08:40 +02:00
|
|
|
"web/src/message_list_tooltips.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/message_list_view.js",
|
|
|
|
"web/src/message_lists.js",
|
|
|
|
"web/src/message_live_update.js",
|
2023-10-06 22:29:25 +02:00
|
|
|
"web/src/message_notifications.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/message_scroll.js",
|
2023-10-02 23:16:51 +02:00
|
|
|
"web/src/message_scroll_state.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/message_util.js",
|
|
|
|
"web/src/message_view_header.js",
|
|
|
|
"web/src/message_viewport.js",
|
2023-07-10 11:30:52 +02:00
|
|
|
"web/src/messages_overlay_ui.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/muted_users_ui.js",
|
|
|
|
"web/src/narrow.js",
|
2023-08-08 23:59:25 +02:00
|
|
|
"web/src/narrow_history.js",
|
2023-10-05 23:18:00 +02:00
|
|
|
"web/src/narrow_title.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/navbar_alerts.js",
|
|
|
|
"web/src/navigate.js",
|
|
|
|
"web/src/notifications.js",
|
2023-03-29 16:34:02 +02:00
|
|
|
"web/src/overlays.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/padded_widget.ts",
|
|
|
|
"web/src/page_params.ts",
|
2023-09-15 15:58:47 +02:00
|
|
|
"web/src/playground_links_popover.js",
|
2023-06-27 17:27:20 +02:00
|
|
|
"web/src/plotly.js.d.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/pm_list.js",
|
2023-10-03 00:57:17 +02:00
|
|
|
"web/src/pm_list_dom.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/poll_widget.js",
|
|
|
|
"web/src/popover_menus.js",
|
|
|
|
"web/src/popover_menus_data.js",
|
|
|
|
"web/src/popovers.js",
|
|
|
|
"web/src/read_receipts.js",
|
|
|
|
"web/src/ready.ts",
|
2023-09-09 10:25:05 +02:00
|
|
|
"web/src/realm_icon.ts",
|
2023-09-09 10:32:13 +02:00
|
|
|
"web/src/realm_logo.ts",
|
2023-07-11 18:48:37 +02:00
|
|
|
"web/src/realm_playground.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/realm_user_settings_defaults.ts",
|
2023-09-06 23:14:37 +02:00
|
|
|
"web/src/recent_view_ui.js",
|
2023-09-06 23:21:13 +02:00
|
|
|
"web/src/recent_view_util.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/reload.js",
|
2023-10-06 09:04:22 +02:00
|
|
|
"web/src/reload_setup.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/reminder.js",
|
|
|
|
"web/src/resize.js",
|
2023-10-05 17:54:50 +02:00
|
|
|
"web/src/resize_handler.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/rows.js",
|
2023-04-14 21:35:56 +02:00
|
|
|
"web/src/scheduled_messages.js",
|
|
|
|
"web/src/scheduled_messages_overlay_ui.js",
|
2023-10-03 11:40:45 +02:00
|
|
|
"web/src/scheduled_messages_popover.js",
|
2023-10-03 23:59:24 +02:00
|
|
|
"web/src/scheduled_messages_ui.js",
|
2023-04-01 20:43:58 +02:00
|
|
|
"web/src/scroll_bar.ts",
|
2023-04-27 05:44:27 +02:00
|
|
|
"web/src/scroll_util.ts",
|
2022-05-16 19:24:49 +02:00
|
|
|
"web/src/search.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/sent_messages.js",
|
sentry: Add frontend event monitoring.
Zulip already has integrations for server-side Sentry integration;
however, it has historically used the Zulip-specific `blueslip`
library for monitoring browser-side errors. However, the latter sends
errors to email, as well optionally to an internal `#errors` stream.
While this is sufficient for low volumes of users, and useful in that
it does not rely on outside services, at higher volumes it is very
difficult to do any analysis or filtering of the errors. Client-side
errors are exceptionally noisy, with many false positives due to
browser extensions or similar, so determining real real errors from a
stream of un-grouped emails or messages in a stream is quite
difficult.
Add a client-side Javascript sentry integration. To provide useful
backtraces, this requires extending the pre-deploy hooks to upload the
source-maps to Sentry. Additional keys are added to the non-public
API of `page_params` to control the DSN, realm identifier, and sample
rates.
2023-02-13 20:50:57 +01:00
|
|
|
"web/src/sentry.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/server_events.js",
|
|
|
|
"web/src/settings.js",
|
|
|
|
"web/src/settings_account.js",
|
|
|
|
"web/src/settings_bots.js",
|
|
|
|
"web/src/settings_display.js",
|
2023-09-02 11:40:52 +02:00
|
|
|
"web/src/settings_emoji.ts",
|
2023-10-03 01:24:27 +02:00
|
|
|
"web/src/settings_exports.ts",
|
2023-10-03 02:10:04 +02:00
|
|
|
"web/src/settings_invites.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/settings_linkifiers.js",
|
2023-10-03 03:16:24 +02:00
|
|
|
"web/src/settings_muted_users.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/settings_notifications.js",
|
|
|
|
"web/src/settings_org.js",
|
|
|
|
"web/src/settings_panel_menu.js",
|
|
|
|
"web/src/settings_playgrounds.js",
|
|
|
|
"web/src/settings_profile_fields.js",
|
|
|
|
"web/src/settings_realm_domains.js",
|
|
|
|
"web/src/settings_realm_user_settings_defaults.js",
|
|
|
|
"web/src/settings_sections.js",
|
|
|
|
"web/src/settings_streams.js",
|
|
|
|
"web/src/settings_toggle.js",
|
2023-08-04 09:58:55 +02:00
|
|
|
"web/src/settings_ui.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/settings_user_groups_legacy.js",
|
2023-04-16 06:33:06 +02:00
|
|
|
"web/src/settings_user_topics.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/settings_users.js",
|
2023-05-11 11:44:10 +02:00
|
|
|
"web/src/setup.ts",
|
2023-09-28 11:08:09 +02:00
|
|
|
"web/src/sidebar_ui.js",
|
2023-08-15 04:48:25 +02:00
|
|
|
"web/src/spectators.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/spoilers.ts",
|
|
|
|
"web/src/starred_messages_ui.js",
|
|
|
|
"web/src/stream_bar.js",
|
|
|
|
"web/src/stream_color.js",
|
2023-10-03 10:38:49 +02:00
|
|
|
"web/src/stream_color_events.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/stream_create.js",
|
|
|
|
"web/src/stream_create_subscribers.js",
|
|
|
|
"web/src/stream_edit.js",
|
|
|
|
"web/src/stream_edit_subscribers.js",
|
|
|
|
"web/src/stream_list.js",
|
|
|
|
"web/src/stream_muting.js",
|
|
|
|
"web/src/stream_popover.js",
|
2023-10-03 21:27:54 +02:00
|
|
|
"web/src/stream_settings_api.js",
|
2023-06-09 00:48:46 +02:00
|
|
|
"web/src/stream_settings_containers.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/stream_settings_ui.js",
|
|
|
|
"web/src/stream_ui_updates.js",
|
|
|
|
"web/src/submessage.js",
|
|
|
|
"web/src/subscriber_api.js",
|
|
|
|
"web/src/timerender.ts",
|
|
|
|
"web/src/tippyjs.js",
|
|
|
|
"web/src/todo_widget.js",
|
|
|
|
"web/src/topic_list.js",
|
2023-10-03 19:21:19 +02:00
|
|
|
"web/src/topic_popover.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/topic_zoom.js",
|
|
|
|
"web/src/tutorial.js",
|
|
|
|
"web/src/types.ts",
|
|
|
|
"web/src/typing.js",
|
|
|
|
"web/src/typing_events.js",
|
|
|
|
"web/src/ui_init.js",
|
|
|
|
"web/src/ui_report.ts",
|
|
|
|
"web/src/ui_util.ts",
|
|
|
|
"web/src/unread.js",
|
|
|
|
"web/src/unread_ops.js",
|
|
|
|
"web/src/unread_ui.js",
|
|
|
|
"web/src/upload.js",
|
|
|
|
"web/src/upload_widget.ts",
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
"web/src/url-template.d.ts",
|
2023-09-13 10:59:24 +02:00
|
|
|
"web/src/user_card_popover.js",
|
2023-09-25 19:19:40 +02:00
|
|
|
"web/src/user_deactivation_ui.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/user_group_create.js",
|
|
|
|
"web/src/user_group_create_members.js",
|
2023-08-15 09:06:16 +02:00
|
|
|
"web/src/user_group_create_members_data.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/user_group_edit.js",
|
|
|
|
"web/src/user_group_edit_members.js",
|
2023-09-08 16:44:48 +02:00
|
|
|
"web/src/user_group_popover.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/user_group_ui_updates.js",
|
2023-04-10 12:56:36 +02:00
|
|
|
"web/src/user_groups.ts",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/user_groups_settings_ui.js",
|
|
|
|
"web/src/user_profile.js",
|
|
|
|
"web/src/user_settings.ts",
|
2023-10-03 10:16:10 +02:00
|
|
|
"web/src/user_sort.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/user_status.js",
|
|
|
|
"web/src/user_status_ui.js",
|
2023-10-03 19:24:00 +02:00
|
|
|
"web/src/user_topic_popover.js",
|
2023-09-10 14:14:26 +02:00
|
|
|
"web/src/user_topics.ts",
|
2023-06-24 07:57:58 +02:00
|
|
|
"web/src/user_topics_ui.js",
|
2023-09-23 08:20:22 +02:00
|
|
|
"web/src/views_util.js",
|
2023-02-22 23:03:47 +01:00
|
|
|
"web/src/webpack_public_path.js",
|
|
|
|
"web/src/zcommand.js",
|
|
|
|
"web/src/zform.js",
|
|
|
|
"web/src/zulip.js",
|
|
|
|
"web/src/zulip_test.js",
|
2023-02-22 23:04:10 +01:00
|
|
|
"web/tests/lib/mdiff.js",
|
2023-05-01 23:17:55 +02:00
|
|
|
"web/tests/lib/zjquery_element.js",
|
2021-12-18 21:49:09 +01:00
|
|
|
]
|
|
|
|
)
|
2017-06-07 04:24:39 +02:00
|
|
|
|
2021-03-03 05:00:15 +01:00
|
|
|
from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok
|
|
|
|
|
2017-11-09 08:41:47 +01:00
|
|
|
parser = argparse.ArgumentParser(USAGE)
|
2021-02-12 08:20:45 +01:00
|
|
|
parser.add_argument("--coverage", action="store_true", help="Get coverage report")
|
2021-03-03 05:00:15 +01:00
|
|
|
add_provision_check_override_param(parser)
|
2021-02-12 08:20:45 +01:00
|
|
|
parser.add_argument("args", nargs=argparse.REMAINDER)
|
2021-08-04 18:19:54 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"--parallel",
|
|
|
|
dest="parallel",
|
|
|
|
action="store",
|
|
|
|
type=int,
|
|
|
|
# Since process startup time is a significant portion of total
|
|
|
|
# runtime, so rather than doing os.cpu_count, we just do a fixed 4
|
|
|
|
# processes by default.
|
|
|
|
default=4,
|
|
|
|
help="Specify the number of processes to run the "
|
|
|
|
"tests in. Default is the number of logical CPUs",
|
|
|
|
)
|
2017-11-09 08:41:47 +01:00
|
|
|
options = parser.parse_args()
|
2020-01-26 15:57:47 +01:00
|
|
|
individual_files = options.args
|
2021-08-04 18:19:54 +02:00
|
|
|
parallel = options.parallel
|
|
|
|
|
|
|
|
if options.coverage and parallel > 1:
|
|
|
|
parallel = 1
|
|
|
|
print(
|
|
|
|
BOLDRED + "You cannot use --coverage with parallel tests. Running in serial mode.\n" + ENDC
|
|
|
|
)
|
2017-04-05 17:47:13 +02:00
|
|
|
|
2021-03-02 20:59:19 +01:00
|
|
|
assert_provisioning_status_ok(options.skip_provision_check)
|
2017-04-05 17:47:13 +02:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-04-11 12:24:10 +02:00
|
|
|
def get_dev_host() -> str:
|
|
|
|
# See similar code in dev_settings.py. We only use
|
|
|
|
# this to report where you can find coverage reports.
|
|
|
|
# We duplicate the code here to avoid depending on
|
|
|
|
# Django.
|
|
|
|
|
2021-02-12 08:20:45 +01:00
|
|
|
host = os.getenv("EXTERNAL_HOST")
|
2020-04-11 12:24:10 +02:00
|
|
|
if host is not None:
|
|
|
|
return host
|
|
|
|
|
|
|
|
user_id = os.getuid()
|
|
|
|
user_name = pwd.getpwuid(user_id).pw_name
|
|
|
|
if user_name == "zulipdev":
|
2020-04-24 11:54:55 +02:00
|
|
|
hostname = os.uname()[1].lower()
|
2021-02-12 08:20:45 +01:00
|
|
|
if ".zulipdev.org" not in hostname:
|
|
|
|
hostname += ".zulipdev.org"
|
|
|
|
return hostname + ":9991"
|
2020-04-11 12:24:10 +02:00
|
|
|
else:
|
|
|
|
# For local development environments, we use localhost by
|
|
|
|
# default, via the "zulipdev.com" hostname.
|
2021-02-12 08:20:45 +01:00
|
|
|
return "zulipdev.com:9991"
|
2020-04-11 12:24:10 +02:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-02-09 15:09:37 +01:00
|
|
|
def print_error(msg: str) -> None:
|
2021-02-12 08:20:45 +01:00
|
|
|
print(BOLDRED + "ERROR:" + ENDC + " " + msg)
|
2020-02-09 15:09:37 +01:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-09-02 21:58:08 +02:00
|
|
|
def clean_file(orig_fn: str) -> str:
|
|
|
|
fn = orig_fn
|
2023-02-22 23:04:10 +01:00
|
|
|
if not fn.endswith(".test.js"):
|
|
|
|
fn += ".test.js"
|
|
|
|
if "web/tests/" not in fn:
|
|
|
|
fn = os.path.join(ROOT_DIR, "web", "tests", fn)
|
2020-09-02 21:58:08 +02:00
|
|
|
fn = os.path.abspath(fn)
|
|
|
|
if not os.path.exists(fn):
|
|
|
|
print(f"Cannot find {orig_fn} ({fn})")
|
|
|
|
sys.exit(1)
|
|
|
|
return fn
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-09-02 21:58:08 +02:00
|
|
|
def clean_files(fns: List[str]) -> List[str]:
|
|
|
|
cleaned_files = [clean_file(fn) for fn in fns]
|
|
|
|
return cleaned_files
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-01-26 15:57:47 +01:00
|
|
|
def run_tests_via_node_js() -> int:
|
2021-02-12 08:20:45 +01:00
|
|
|
os.environ["TZ"] = "UTC"
|
2020-01-26 15:57:47 +01:00
|
|
|
|
|
|
|
# The index.js test runner is the real "driver" here, and we launch
|
|
|
|
# with either nyc or node, depending on whether we want coverage
|
|
|
|
# reports. Running under nyc is slower and creates funny
|
|
|
|
# tracebacks, so you generally want to get coverage reports only
|
|
|
|
# after making sure tests will pass.
|
2021-02-12 08:20:45 +01:00
|
|
|
node_tests_cmd = ["node", "--stack-trace-limit=100", INDEX_JS]
|
2020-09-02 21:58:08 +02:00
|
|
|
if individual_files:
|
2021-08-04 18:19:54 +02:00
|
|
|
# If we passed a specific set of tests, run in serial mode.
|
|
|
|
global parallel
|
|
|
|
parallel = 1
|
2020-09-02 21:58:08 +02:00
|
|
|
files = individual_files
|
|
|
|
else:
|
2023-02-22 23:04:10 +01:00
|
|
|
files = sorted(glob.glob(os.path.join(ROOT_DIR, "web/tests/*.test.js")))
|
2020-09-02 21:58:08 +02:00
|
|
|
|
2021-08-04 18:19:54 +02:00
|
|
|
test_files = clean_files(files)
|
|
|
|
|
|
|
|
print("Starting node tests...")
|
|
|
|
|
|
|
|
# If we got this far, we can run the tests!
|
|
|
|
ret = 0
|
|
|
|
if parallel > 1:
|
|
|
|
sub_tests = [test_files[i::parallel] for i in range(parallel)]
|
|
|
|
parallel_processes = [subprocess.Popen(node_tests_cmd + sub_test) for sub_test in sub_tests]
|
|
|
|
|
|
|
|
for process in parallel_processes:
|
|
|
|
status_code = process.wait()
|
|
|
|
if status_code != 0:
|
|
|
|
ret = status_code
|
|
|
|
return ret
|
|
|
|
|
|
|
|
node_tests_cmd += test_files
|
2020-01-26 15:57:47 +01:00
|
|
|
if options.coverage:
|
2022-01-07 20:40:15 +01:00
|
|
|
os.environ["USING_INSTRUMENTED_CODE"] = "TRUE"
|
2021-02-12 08:20:45 +01:00
|
|
|
coverage_dir = os.path.join(ROOT_DIR, "var/node-coverage")
|
2020-01-26 15:57:47 +01:00
|
|
|
|
2020-09-02 22:45:59 +02:00
|
|
|
nyc = os.path.join(ROOT_DIR, "node_modules/.bin/nyc")
|
|
|
|
command = [nyc, "--extension", ".hbs", "--extension", ".ts"]
|
|
|
|
command += ["--report-dir", coverage_dir]
|
|
|
|
command += ["--temp-directory", coverage_dir]
|
|
|
|
command += ["-r=lcov", "-r=json", "-r=text-summary"]
|
2020-01-26 15:57:47 +01:00
|
|
|
command += node_tests_cmd
|
|
|
|
else:
|
|
|
|
# Normal testing, no coverage analysis.
|
|
|
|
# Run the index.js test runner, which runs all the other tests.
|
|
|
|
command = node_tests_cmd
|
|
|
|
|
|
|
|
try:
|
|
|
|
ret = subprocess.check_call(command)
|
|
|
|
except OSError:
|
2021-02-12 08:20:45 +01:00
|
|
|
print(f"Bad command: {command}")
|
2020-01-26 15:57:47 +01:00
|
|
|
raise
|
|
|
|
except subprocess.CalledProcessError:
|
2021-02-12 08:20:45 +01:00
|
|
|
print("\n** Tests failed, PLEASE FIX! **\n")
|
2020-01-26 15:57:47 +01:00
|
|
|
sys.exit(1)
|
|
|
|
return ret
|
2017-04-05 17:47:13 +02:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
|
|
|
def check_line_coverage(
|
|
|
|
fn: str, line_coverage: Dict[Any, Any], line_mapping: Dict[Any, Any], log: bool = True
|
|
|
|
) -> bool:
|
2017-06-23 11:53:25 +02:00
|
|
|
missing_lines = []
|
|
|
|
for line in line_coverage:
|
|
|
|
if line_coverage[line] == 0:
|
|
|
|
actual_line = line_mapping[line]
|
|
|
|
missing_lines.append(str(actual_line["start"]["line"]))
|
|
|
|
if missing_lines:
|
|
|
|
if log:
|
2020-06-10 06:41:04 +02:00
|
|
|
print_error(f"{fn} no longer has complete node test coverage")
|
|
|
|
print(" Lines missing coverage: {}".format(", ".join(sorted(missing_lines, key=int))))
|
2017-06-23 11:53:25 +02:00
|
|
|
print()
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2018-04-16 15:46:40 +02:00
|
|
|
def read_coverage() -> Any:
|
2017-06-07 04:24:39 +02:00
|
|
|
coverage_json = None
|
|
|
|
try:
|
2020-08-07 01:09:47 +02:00
|
|
|
with open(NODE_COVERAGE_PATH, "rb") as f:
|
|
|
|
coverage_json = orjson.loads(f.read())
|
2020-04-09 21:51:58 +02:00
|
|
|
except OSError:
|
2017-06-07 04:24:39 +02:00
|
|
|
print(NODE_COVERAGE_PATH + " doesn't exist. Cannot enforce fully covered files.")
|
|
|
|
raise
|
2018-04-16 15:46:40 +02:00
|
|
|
return coverage_json
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2018-04-16 15:46:40 +02:00
|
|
|
def enforce_proper_coverage(coverage_json: Any) -> bool:
|
2022-04-09 23:44:38 +02:00
|
|
|
all_js_files = {
|
2023-02-22 23:03:47 +01:00
|
|
|
*glob.glob("web/shared/src/*.js"),
|
|
|
|
*glob.glob("web/shared/src/*.ts"),
|
|
|
|
*glob.glob("web/src/*.js"),
|
|
|
|
*glob.glob("web/src/*.ts"),
|
|
|
|
*glob.glob("web/src/billing/*.js"),
|
2023-02-22 23:04:10 +01:00
|
|
|
*glob.glob("web/tests/*.js"),
|
|
|
|
*glob.glob("web/tests/lib/*.js"),
|
2022-04-09 23:44:38 +02:00
|
|
|
}
|
|
|
|
enforce_fully_covered = sorted(all_js_files - EXEMPT_FILES)
|
2020-02-09 14:47:22 +01:00
|
|
|
|
2018-04-16 15:46:40 +02:00
|
|
|
coverage_lost = False
|
2017-06-07 04:24:39 +02:00
|
|
|
for relative_path in enforce_fully_covered:
|
|
|
|
path = ROOT_DIR + "/" + relative_path
|
2022-09-26 01:56:42 +02:00
|
|
|
if path not in coverage_json:
|
2020-02-09 15:11:49 +01:00
|
|
|
coverage_lost = True
|
2020-06-10 06:41:04 +02:00
|
|
|
print_error(f"{relative_path} has no node test coverage")
|
2017-06-07 04:24:39 +02:00
|
|
|
continue
|
2021-02-12 08:20:45 +01:00
|
|
|
line_coverage = coverage_json[path]["s"]
|
|
|
|
line_mapping = coverage_json[path]["statementMap"]
|
2018-04-16 15:46:40 +02:00
|
|
|
if not check_line_coverage(relative_path, line_coverage, line_mapping):
|
|
|
|
coverage_lost = True
|
|
|
|
if coverage_lost:
|
2018-03-25 08:53:40 +02:00
|
|
|
print()
|
2017-06-07 04:24:39 +02:00
|
|
|
print("It looks like your changes lost 100% test coverage in one or more files.")
|
2020-02-09 14:47:22 +01:00
|
|
|
print("Ideally, you should add some tests to restore coverage.")
|
|
|
|
print("A worse option is to update EXEMPT_FILES in `tools/test-js-with-node`.")
|
2017-06-07 04:24:39 +02:00
|
|
|
print("To run this check locally, use `test-js-with-node --coverage`.")
|
2020-02-09 14:47:22 +01:00
|
|
|
print()
|
2017-06-07 04:24:39 +02:00
|
|
|
|
2018-04-16 15:46:40 +02:00
|
|
|
coverage_not_enforced = False
|
2017-06-23 12:04:23 +02:00
|
|
|
for path in coverage_json:
|
2020-02-09 14:47:22 +01:00
|
|
|
relative_path = os.path.relpath(path, ROOT_DIR)
|
|
|
|
if relative_path in EXEMPT_FILES:
|
2021-02-12 08:20:45 +01:00
|
|
|
line_coverage = coverage_json[path]["s"]
|
|
|
|
line_mapping = coverage_json[path]["statementMap"]
|
2020-02-09 14:47:22 +01:00
|
|
|
if check_line_coverage(relative_path, line_coverage, line_mapping, log=False):
|
2018-04-16 15:46:40 +02:00
|
|
|
coverage_not_enforced = True
|
2020-06-09 00:25:09 +02:00
|
|
|
print_error(f"{relative_path} unexpectedly has 100% line coverage.")
|
2018-04-16 15:46:40 +02:00
|
|
|
|
|
|
|
if coverage_not_enforced:
|
2018-03-25 08:53:40 +02:00
|
|
|
print()
|
2020-02-09 14:47:22 +01:00
|
|
|
print("One or more fully covered files are miscategorized.")
|
|
|
|
print("Remove the file(s) from EXEMPT_FILES in `tools/test-js-with-node`.")
|
2018-04-16 15:46:40 +02:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
problems_encountered = coverage_lost or coverage_not_enforced
|
2018-04-16 15:46:40 +02:00
|
|
|
return problems_encountered
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
2020-01-26 15:57:47 +01:00
|
|
|
ret = run_tests_via_node_js()
|
|
|
|
|
2018-04-16 15:46:40 +02:00
|
|
|
if options.coverage and ret == 0:
|
|
|
|
if not individual_files:
|
|
|
|
coverage_json = read_coverage()
|
|
|
|
problems_encountered = enforce_proper_coverage(coverage_json)
|
|
|
|
if problems_encountered:
|
|
|
|
ret = 1
|
2017-06-23 12:04:23 +02:00
|
|
|
|
2020-09-03 13:20:34 +02:00
|
|
|
reports_location = f"http://{get_dev_host()}/node-coverage/index.html"
|
|
|
|
print()
|
|
|
|
print("View coverage reports at " + CYAN + reports_location + ENDC)
|
|
|
|
|
2018-03-25 08:53:40 +02:00
|
|
|
print()
|
2017-04-05 17:47:13 +02:00
|
|
|
if ret == 0:
|
2020-02-09 15:09:37 +01:00
|
|
|
print(GREEN + "Test(s) passed. SUCCESS!" + ENDC)
|
2017-04-05 17:47:13 +02:00
|
|
|
else:
|
2020-02-09 15:09:37 +01:00
|
|
|
print(BOLDRED + "FAIL - Test(s) failed" + ENDC)
|
2017-04-05 17:47:13 +02:00
|
|
|
|
|
|
|
sys.exit(ret)
|