#!/usr/bin/env python3 import argparse import glob import os import pwd import subprocess import sys from typing import Any, Dict, List, Set TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) ROOT_DIR = os.path.dirname(TOOLS_DIR) # check for the venv from tools.lib import sanity_check sanity_check.check_venv(__file__) # Import this after we do the sanity_check so it doesn't crash. import orjson from zulint.printer import BOLDRED, CYAN, ENDC, GREEN INDEX_JS = os.path.join(ROOT_DIR, "web/tests/lib/index.js") NODE_COVERAGE_PATH = os.path.join(ROOT_DIR, "var/node-coverage/coverage-final.json") # Ideally, we wouldn't need this line, but it seems to be required to # avoid problems finding node_modules when running `cd tools; ./test-js-with-node`. os.chdir(ROOT_DIR) USAGE = """ tools/test-js-with-node - to run all tests tools/test-js-with-node util.test.js activity.test.js - to run just a couple tests tools/test-js-with-node --coverage - to generate coverage report """ def make_set(files: List[str]) -> Set[str]: for i in range(1, len(files)): if files[i - 1] > files[i]: raise Exception(f"Please move {files[i]} so that names are sorted.") return set(files) # We do not yet require 100% line coverage for these files: EXEMPT_FILES = make_set( [ "web/shared/src/poll_data.js", "web/src/about_zulip.js", "web/src/add_subscribers_pill.js", "web/src/admin.js", "web/src/alert_popup.ts", "web/src/alert_words_ui.js", "web/src/archive.js", "web/src/assets.d.ts", "web/src/attachments_ui.js", "web/src/avatar.js", "web/src/billing/event_status.js", "web/src/billing/helpers.js", "web/src/billing/upgrade.js", "web/src/blueslip.ts", "web/src/blueslip_stacktrace.ts", "web/src/click_handlers.js", "web/src/compose.js", "web/src/compose_actions.js", "web/src/compose_banner.ts", "web/src/compose_closed_ui.js", "web/src/compose_fade.js", "web/src/compose_state.js", "web/src/compose_ui.js", "web/src/compose_validate.js", "web/src/composebox_typeahead.js", "web/src/condense.js", "web/src/confirm_dialog.js", "web/src/copy_and_paste.js", "web/src/csrf.ts", "web/src/css_variables.ts", "web/src/dark_theme.js", "web/src/debug.js", "web/src/deprecated_feature_notice.js", "web/src/desktop_integration.js", "web/src/dialog_widget.js", "web/src/drafts.js", "web/src/dropdown_list_widget.js", "web/src/echo.js", "web/src/emoji_picker.js", "web/src/emojisets.js", "web/src/favicon.js", "web/src/feature_flags.ts", "web/src/feedback_widget.js", "web/src/flatpickr.js", "web/src/gear_menu.js", "web/src/giphy.js", "web/src/global.d.ts", "web/src/hash_util.js", "web/src/hashchange.js", "web/src/hbs.d.ts", "web/src/hotkey.js", "web/src/hotspots.js", "web/src/info_overlay.js", "web/src/invite.js", "web/src/lightbox.js", "web/src/list_util.ts", "web/src/list_widget.js", "web/src/loading.ts", "web/src/local_message.js", "web/src/localstorage.js", "web/src/message_edit.js", "web/src/message_edit_history.js", "web/src/message_events.js", "web/src/message_fetch.js", "web/src/message_list.js", "web/src/message_list_data.js", "web/src/message_list_view.js", "web/src/message_lists.js", "web/src/message_live_update.js", "web/src/message_scroll.js", "web/src/message_util.js", "web/src/message_view_header.js", "web/src/message_viewport.js", "web/src/muted_topics_ui.js", "web/src/muted_users_ui.js", "web/src/narrow.js", "web/src/navbar_alerts.js", "web/src/navigate.js", "web/src/notifications.js", "web/src/overlays.js", "web/src/padded_widget.ts", "web/src/page_params.ts", "web/src/pm_list.js", "web/src/pm_list_dom.js", "web/src/poll_widget.js", "web/src/popover_menus.js", "web/src/popover_menus_data.js", "web/src/popovers.js", "web/src/read_receipts.js", "web/src/ready.ts", "web/src/realm_icon.js", "web/src/realm_logo.js", "web/src/realm_playground.js", "web/src/realm_user_settings_defaults.ts", "web/src/recent_topics_ui.js", "web/src/recent_topics_util.js", "web/src/reload.js", "web/src/reminder.js", "web/src/resize.js", "web/src/rows.js", "web/src/scroll_bar.js", "web/src/search_pill_widget.js", "web/src/sent_messages.js", "web/src/sentry.ts", "web/src/server_events.js", "web/src/settings.js", "web/src/settings_account.js", "web/src/settings_bots.js", "web/src/settings_display.js", "web/src/settings_emoji.js", "web/src/settings_exports.js", "web/src/settings_invites.js", "web/src/settings_linkifiers.js", "web/src/settings_muted_topics.js", "web/src/settings_muted_users.js", "web/src/settings_notifications.js", "web/src/settings_org.js", "web/src/settings_panel_menu.js", "web/src/settings_playgrounds.js", "web/src/settings_profile_fields.js", "web/src/settings_realm_domains.js", "web/src/settings_realm_user_settings_defaults.js", "web/src/settings_sections.js", "web/src/settings_streams.js", "web/src/settings_toggle.js", "web/src/settings_ui.js", "web/src/settings_user_groups_legacy.js", "web/src/settings_users.js", "web/src/setup.js", "web/src/spectators.js", "web/src/spoilers.ts", "web/src/starred_messages_ui.js", "web/src/stream_bar.js", "web/src/stream_color.js", "web/src/stream_create.js", "web/src/stream_create_subscribers.js", "web/src/stream_edit.js", "web/src/stream_edit_subscribers.js", "web/src/stream_list.js", "web/src/stream_muting.js", "web/src/stream_popover.js", "web/src/stream_settings_containers.js", "web/src/stream_settings_ui.js", "web/src/stream_ui_updates.js", "web/src/submessage.js", "web/src/subscriber_api.js", "web/src/timerender.ts", "web/src/tippyjs.js", "web/src/todo_widget.js", "web/src/topic_list.js", "web/src/topic_zoom.js", "web/src/tutorial.js", "web/src/types.ts", "web/src/typing.js", "web/src/typing_events.js", "web/src/ui.js", "web/src/ui_init.js", "web/src/ui_report.ts", "web/src/ui_util.ts", "web/src/unread.js", "web/src/unread_ops.js", "web/src/unread_ui.js", "web/src/upload.js", "web/src/upload_widget.ts", "web/src/user_group_create.js", "web/src/user_group_create_members.js", "web/src/user_group_create_members_data.js", "web/src/user_group_edit.js", "web/src/user_group_edit_members.js", "web/src/user_group_ui_updates.js", "web/src/user_groups_settings_ui.js", "web/src/user_profile.js", "web/src/user_settings.ts", "web/src/user_status.js", "web/src/user_status_ui.js", "web/src/webpack_public_path.js", "web/src/zcommand.js", "web/src/zform.js", "web/src/zulip.js", "web/src/zulip_test.js", "web/tests/lib/mdiff.js", "web/tests/lib/zjquery_element.js", ] ) from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok parser = argparse.ArgumentParser(USAGE) parser.add_argument("--coverage", action="store_true", help="Get coverage report") add_provision_check_override_param(parser) parser.add_argument("args", nargs=argparse.REMAINDER) parser.add_argument( "--parallel", dest="parallel", action="store", type=int, # Since process startup time is a significant portion of total # runtime, so rather than doing os.cpu_count, we just do a fixed 4 # processes by default. default=4, help="Specify the number of processes to run the " "tests in. Default is the number of logical CPUs", ) options = parser.parse_args() individual_files = options.args parallel = options.parallel if options.coverage and parallel > 1: parallel = 1 print( BOLDRED + "You cannot use --coverage with parallel tests. Running in serial mode.\n" + ENDC ) assert_provisioning_status_ok(options.skip_provision_check) def get_dev_host() -> str: # See similar code in dev_settings.py. We only use # this to report where you can find coverage reports. # We duplicate the code here to avoid depending on # Django. host = os.getenv("EXTERNAL_HOST") if host is not None: return host user_id = os.getuid() user_name = pwd.getpwuid(user_id).pw_name if user_name == "zulipdev": hostname = os.uname()[1].lower() if ".zulipdev.org" not in hostname: hostname += ".zulipdev.org" return hostname + ":9991" else: # For local development environments, we use localhost by # default, via the "zulipdev.com" hostname. return "zulipdev.com:9991" def print_error(msg: str) -> None: print(BOLDRED + "ERROR:" + ENDC + " " + msg) def clean_file(orig_fn: str) -> str: fn = orig_fn if not fn.endswith(".test.js"): fn += ".test.js" if "web/tests/" not in fn: fn = os.path.join(ROOT_DIR, "web", "tests", fn) fn = os.path.abspath(fn) if not os.path.exists(fn): print(f"Cannot find {orig_fn} ({fn})") sys.exit(1) return fn def clean_files(fns: List[str]) -> List[str]: cleaned_files = [clean_file(fn) for fn in fns] return cleaned_files def run_tests_via_node_js() -> int: os.environ["TZ"] = "UTC" # The index.js test runner is the real "driver" here, and we launch # with either nyc or node, depending on whether we want coverage # reports. Running under nyc is slower and creates funny # tracebacks, so you generally want to get coverage reports only # after making sure tests will pass. node_tests_cmd = ["node", "--stack-trace-limit=100", INDEX_JS] if individual_files: # If we passed a specific set of tests, run in serial mode. global parallel parallel = 1 files = individual_files else: files = sorted(glob.glob(os.path.join(ROOT_DIR, "web/tests/*.test.js"))) test_files = clean_files(files) print("Starting node tests...") # If we got this far, we can run the tests! ret = 0 if parallel > 1: sub_tests = [test_files[i::parallel] for i in range(parallel)] parallel_processes = [subprocess.Popen(node_tests_cmd + sub_test) for sub_test in sub_tests] for process in parallel_processes: status_code = process.wait() if status_code != 0: ret = status_code return ret node_tests_cmd += test_files if options.coverage: os.environ["USING_INSTRUMENTED_CODE"] = "TRUE" coverage_dir = os.path.join(ROOT_DIR, "var/node-coverage") nyc = os.path.join(ROOT_DIR, "node_modules/.bin/nyc") command = [nyc, "--extension", ".hbs", "--extension", ".ts"] command += ["--report-dir", coverage_dir] command += ["--temp-directory", coverage_dir] command += ["-r=lcov", "-r=json", "-r=text-summary"] command += node_tests_cmd else: # Normal testing, no coverage analysis. # Run the index.js test runner, which runs all the other tests. command = node_tests_cmd try: ret = subprocess.check_call(command) except OSError: print(f"Bad command: {command}") raise except subprocess.CalledProcessError: print("\n** Tests failed, PLEASE FIX! **\n") sys.exit(1) return ret def check_line_coverage( fn: str, line_coverage: Dict[Any, Any], line_mapping: Dict[Any, Any], log: bool = True ) -> bool: missing_lines = [] for line in line_coverage: if line_coverage[line] == 0: actual_line = line_mapping[line] missing_lines.append(str(actual_line["start"]["line"])) if missing_lines: if log: print_error(f"{fn} no longer has complete node test coverage") print(" Lines missing coverage: {}".format(", ".join(sorted(missing_lines, key=int)))) print() return False return True def read_coverage() -> Any: coverage_json = None try: with open(NODE_COVERAGE_PATH, "rb") as f: coverage_json = orjson.loads(f.read()) except OSError: print(NODE_COVERAGE_PATH + " doesn't exist. Cannot enforce fully covered files.") raise return coverage_json def enforce_proper_coverage(coverage_json: Any) -> bool: all_js_files = { *glob.glob("web/shared/src/*.js"), *glob.glob("web/shared/src/*.ts"), *glob.glob("web/src/*.js"), *glob.glob("web/src/*.ts"), *glob.glob("web/src/billing/*.js"), *glob.glob("web/tests/*.js"), *glob.glob("web/tests/lib/*.js"), } enforce_fully_covered = sorted(all_js_files - EXEMPT_FILES) coverage_lost = False for relative_path in enforce_fully_covered: path = ROOT_DIR + "/" + relative_path if path not in coverage_json: coverage_lost = True print_error(f"{relative_path} has no node test coverage") continue line_coverage = coverage_json[path]["s"] line_mapping = coverage_json[path]["statementMap"] if not check_line_coverage(relative_path, line_coverage, line_mapping): coverage_lost = True if coverage_lost: print() print("It looks like your changes lost 100% test coverage in one or more files.") print("Ideally, you should add some tests to restore coverage.") print("A worse option is to update EXEMPT_FILES in `tools/test-js-with-node`.") print("To run this check locally, use `test-js-with-node --coverage`.") print() coverage_not_enforced = False for path in coverage_json: relative_path = os.path.relpath(path, ROOT_DIR) if relative_path in EXEMPT_FILES: line_coverage = coverage_json[path]["s"] line_mapping = coverage_json[path]["statementMap"] if check_line_coverage(relative_path, line_coverage, line_mapping, log=False): coverage_not_enforced = True print_error(f"{relative_path} unexpectedly has 100% line coverage.") if coverage_not_enforced: print() print("One or more fully covered files are miscategorized.") print("Remove the file(s) from EXEMPT_FILES in `tools/test-js-with-node`.") problems_encountered = coverage_lost or coverage_not_enforced return problems_encountered ret = run_tests_via_node_js() if options.coverage and ret == 0: if not individual_files: coverage_json = read_coverage() problems_encountered = enforce_proper_coverage(coverage_json) if problems_encountered: ret = 1 reports_location = f"http://{get_dev_host()}/node-coverage/index.html" print() print("View coverage reports at " + CYAN + reports_location + ENDC) print() if ret == 0: print(GREEN + "Test(s) passed. SUCCESS!" + ENDC) else: print(BOLDRED + "FAIL - Test(s) failed" + ENDC) sys.exit(ret)