zulip/frontend_tests/run-casper

#!/usr/bin/env python3
import argparse
import subprocess
import sys
import os
import glob
import shlex

#
# In order to use remote casperjs debugging, pass the --remote-debug flag
# This will start a remote debugging session listening on port 7777
#
# See https://zulip.readthedocs.io/en/latest/testing/testing-with-casper.html
# for more information on how to use remote debugging
#

ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

os.environ["CASPER_TESTS"] = "1"
os.environ["PHANTOMJS_EXECUTABLE"] = os.path.join(ZULIP_PATH, "node_modules/.bin/phantomjs")
os.environ.pop("http_proxy", "")
os.environ.pop("https_proxy", "")

usage = """

    test-js-with-casper # Run all test files
    test-js-with-casper 09-navigation.js # Run a single test file
    test-js-with-casper 09 # Run a single test file 09-navigation.js
    test-js-with-casper 01-login.js 03-narrow.js # Run a few test files
    test-js-with-casper 01 03 # Run a few test files, 01-login.js and 03-narrow.js here

    Using loops:

        test-js-with-capser --loop 5 07 # run 5 loops of test 07
---
"""
parser = argparse.ArgumentParser(usage=usage)

parser.add_argument('--skip-flaky-tests', dest='skip_flaky',
                    action="store_true",
                    default=False, help='Skip flaky tests')
parser.add_argument('--loop', dest='loop', nargs=1,
                    action="store", type=int,
                    default=None, help='Run tests in a loop.')
parser.add_argument('--force', dest='force',
                    action="store_true",
                    default=False, help='Run tests despite possible problems.')
parser.add_argument('--verbose',
                    help='Whether or not to enable verbose mode',
                    action="store_true",
                    default=False)
parser.add_argument('--remote-debug',
                    help='Whether or not to enable remote debugging on port 7777',
                    action="store_true",
                    default=False)
parser.add_argument('--xunit-export', dest='xunit_export',
                    action="store_true",
                    default=False, help='Export the results of the test suite to an XUnit XML file,')
parser.add_argument('tests', nargs=argparse.REMAINDER,
                    help='Specific tests to run; by default, runs all tests')
options = parser.parse_args()

sys.path.insert(0, ZULIP_PATH)

# check for the venv
from tools.lib import sanity_check
sanity_check.check_venv(__file__)

from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files
from tools.lib.test_server import test_server_running

from typing import Iterable, List

assert_provisioning_status_ok(options.force)

os.chdir(ZULIP_PATH)

subprocess.check_call(['node', 'node_modules/phantomjs-prebuilt/install.js'])

os.makedirs('var/casper', exist_ok=True)

for f in glob.glob('var/casper/casper-failure*.png'):
    os.remove(f)

def reset_database() -> None:
    from zerver.lib.test_helpers import reset_emails_in_zulip_realm
    reset_emails_in_zulip_realm()

def run_tests(files: Iterable[str], external_host: str) -> None:
    test_dir = os.path.join(ZULIP_PATH, 'frontend_tests/casper_tests')
    test_files = find_js_test_files(test_dir, files)

    # 10-admin.js is too flaky!
    if options.skip_flaky:
        test_files = [fn for fn in test_files if '10-admin' not in fn]

    if options.loop:
        loop_cnt = options.loop[0]
        print('\n\nWe will use loop mode for these tests:\n')
        for test_file in test_files:
            print('    ' + os.path.basename(test_file))
        print(f'\nnumber of loops: {loop_cnt}\n')
        print()
    else:
        loop_cnt = None

    remote_debug: List[str] = []
    if options.remote_debug:
        remote_debug = ["--remote-debugger-port=7777", "--remote-debugger-autorun=yes"]

    verbose: List[str] = []
    if options.verbose:
        verbose = ["--verbose", "--log-level=debug"]

    xunit_export: List[str] = []
    if options.xunit_export:
        xunit_export = ["--xunit=var/xunit-test-results/casper/result.xml"]

    def run_tests() -> int:
        ret = 1
        for test_file in test_files:
            test_name = os.path.basename(test_file)
            cmd = ["node_modules/.bin/casperjs"] + remote_debug + verbose + xunit_export + ["test", test_file]
            print("\n\n===================== {}\nRunning {}\n\n".format(test_name, " ".join(map(shlex.quote, cmd))), flush=True)
            ret = subprocess.call(cmd)
            if ret != 0:
                return ret
        return 0

    def run_loops(loop_cnt: int) -> None:
        while True:
            for trial in range(1, loop_cnt + 1):
                print(f'\n\n\nSTARTING TRIAL {trial} / {loop_cnt}\n')
                ret = run_tests()
                if ret == 0:
                    print(f'`\n\nSUCCESS! trial #{trial}\n\n')
                else:
                    print(f'\n\nFAIL! trial #{trial}\n')
                    break

            while True:
                response = input('Press "q" to quit or enter number of loops: ')
                if response == 'q':
                    return
                try:
                    loop_cnt = int(response)
                    break
                except ValueError:
                    continue

    with test_server_running(options.force, external_host):
        # Important: do next things inside the `with` block, when Django
        #            will be pointing at the test database.
        reset_database()
        subprocess.check_call('tools/setup/generate-test-credentials')

        # RUN THE TESTS!!!
        if loop_cnt:
            run_loops(loop_cnt)
            ret = 0
        else:
            ret = run_tests()

    if ret != 0:
        print("""
The Casper frontend tests failed!  For help debugging, read:
  https://zulip.readthedocs.io/en/latest/testing/testing-with-casper.html""", file=sys.stderr)
        if os.environ.get("CIRCLECI"):
            print("", file=sys.stderr)
            print("In CircleCI, the Artifacts tab contains screenshots of the failure.", file=sys.stderr)
            print("", file=sys.stderr)

        sys.exit(ret)

external_host = "zulipdev.com:9981"
run_tests(options.tests, external_host)
sys.exit(0)