zulip/tools/test-backend

#!/usr/bin/env python3

# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)

from typing import List, Iterator
import glob
import argparse
import contextlib
import mock
import os
import shlex
import sys
import subprocess
import tempfile
import ujson
import responses

import django
from django.conf import settings
from django.test.utils import get_runner

target_fully_covered = {path for target in [
    'analytics/lib/*.py',
    'analytics/models.py',
    'analytics/tests/*.py',
    'analytics/views.py',
    # zerver/ and zerver/lib/ are important core files
    'zerver/*.py',
    'zerver/lib/*.py',
    'zerver/lib/*/*.py',
    'zerver/lib/*/*/*.py',
    'zerver/data_import/*.py',
    'zerver/templatetags/*.py',
    'zerver/tornado/*.py',
    # Billing files require 100% test coverage
    'corporate/lib/stripe.py',
    'corporate/views.py',
    # Test files should have 100% coverage; test code that isn't run
    # is likely a bug in the test.
    'zerver/tests/*.py',
    'corporate/tests/*.py',
    # As a project, we require 100% test coverage in the views files.
    'zerver/views/*.py',
    'zproject/backends.py',
    'confirmation/*.py',
    'zerver/webhooks/*/*.py',
    # Once we have a nice negative tests system, we can add these:
    # 'zerver/webhooks/*/*.py',
    # 'zerver/webhooks/*/*/*.py',
    'zerver/worker/*.py',
] for path in glob.glob(target)}

not_yet_fully_covered = {path for target in [
    # Analytics fixtures library is used to generate test fixtures;
    # isn't properly accounted for in test coverage analysis since it
    # runs before tests.
    'analytics/lib/fixtures.py',
    # We have 100% coverage on the new stuff; need to refactor old stuff.
    'analytics/views.py',
    # Major lib files should have 100% coverage
    'zerver/lib/addressee.py',
    'zerver/lib/bugdown/__init__.py',
    'zerver/lib/cache.py',
    'zerver/lib/cache_helpers.py',
    'zerver/lib/i18n.py',
    'zerver/lib/email_notifications.py',
    'zerver/lib/send_email.py',
    'zerver/lib/url_preview/preview.py',
    'zerver/worker/queue_processors.py',
    # Bugdown sub-libs should have full coverage too; a lot are really close
    'zerver/lib/bugdown/api_arguments_table_generator.py',
    'zerver/lib/bugdown/fenced_code.py',
    'zerver/lib/bugdown/help_relative_links.py',
    'zerver/lib/bugdown/nested_code_blocks.py',
    # Other lib files that ideally would coverage, but aren't sorted
    'zerver/__init__.py',
    'zerver/filters.py',
    'zerver/middleware.py',
    'zerver/lib/bot_lib.py',
    'zerver/lib/camo.py',
    'zerver/lib/debug.py',
    'zerver/lib/digest.py',
    'zerver/lib/error_notify.py',
    'zerver/lib/export.py',
    'zerver/lib/fix_unreads.py',
    'zerver/lib/html_diff.py',
    'zerver/lib/import_realm.py',
    'zerver/lib/logging_util.py',
    'zerver/lib/migrate.py',
    'zerver/lib/outgoing_webhook.py',
    'zerver/lib/parallel.py',
    'zerver/lib/profile.py',
    'zerver/lib/queue.py',
    'zerver/lib/sqlalchemy_utils.py',
    'zerver/lib/storage.py',
    'zerver/lib/timeout.py',
    'zerver/lib/unminify.py',
    'zerver/lib/utils.py',
    'zerver/lib/zephyr.py',
    'zerver/templatetags/app_filters.py',
    'zerver/templatetags/minified_js.py',
    # Low priority for coverage
    'zerver/lib/ccache.py',
    'zerver/lib/generate_test_data.py',
    'zerver/lib/server_initialization.py',
    'zerver/lib/test_fixtures.py',
    'zerver/lib/test_runner.py',
    'zerver/openapi/python_examples.py',
    # Tornado should ideally have full coverage, but we're not there.
    'zerver/tornado/autoreload.py',
    'zerver/tornado/descriptors.py',
    'zerver/tornado/event_queue.py',
    'zerver/tornado/exceptions.py',
    'zerver/tornado/handlers.py',
    'zerver/tornado/ioloop_logging.py',
    'zerver/tornado/sharding.py',
    'zerver/tornado/views.py',
    # Data import files; relatively low priority
    'zerver/data_import/hipchat*.py',
    'zerver/data_import/sequencer.py',
    'zerver/data_import/slack.py',
    'zerver/data_import/gitter.py',
    'zerver/data_import/import_util.py',
    # Webhook integrations with incomplete coverage
    'zerver/webhooks/greenhouse/view.py',
    'zerver/webhooks/jira/view.py',
    'zerver/webhooks/librato/view.py',
    'zerver/webhooks/pivotal/view.py',
    'zerver/webhooks/semaphore/view.py',
    'zerver/webhooks/solano/view.py',
    'zerver/webhooks/teamcity/view.py',
    'zerver/webhooks/travis/view.py',
    'zerver/webhooks/zapier/view.py',
] for path in glob.glob(target)}

enforce_fully_covered = sorted(target_fully_covered - not_yet_fully_covered)

FAILED_TEST_PATH = 'var/last_test_failure.json'

def get_failed_tests() -> List[str]:
    try:
        with open(FAILED_TEST_PATH) as f:
            return ujson.load(f)
    except OSError:
        print("var/last_test_failure.json doesn't exist; running all tests.")
        return []

def write_failed_tests(failed_tests: List[str]) -> None:
    if failed_tests:
        with open(FAILED_TEST_PATH, 'w') as f:
            ujson.dump(failed_tests, f)

@contextlib.contextmanager
def block_internet() -> Iterator[None]:
    # Monkey-patching - responses library raises requests.ConnectionError when access to an unregistered URL
    # is attempted. We want to replace that with our own exception, so that it propagates all the way:
    with mock.patch.object(responses, 'ConnectionError', new=ZulipInternetBlockedError):
        # We'll run all tests in this context manager. It'll cause an error to be raised (see above comment),
        # if any code attempts to access the internet.
        with responses.RequestsMock():
            yield

class ZulipInternetBlockedError(Exception):
    def __init__(self, original_msg: str) -> None:
        zulip_msg = (
            "Outgoing network requests are not allowed in the Zulip tests. "
            "More details and advice are available here:"
            "https://zulip.readthedocs.io/en/latest/testing/testing.html#internet-access-inside-test-suites"
        )
        msg = "{}\nResponses library error message: {}".format(zulip_msg, original_msg)
        super().__init__(msg)

def main() -> None:
    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
    os.chdir(os.path.dirname(TOOLS_DIR))
    sys.path.insert(0, os.path.dirname(TOOLS_DIR))

    default_parallel = os.cpu_count()

    # Remove proxy settings for running backend tests
    os.environ.pop("http_proxy", "")
    os.environ.pop("https_proxy", "")

    from zerver.lib.test_fixtures import update_test_databases_if_required, \
        remove_test_run_directories

    from tools.lib.test_script import (
        assert_provisioning_status_ok,
    )

    os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings'
    # "-u" uses unbuffered IO, which is important when wrapping it in subprocess
    os.environ['PYTHONUNBUFFERED'] = 'y'

    usage = """test-backend [options]
    test-backend # Runs all backend tests
    test-backend zerver.tests.test_bugdown # run all tests in a test module
    test-backend zerver/tests/test_bugdown.py # run all tests in a test module
    test-backend test_bugdown # run all tests in a test module
    test-backend zerver.tests.test_bugdown.BugdownTest # run all tests in a test class
    test-backend BugdownTest # run all tests in a test class
    test-backend zerver.tests.test_bugdown.BugdownTest.test_inline_youtube # run a single test
    test-backend BugdownTest.test_inline_youtube # run a single test"""

    parser = argparse.ArgumentParser(description=usage,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('--nonfatal-errors', action="store_false", default=True,
                        dest="fatal_errors", help="Continue past test failures to run all tests")
    parser.add_argument('--coverage', dest='coverage',
                        action="store_true",
                        default=False,
                        help='Compute test coverage.')
    parser.add_argument('--verbose-coverage', dest='verbose_coverage',
                        action="store_true",
                        default=False, help='Enable verbose print of coverage report.')

    parser.add_argument('--parallel', dest='processes',
                        type=int,
                        action='store',
                        default=default_parallel,
                        help='Specify the number of processes to run the '
                             'tests in. Default is the number of logical CPUs')
    parser.add_argument('--profile', dest='profile',
                        action="store_true",
                        default=False, help='Profile test runtime.')
    parser.add_argument('--force', dest='force',
                        action="store_true",
                        default=False, help='Run tests despite possible problems.')
    parser.add_argument('--no-shallow', dest='no_shallow',
                        action="store_true",
                        default=False,
                        help="Don't allow shallow testing of templates (deprecated)")
    parser.add_argument('--verbose', dest='verbose',
                        action="store_true",
                        default=False,
                        help="Show detailed output")
    parser.add_argument('--report-slow-tests', dest='report_slow_tests',
                        action="store_true",
                        default=False,
                        help="Show which tests are slowest.")
    parser.add_argument('--reverse', dest='reverse',
                        action="store_true",
                        default=False,
                        help="Run tests in reverse order.")
    parser.add_argument('--rerun', dest="rerun",
                        action="store_true",
                        default=False,
                        help=("Run the tests which failed the last time "
                              "test-backend was run.  Implies --nonfatal-errors."))
    parser.add_argument('--include-webhooks', dest="include_webhooks",
                        action="store_true",
                        default=False,
                        help=("Include webhook tests.  By default, they are skipped for performance."))
    parser.add_argument('--generate-stripe-fixtures', dest="generate_stripe_fixtures",
                        action="store_true",
                        default=False,
                        help=("Generate Stripe test fixtures by making requests to Stripe test network"))
    parser.add_argument('args', nargs='*')

    options = parser.parse_args()
    args = options.args
    parallel = options.processes
    include_webhooks = options.coverage or options.include_webhooks

    if parallel < 1:
        raise argparse.ArgumentError(
            "option processes: Only positive integers are allowed.")

    zerver_test_dir = 'zerver/tests/'

    # While running --rerun, we read var/last_test_failure.json to get
    # the list of tests that failed on the last run, and then pretend
    # those tests were passed explicitly.  --rerun implies
    # --nonfatal-errors, so that we don't end up removing tests from
    # the list that weren't run.
    if options.rerun:
        parallel = 1
        options.fatal_errors = False
        failed_tests = get_failed_tests()
        if failed_tests:
            args = failed_tests
    if len(args) > 0:
        # If we passed a specific set of tests, run in serial mode.
        parallel = 1

        # to transform forward slashes '/' present in the argument into dots '.'
        for i, suite in enumerate(args):
            args[i] = suite.rstrip('/').replace("/", ".")

        def rewrite_arguments(search_key: str) -> None:
            for root, dirs, files_names in os.walk(zerver_test_dir, topdown=False):
                for file_name in files_names:
                    # Check for files starting with alphanumeric characters and ending with '.py'
                    # Ignore backup files if any
                    if not file_name[0].isalnum() or not file_name.endswith(".py"):
                        continue
                    filepath = os.path.join(root, file_name)
                    for line in open(filepath):
                        if search_key not in line:
                            continue
                        new_suite = filepath.replace(".py", ".") + suite
                        args[i] = new_suite
                        return

        for suite in args:
            if suite[0].isupper() and "test_" in suite:
                classname = suite.rsplit('.', 1)[0]
                rewrite_arguments(classname)
            elif suite[0].isupper():
                rewrite_arguments('class %s(' % (suite,))

        for i, suite in enumerate(args):
            if suite.startswith('test'):
                for root, dirs, files_names in os.walk(zerver_test_dir):
                    for file_name in files_names:
                        if file_name == suite or file_name == suite + ".py":
                            new_suite = os.path.join(root, file_name)
                            args[i] = new_suite
                            break

        for i, suite in enumerate(args):
            args[i] = suite.replace(".py", "")

        # to transform forward slashes '/' introduced by the zerver_test_dir into dots '.'
        # taking care of any forward slashes that might be present
        for i, suite in enumerate(args):
            args[i] = suite.replace("/", ".")

    full_suite = len(args) == 0

    if full_suite:
        suites = [
            "zerver.tests",
            "analytics.tests",
            "corporate.tests",
        ]
    else:
        suites = args

    if full_suite and include_webhooks:
        suites.append("zerver.webhooks")

    if options.generate_stripe_fixtures:
        if full_suite:
            suites = [
                "corporate.tests.test_stripe",
            ]
            full_suite = False
        os.environ["GENERATE_STRIPE_FIXTURES"] = "1"

    assert_provisioning_status_ok(options.force)

    if options.coverage:
        import atexit
        import coverage
        cov = coverage.Coverage(config_file="tools/coveragerc", concurrency='multiprocessing')
        atexit.register(lambda: cov.erase())  # Ensure the data file gets cleaned up at the end.
        cov.start()
    if options.profile:
        import cProfile
        prof = cProfile.Profile()
        prof.enable()

    # This is kind of hacky, but it's the most reliable way
    # to make sure instrumentation decorators know the
    # setting when they run.
    os.environ['TEST_INSTRUMENT_URL_COVERAGE'] = 'TRUE'

    # setup() needs to be called after coverage is started to get proper coverage reports of model
    # files, since part of setup is importing the models for all applications in INSTALLED_APPS.
    django.setup()

    update_test_databases_if_required()

    subprocess.check_call(['tools/webpack', '--test'])

    TestRunner = get_runner(settings)

    if parallel > 1:
        print("-- Running tests in parallel mode with {} "
              "processes.".format(parallel), flush=True)
    else:
        print("-- Running tests in serial mode.", flush=True)

    with block_internet():
        test_runner = TestRunner(failfast=options.fatal_errors, verbosity=2,
                                 parallel=parallel, reverse=options.reverse,
                                 keepdb=True)
        failures, failed_tests = test_runner.run_tests(suites, full_suite=full_suite,
                                                       include_webhooks=include_webhooks)
    write_failed_tests(failed_tests)

    templates_not_rendered = test_runner.get_shallow_tested_templates()
    # We only check the templates if all the tests ran and passed
    if not failures and full_suite and templates_not_rendered:
        missed_count = len(templates_not_rendered)
        print("\nError: %s templates have no tests!" % (missed_count,))
        for template in templates_not_rendered:
            print('  {}'.format(template))
        print("See zerver/tests/test_templates.py for the exclude list.")
        failures = True

    if options.coverage:
        cov.stop()
        cov.save()
        cov.combine()
        cov.data_suffix = False  # Disable suffix so that filename is .coverage
        cov.save()
        if options.verbose_coverage:
            print("Printing coverage data")
            cov.report(show_missing=False)
        cov.html_report(directory='var/coverage')
        print("HTML report saved; visit at http://127.0.0.1:9991/coverage/index.html")
    if full_suite and not failures and options.coverage:
        # Assert that various files have full coverage
        for path in enforce_fully_covered:
            missing_lines = cov.analysis2(path)[3]
            if len(missing_lines) > 0:
                print("ERROR: %s no longer has complete backend test coverage" % (path,))
                print("  Lines missing coverage: %s" % (missing_lines,))
                print()
                failures = True
        if failures:
            print("It looks like your changes lost 100% test coverage in one or more files")
            print("Usually, the right fix for this is to add some tests.")
            print("But also check out the include/exclude lists in tools/test-backend.")
            print("If this line intentionally is not tested, you can use a # nocoverage comment.")
            print("To run this check locally, use `test-backend --coverage`.")
        ok = True
        for path in not_yet_fully_covered:
            try:
                missing_lines = cov.analysis2(path)[3]
                if len(missing_lines) == 0 and path != "zerver/lib/migrate.py":
                    print("ERROR: %s has complete backend test coverage but is still in not_yet_fully_covered." % (path,))
                    ok = False
            except coverage.misc.NoSource:
                continue
        if not ok:
            print()
            print("There are one or more fully covered files that are still in not_yet_fully_covered.")
            print("Remove the file(s) from not_yet_fully_covered in `tools/test-backend`.")
            failures = True
    if options.profile:
        prof.disable()
        with tempfile.NamedTemporaryFile(prefix='profile.data.', delete=False) as stats_file:
            prof.dump_stats(stats_file.name)
            print("Profile data saved to {}".format(stats_file.name))
            print("You can visualize it using e.g. `snakeviz {}`".format(shlex.quote(stats_file.name)))
            print("Note: If you are using vagrant for development environment you will need to do:")
            print("1.) `vagrant ssh -- -L 8080:127.0.0.1:8080`")
            print("2.) `snakeviz -s {}`".format(shlex.quote(stats_file.name)))

    if options.report_slow_tests:
        from zerver.lib.test_runner import report_slow_tests
        # We do this even with failures, since slowness can be
        # an important clue as to why tests fail.
        report_slow_tests()

    # Ideally, we'd check for any leaked test databases here;
    # but that needs some hackery with database names.
    #
    # destroy_leaked_test_databases()

    removed = remove_test_run_directories()
    if removed:
        print("Removed %s stale test run directories!" % (removed,))

    # We'll have printed whether tests passed or failed above
    sys.exit(bool(failures))

if __name__ == "__main__":
    main()