zulip/tools/test-backend

#!/usr/bin/env python3

from typing import List, Any
import glob
import argparse
import os
import shlex
import sys
import subprocess
import tempfile
import ujson
import httplib2
import httpretty
import requests

# check for the venv
from lib import sanity_check
sanity_check.check_venv(__file__)

import django
from django.conf import settings
from django.test.utils import get_runner

target_fully_covered = {path for target in [
    'analytics/lib/*.py',
    'analytics/models.py',
    'analytics/tests/*.py',
    'analytics/views.py',
    # zerver/ and zerver/lib/ are important core files
    'zerver/*.py',
    'zerver/lib/*.py',
    'zerver/lib/*/*.py',
    'zerver/lib/*/*/*.py',
    'zerver/data_import/*.py',
    'zerver/templatetags/*.py',
    'zerver/tornado/*.py',
    # Billing files require 100% test coverage
    'corporate/lib/stripe.py',
    'corporate/views.py',
    # Test files should have 100% coverage; test code that isn't run
    # is likely a bug in the test.
    'zerver/tests/*.py',
    'corporate/tests/*.py',
    # As a project, we require 100% test coverage in the views files.
    'zerver/views/*.py',
    'zproject/backends.py',
    'confirmation/*.py',
    'zerver/webhooks/*/*.py',
    # Once we have a nice negative tests system, we can add these:
    # 'zerver/webhooks/*/*.py',
    # 'zerver/webhooks/*/*/*.py',
    'zerver/worker/*.py',
] for path in glob.glob(target)}

not_yet_fully_covered = {path for target in [
    # Analytics fixtures library is used to generate test fixtures;
    # isn't properly accounted for in test coverage analysis since it
    # runs before tests.
    'analytics/lib/fixtures.py',
    # We have 100% coverage on the new stuff; need to refactor old stuff.
    'analytics/views.py',
    # Major lib files should have 100% coverage
    'zerver/lib/addressee.py',
    'zerver/lib/bugdown/__init__.py',
    'zerver/lib/cache.py',
    'zerver/lib/cache_helpers.py',
    'zerver/lib/i18n.py',
    'zerver/lib/notifications.py',
    'zerver/lib/send_email.py',
    'zerver/lib/webhooks/common.py',
    'zerver/lib/url_preview/preview.py',
    'zerver/worker/queue_processors.py',
    # Bugdown sub-libs should have full coverage too; a lot are really close
    'zerver/lib/bugdown/api_arguments_table_generator.py',
    'zerver/lib/bugdown/fenced_code.py',
    'zerver/lib/bugdown/help_relative_links.py',
    'zerver/lib/bugdown/nested_code_blocks.py',
    # Other lib files that ideally would coverage, but aren't sorted
    'zerver/__init__.py',
    'zerver/filters.py',
    'zerver/middleware.py',
    'zerver/lib/bot_lib.py',
    'zerver/lib/camo.py',
    'zerver/lib/debug.py',
    'zerver/lib/digest.py',
    'zerver/lib/email_mirror.py',
    'zerver/lib/error_notify.py',
    'zerver/lib/export.py',
    'zerver/lib/feedback.py',
    'zerver/lib/fix_unreads.py',
    'zerver/lib/html_diff.py',
    'zerver/lib/import_realm.py',
    'zerver/lib/logging_util.py',
    'zerver/lib/outgoing_webhook.py',
    'zerver/lib/parallel.py',
    'zerver/lib/profile.py',
    'zerver/lib/queue.py',
    'zerver/lib/rate_limiter.py',
    'zerver/lib/sqlalchemy_utils.py',
    'zerver/lib/storage.py',
    'zerver/lib/stream_recipient.py',
    'zerver/lib/timeout.py',
    'zerver/lib/unminify.py',
    'zerver/lib/utils.py',
    'zerver/lib/zephyr.py',
    'zerver/templatetags/app_filters.py',
    'zerver/templatetags/minified_js.py',
    # Low priority for coverage
    'zerver/lib/ccache.py',
    'zerver/lib/generate_test_data.py',
    'zerver/lib/test_fixtures.py',
    'zerver/lib/test_runner.py',
    'zerver/lib/api_test_helpers.py',
    # Tornado should ideally have full coverage, but we're not there.
    'zerver/tornado/autoreload.py',
    'zerver/tornado/descriptors.py',
    'zerver/tornado/event_queue.py',
    'zerver/tornado/exceptions.py',
    'zerver/tornado/handlers.py',
    'zerver/tornado/ioloop_logging.py',
    'zerver/tornado/sharding.py',
    'zerver/tornado/socket.py',
    'zerver/tornado/views.py',
    'zerver/tornado/websocket_client.py',
    # Data import files; relatively low priority
    'zerver/data_import/hipchat*.py',
    'zerver/data_import/sequencer.py',
    'zerver/data_import/slack.py',
    'zerver/data_import/gitter.py',
    'zerver/data_import/import_util.py',
    # Webhook integrations with incomplete coverage
    'zerver/webhooks/github_legacy/view.py',
    'zerver/webhooks/greenhouse/view.py',
    'zerver/webhooks/jira/view.py',
    'zerver/webhooks/librato/view.py',
    'zerver/webhooks/pivotal/view.py',
    'zerver/webhooks/semaphore/view.py',
    'zerver/webhooks/solano/view.py',
    'zerver/webhooks/teamcity/view.py',
    'zerver/webhooks/travis/view.py',
    'zerver/webhooks/zapier/view.py',
] for path in glob.glob(target)}

enforce_fully_covered = sorted(target_fully_covered - not_yet_fully_covered)

FAILED_TEST_PATH = 'var/last_test_failure.json'

def get_failed_tests():
    # type: () -> List[str]
    try:
        with open(FAILED_TEST_PATH, 'r') as f:
            return ujson.load(f)
    except IOError:
        print("var/last_test_failure.json doesn't exist; running all tests.")
        return []

def write_failed_tests(failed_tests):
    # type: (List[str]) -> None
    if failed_tests:
        with open(FAILED_TEST_PATH, 'w') as f:
            ujson.dump(failed_tests, f)

def block_internet():
    # type: () -> None
    # We are blocking internet currently by assuming mostly any test would use
    # httplib2 to access internet.
    requests_orig = requests.request

    def internet_guard_requests(*args: Any, **kwargs: Any) -> Any:
        if httpretty.is_enabled():
            return requests_orig(*args, **kwargs)
        raise Exception("Outgoing network requests are not allowed in the Zulip tests. "
                        "More details and advice are available here:"
                        "https://zulip.readthedocs.io/en/latest/testing/testing.html#internet-access-inside-test-suites")

    requests.request = internet_guard_requests

    http2lib_request_orig = httplib2.Http.request

    def internet_guard_httplib2(*args: Any, **kwargs: Any) -> Any:
        if httpretty.is_enabled():
            return http2lib_request_orig(*args, **kwargs)
        raise Exception("Outgoing network requests are not allowed in the Zulip tests. "
                        "More details and advice are available here:"
                        "https://zulip.readthedocs.io/en/latest/testing/testing.html#internet-access-inside-test-suites")
    httplib2.Http.request = internet_guard_httplib2

def main() -> None:
    block_internet()
    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
    os.chdir(os.path.dirname(TOOLS_DIR))
    sys.path.insert(0, os.path.dirname(TOOLS_DIR))

    # Remove proxy settings for running backend tests
    os.environ["http_proxy"] = ""
    os.environ["https_proxy"] = ""

    from zerver.lib.test_fixtures import run_generate_fixtures_if_required

    from tools.lib.test_script import (
        get_provisioning_status,
    )

    os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings'
    # "-u" uses unbuffered IO, which is important when wrapping it in subprocess
    os.environ['PYTHONUNBUFFERED'] = 'y'

    usage = """test-backend [options]
    test-backend # Runs all backend tests
    test-backend zerver.tests.test_bugdown # run all tests in a test module
    test-backend zerver/tests/test_bugdown.py # run all tests in a test module
    test-backend test_bugdown # run all tests in a test module
    test-backend zerver.tests.test_bugdown.BugdownTest # run all tests in a test class
    test-backend BugdownTest # run all tests in a test class
    test-backend zerver.tests.test_bugdown.BugdownTest.test_inline_youtube # run a single test
    test-backend BugdownTest.test_inline_youtube # run a single test"""

    parser = argparse.ArgumentParser(description=usage,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('--nonfatal-errors', action="store_false", default=True,
                        dest="fatal_errors", help="Continue past test failures to run all tests")
    parser.add_argument('--coverage', dest='coverage',
                        action="store_true",
                        default=False,
                        help='Compute test coverage.')
    parser.add_argument('--verbose-coverage', dest='verbose_coverage',
                        action="store_true",
                        default=False, help='Enable verbose print of coverage report.')

    parser.add_argument('--parallel', dest='processes',
                        type=int,
                        action='store',
                        default=4,
                        help='Specify the number of processes to run the '
                             'tests in. Default is 4.')
    parser.add_argument('--profile', dest='profile',
                        action="store_true",
                        default=False, help='Profile test runtime.')
    parser.add_argument('--force', dest='force',
                        action="store_true",
                        default=False, help='Run tests despite possible problems.')
    parser.add_argument('--no-shallow', dest='no_shallow',
                        action="store_true",
                        default=False,
                        help="Don't allow shallow testing of templates (deprecated)")
    parser.add_argument('--verbose', dest='verbose',
                        action="store_true",
                        default=False,
                        help="Show detailed output")
    parser.add_argument('--generate-fixtures', action="store_true", default=False,
                        dest="generate_fixtures",
                        help=("Force a call to generate-fixtures."))
    parser.add_argument('--report-slow-tests', dest='report_slow_tests',
                        action="store_true",
                        default=False,
                        help="Show which tests are slowest.")
    parser.add_argument('--reverse', dest='reverse',
                        action="store_true",
                        default=False,
                        help="Run tests in reverse order.")
    parser.add_argument('--rerun', dest="rerun",
                        action="store_true",
                        default=False,
                        help=("Run the tests which failed the last time "
                              "test-backend was run.  Implies --nonfatal-errors."))
    parser.add_argument('--include-webhooks', dest="include_webhooks",
                        action="store_true",
                        default=False,
                        help=("Include webhook tests.  By default, they are skipped for performance."))
    parser.add_argument('args', nargs='*')

    options = parser.parse_args()
    args = options.args
    parallel = options.processes
    include_webhooks = options.coverage or options.include_webhooks

    if parallel < 1:
        raise argparse.ArgumentError(
            "option processes: Only positive integers are allowed.")

    zerver_test_dir = 'zerver/tests/'

    # While running --rerun, we read var/last_test_failure.json to get
    # the list of tests that failed on the last run, and then pretend
    # those tests were passed explicitly.  --rerun implies
    # --nonfatal-errors, so that we don't end up removing tests from
    # the list that weren't run.
    if options.rerun:
        parallel = 1
        options.fatal_errors = False
        failed_tests = get_failed_tests()
        if failed_tests:
            args = failed_tests
    if len(args) > 0:
        # If we passed a specific set of tests, run in serial mode.
        parallel = 1

        # to transform forward slashes '/' present in the argument into dots '.'
        for i, suite in enumerate(args):
            args[i] = suite.rstrip('/').replace("/", ".")

        def rewrite_arguments(search_key):
            # type: (str) -> None
            for root, dirs, files_names in os.walk(zerver_test_dir, topdown=False):
                for file_name in files_names:
                    # Check for files starting with alphanumeric characters and ending with '.py'
                    # Ignore backup files if any
                    if not file_name[0].isalnum() or not file_name.endswith(".py"):
                        continue
                    filepath = os.path.join(root, file_name)
                    for line in open(filepath):
                        if search_key not in line:
                            continue
                        new_suite = filepath.replace(".py", ".") + suite
                        args[i] = new_suite
                        return

        for suite in args:
            if suite[0].isupper() and "test_" in suite:
                classname = suite.rsplit('.', 1)[0]
                rewrite_arguments(classname)
            elif suite[0].isupper():
                rewrite_arguments('class %s(' % (suite,))

        for i, suite in enumerate(args):
            if suite.startswith('test'):
                for root, dirs, files_names in os.walk(zerver_test_dir):
                    for file_name in files_names:
                        if file_name == suite or file_name == suite + ".py":
                            new_suite = os.path.join(root, file_name)
                            args[i] = new_suite
                            break

        for i, suite in enumerate(args):
            args[i] = suite.replace(".py", "")

        # to transform forward slashes '/' introduced by the zerver_test_dir into dots '.'
        # taking care of any forward slashes that might be present
        for i, suite in enumerate(args):
            args[i] = suite.replace("/", ".")

    full_suite = len(args) == 0

    if full_suite:
        suites = [
            "zerver.tests",
            "analytics.tests",
            "corporate.tests",
        ]
    else:
        suites = args

    if full_suite and include_webhooks:
        suites.append("zerver.webhooks")

    if not options.force:
        ok, msg = get_provisioning_status()
        if not ok:
            print(msg)
            print('If you really know what you are doing, use --force to run anyway.')
            sys.exit(1)

    if options.coverage:
        import coverage
        cov = coverage.Coverage(config_file="tools/coveragerc", concurrency='multiprocessing')
        cov.start()
    if options.profile:
        import cProfile
        prof = cProfile.Profile()
        prof.enable()

    # This is kind of hacky, but it's the most reliable way
    # to make sure instrumentation decorators know the
    # setting when they run.
    os.environ['TEST_INSTRUMENT_URL_COVERAGE'] = 'TRUE'

    # setup() needs to be called after coverage is started to get proper coverage reports of model
    # files, since part of setup is importing the models for all applications in INSTALLED_APPS.
    django.setup()

    run_generate_fixtures_if_required(options.generate_fixtures)

    subprocess.check_call(['tools/webpack', '--test'])

    TestRunner = get_runner(settings)

    if parallel > 1:
        print("-- Running tests in parallel mode with {} "
              "processes.".format(parallel))
    else:
        print("-- Running tests in serial mode.")

    test_runner = TestRunner(failfast=options.fatal_errors, verbosity=2,
                             parallel=parallel, reverse=options.reverse,
                             keepdb=True)
    failures, failed_tests = test_runner.run_tests(suites, full_suite=full_suite,
                                                   include_webhooks=include_webhooks)
    write_failed_tests(failed_tests)

    templates_not_rendered = test_runner.get_shallow_tested_templates()
    # We only check the templates if all the tests ran and passed
    if not failures and full_suite and templates_not_rendered:
        missed_count = len(templates_not_rendered)
        print("\nError: %s templates have no tests!" % (missed_count,))
        for template in templates_not_rendered:
            print('  {}'.format(template))
        print("See zerver/tests/test_templates.py for the exclude list.")
        failures = True

    if options.coverage:
        cov.stop()
        cov.save()
        cov.combine()
        cov.data_suffix = False  # Disable suffix so that filename is .coverage
        cov.save()
        if options.verbose_coverage:
            print("Printing coverage data")
            cov.report(show_missing=False)
        cov.html_report(directory='var/coverage')
        print("HTML report saved; visit at http://127.0.0.1:9991/coverage/index.html")
    if full_suite and not failures and options.coverage:
        # Assert that various files have full coverage
        for path in enforce_fully_covered:
            missing_lines = cov.analysis2(path)[3]
            if len(missing_lines) > 0:
                print("ERROR: %s no longer has complete backend test coverage" % (path,))
                print("  Lines missing coverage: %s" % (missing_lines,))
                print()
                failures = True
        if failures:
            print("It looks like your changes lost 100% test coverage in one or more files")
            print("Usually, the right fix for this is to add some tests.")
            print("But also check out the include/exclude lists in tools/test-backend.")
            print("If this line intentionally is not tested, you can use a # nocoverage comment.")
            print("To run this check locally, use `test-backend --coverage`.")
        ok = True
        for path in not_yet_fully_covered:
            try:
                missing_lines = cov.analysis2(path)[3]
                if len(missing_lines) == 0:
                    print("ERROR: %s has complete backend test coverage but is still in not_yet_fully_covered." % (path,))
                    ok = False
            except coverage.misc.NoSource:
                continue
        if not ok:
            print()
            print("There are one or more fully covered files that are still in not_yet_fully_covered.")
            print("Remove the file(s) from not_yet_fully_covered in `tools/test-backend`.")
            failures = True
    if options.profile:
        prof.disable()
        with tempfile.NamedTemporaryFile(prefix='profile.data.', delete=False) as stats_file:
            prof.dump_stats(stats_file.name)
            print("Profile data saved to {}".format(stats_file.name))
            print("You can visualize it using e.g. `snakeviz {}`".format(shlex.quote(stats_file.name)))
            print("Note: If you are using vagrant for development environment you will need to do:")
            print("1.) `vagrant ssh -- -L 8080:127.0.0.1:8080`")
            print("2.) `snakeviz -s {}`".format(shlex.quote(stats_file.name)))

    if options.report_slow_tests:
        from zerver.lib.test_runner import report_slow_tests
        # We do this even with failures, since slowness can be
        # an important clue as to why tests fail.
        report_slow_tests()

    # We'll have printed whether tests passed or failed above
    sys.exit(bool(failures))

if __name__ == "__main__":
    main()