Revert "test-backend: Enable test coverage in multi-process mode."

This reverts commit a22d6d2c2a.

tools/test-backend

@@ -14,7 +14,6 @@ import ujson
 from lib import sanity_check
 sanity_check.check_venv(__file__)
 
-import coverage
 import django
 from django.conf import settings
 from django.test.utils import get_runner
@@ -198,6 +197,12 @@ if __name__ == "__main__":
                             "test-backend was run.  Implies --nonfatal-errors."))
 
     (options, args) = parser.parse_args()
+    if options.coverage:
+        # Currently coverage doesn't work with parallel mode, so when
+        # coverage parameter is supplied we enfore serial mode.
+        print("Disabling parallel mode because coverage isn't supported.")
+        options.processes = 1
+
     zerver_test_dir = 'zerver/tests/'
 
     # While running --rerun, we read var/last_test_failure.json to get
@@ -273,15 +278,9 @@ if __name__ == "__main__":
             sys.exit(1)
 
     if options.coverage:
-        settings.TEST_RUNNER = 'zerver.lib.test_runner.CoverageRunner'
-        cov = coverage.Coverage(config_file="tools/coveragerc", data_suffix=True)
-        cov.erase()
-        # We run coverage here in the main process to get covrage over imports
-        # but we also collect coverage in test_runner.run_subsuite. Each
-        # coverage instance writes its own stats to disk and we merge them at
-        # the end of the test run.
+        import coverage
+        cov = coverage.Coverage(config_file="tools/coveragerc")
         cov.start()
-
     if options.profile:
         import cProfile
         prof = cProfile.Profile()
@@ -334,7 +333,6 @@ if __name__ == "__main__":
     if options.coverage:
         cov.stop()
         cov.save()
-        cov.combine()
         if options.verbose_coverage:
             print("Printing coverage data")
             cov.report(show_missing=False)

zerver/lib/test_runner.py

@@ -3,13 +3,11 @@ from __future__ import print_function
 from functools import partial
 import random
 
-import types
 from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, \
-    Text, Type, cast
+    Text, Type
 from unittest import loader, runner  # type: ignore  # Mypy cannot pick these up.
 from unittest.result import TestResult
 
-import coverage
 from django.conf import settings
 from django.db import connections, ProgrammingError
 from django.urls.resolvers import RegexURLPattern
@@ -17,7 +15,6 @@ from django.test import TestCase
 from django.test import runner as django_runner
 from django.test.runner import DiscoverRunner
 from django.test.signals import template_rendered
-import six
 
 from zerver.lib import test_classes, test_helpers
 from zerver.lib.cache import bounce_key_prefix_for_testing
@@ -242,11 +239,8 @@ def process_instrumented_calls(func):
     for call in test_helpers.INSTRUMENTED_CALLS:
         func(call)
 
-def run_subsuite(collect_coverage, args):
-    # type: (bool, Tuple[int, Tuple[Type[Iterable[TestCase]], List[str]], bool]) -> Tuple[int, Any]
-    if collect_coverage:
-        cov = coverage.Coverage(config_file="tools/coveragerc", data_suffix=True)
-        cov.start()
+def run_subsuite(args):
+    # type: (Tuple[int, Tuple[Type[Iterable[TestCase]], List[str]], bool]) -> Tuple[int, Any]
     # Reset the accumulated INSTRUMENTED_CALLS before running this subsuite.
     test_helpers.INSTRUMENTED_CALLS = []
     subsuite_index, subsuite, failfast = args
@@ -258,10 +252,6 @@ def run_subsuite(collect_coverage, args):
     # TestResult are passed TestCase as the first argument but
     # addInstrumentation does not need it.
     process_instrumented_calls(partial(result.addInstrumentation, None))  # type: ignore
-
-    if collect_coverage:
-        cov.stop()
-        cov.save()
     return subsuite_index, result.events
 
 # Monkey-patch database creation to fix unnecessary sleep(1)
@@ -359,8 +349,6 @@ def init_worker(counter):
         print("*** Upload directory not found.")
 
 class TestSuite(unittest.TestSuite):
-    collect_coverage = False
-
     def run(self, result, debug=False):
         # type: (TestResult, Optional[bool]) -> TestResult
         """
@@ -399,34 +387,25 @@ class TestSuite(unittest.TestSuite):
             result._testRunEntered = False
         return result
 
-class CoverageTestSuite(TestSuite):
-    collect_coverage = True
-
 class TestLoader(loader.TestLoader):
     suiteClass = TestSuite
 
-class CoverageTestLoader(TestLoader):
-    suiteClass = CoverageTestSuite
-
 class ParallelTestSuite(django_runner.ParallelTestSuite):
+    run_subsuite = run_subsuite
     init_worker = init_worker
 
     def __init__(self, suite, processes, failfast):
         # type: (TestSuite, int, bool) -> None
         super(ParallelTestSuite, self).__init__(suite, processes, failfast)
         self.subsuites = SubSuiteList(self.subsuites)  # type: SubSuiteList
-        # ParallelTestSuite expects this to be a bound method so it can access
-        # __func__ when passing it to multiprocessing.
-        method = partial(run_subsuite, suite.collect_coverage)
-        self.run_subsuite = six.create_bound_method(cast(types.FunctionType, method), self)
 
 class Runner(DiscoverRunner):
     test_suite = TestSuite
     test_loader = TestLoader()
     parallel_test_suite = ParallelTestSuite
 
-    def __init__(self, coverage=False, *args, **kwargs):
-        # type: (bool, *Any, **Any) -> None
+    def __init__(self, *args, **kwargs):
+        # type: (*Any, **Any) -> None
         DiscoverRunner.__init__(self, *args, **kwargs)
 
         # `templates_rendered` holds templates which were rendered
@@ -513,10 +492,6 @@ class Runner(DiscoverRunner):
             write_instrumentation_reports(full_suite=full_suite)
         return failed, result.failed_tests
 
-class CoverageRunner(Runner):
-    test_suite = CoverageTestSuite
-    test_loader = CoverageTestLoader()
-
 def get_test_names(suite):
     # type: (TestSuite) -> List[str]
     return [full_test_name(t) for t in get_tests_from_suite(suite)]