Add --url-coverage option to ./tools/test-backend.

tools/test-backend

@@ -39,6 +39,9 @@ if __name__ == "__main__":
     parser.add_option('--coverage', dest='coverage',
                       action="store_true",
                       default=False, help='Compute test coverage.')
+    parser.add_option('--url-coverage', dest='url_coverage',
+                      action="store_true",
+                      default=False, help='Write url coverage data.')
     parser.add_option('--no-verbose-coverage', dest='verbose_coverage',
                       action="store_false",
                       default=True, help='Disable verbose print of coverage report.')
@@ -76,6 +79,11 @@ if __name__ == "__main__":
         import cProfile
         prof = cProfile.Profile()
         prof.enable()
+    if options.url_coverage:
+        # This is kind of hacky, but it's the most reliable way
+        # to make sure instrumentation decorators know the
+        # setting when they run.
+        os.environ['TEST_INSTRUMENT_URL_COVERAGE'] = 'TRUE'
 
     # setup() needs to be called after coverage is started to get proper coverage reports of model
     # files, since part of setup is importing the models for all applications in INSTALLED_APPS.

zerver/lib/test_helpers.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 from contextlib import contextmanager
 from typing import (cast, Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional,
     Sized, Tuple, Union)
@@ -195,6 +196,45 @@ class POSTRequestMock(object):
         self._log_data = {} # type: Dict[str, Any]
         self.META = {'PATH_INFO': 'test'}
 
+INSTRUMENTING = os.environ.get('TEST_INSTRUMENT_URL_COVERAGE', '') == 'TRUE'
+INSTRUMENTED_CALLS = []
+
+def instrument_url(f):
+    if not INSTRUMENTING:
+        return f
+    else:
+        def wrapper(self, url, info={}, **kwargs):
+            start = time.time()
+            result = f(self, url, info, **kwargs)
+            delay = time.time() - start
+            test_name = self.id()
+            if '?' in url:
+                url, extra_info = url.split('?', 1)
+            else:
+                extra_info = ''
+
+            INSTRUMENTED_CALLS.append(dict(
+                url=url,
+                status_code=result.status_code,
+                method=f.__name__,
+                delay=delay,
+                extra_info=extra_info,
+                info=info,
+                test_name=test_name,
+                kwargs=kwargs))
+            return result
+        return wrapper
+
+def write_instrumentation_report():
+    if INSTRUMENTING:
+        var_dir = 'var' # TODO make sure path is robust here
+        fn = os.path.join(var_dir, 'url_coverage.txt')
+        with open(fn, 'w') as f:
+            for call in INSTRUMENTED_CALLS:
+                line = ujson.dumps(call)
+                f.write(line + '\n')
+        print('URL coverage report is in %s' % (fn,))
+
 class AuthedTestCase(TestCase):
     '''
     WRAPPER_COMMENT:
@@ -209,6 +249,7 @@ class AuthedTestCase(TestCase):
     django_client to fool the regext.
     '''
 
+    @instrument_url
     def client_patch(self, url, info={}, **kwargs):
         # type: (text_type, Dict[str, Any], **Any) -> HttpResponse
         """
@@ -218,6 +259,7 @@ class AuthedTestCase(TestCase):
         django_client = self.client # see WRAPPER_COMMENT
         return django_client.patch(url, encoded, **kwargs)
 
+    @instrument_url
     def client_patch_multipart(self, url, info={}, **kwargs):
         # type: (text_type, Dict[str, Any], **Any) -> HttpResponse
         """
@@ -236,22 +278,26 @@ class AuthedTestCase(TestCase):
             content_type=MULTIPART_CONTENT,
             **kwargs)
 
+    @instrument_url
     def client_put(self, url, info={}, **kwargs):
         # type: (text_type, Dict[str, Any], **Any) -> HttpResponse
         encoded = urllib.parse.urlencode(info)
         django_client = self.client # see WRAPPER_COMMENT
         return django_client.put(url, encoded, **kwargs)
 
+    @instrument_url
     def client_delete(self, url, info={}, **kwargs):
         # type: (text_type, Dict[str, Any], **Any) -> HttpResponse
         encoded = urllib.parse.urlencode(info)
         django_client = self.client # see WRAPPER_COMMENT
         return django_client.delete(url, encoded, **kwargs)
 
+    @instrument_url
     def client_post(self, url, info={}, **kwargs):
         django_client = self.client # see WRAPPER_COMMENT
         return django_client.post(url, info, **kwargs)
 
+    @instrument_url
     def client_get(self, url, info={}, **kwargs):
         django_client = self.client # see WRAPPER_COMMENT
         return django_client.get(url, info, **kwargs)

zerver/lib/test_runner.py

@@ -8,7 +8,9 @@ from django.test.signals import template_rendered
 
 from zerver.lib.cache import bounce_key_prefix_for_testing
 from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection
-from zerver.lib.test_helpers import get_all_templates
+from zerver.lib.test_helpers import (
+    get_all_templates, write_instrumentation_report,
+    )
 
 import os
 import subprocess
@@ -203,4 +205,6 @@ class Runner(DiscoverRunner):
         get_sqlalchemy_connection()
         failed = self.run_suite(suite, fatal_errors=kwargs.get('fatal_errors'))
         self.teardown_test_environment()
+        if not failed:
+            write_instrumentation_report()
         return failed