From 21f83afe3a8232dc77ce8700fa777accb1afdd5b Mon Sep 17 00:00:00 2001 From: Steve Howell Date: Wed, 27 Jul 2016 16:40:28 -0700 Subject: [PATCH] Add --url-coverage option to ./tools/test-backend. --- tools/test-backend | 8 +++++++ zerver/lib/test_helpers.py | 46 ++++++++++++++++++++++++++++++++++++++ zerver/lib/test_runner.py | 6 ++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/tools/test-backend b/tools/test-backend index 80401469c3..86bbc0d154 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -39,6 +39,9 @@ if __name__ == "__main__": parser.add_option('--coverage', dest='coverage', action="store_true", default=False, help='Compute test coverage.') + parser.add_option('--url-coverage', dest='url_coverage', + action="store_true", + default=False, help='Write url coverage data.') parser.add_option('--no-verbose-coverage', dest='verbose_coverage', action="store_false", default=True, help='Disable verbose print of coverage report.') @@ -76,6 +79,11 @@ if __name__ == "__main__": import cProfile prof = cProfile.Profile() prof.enable() + if options.url_coverage: + # This is kind of hacky, but it's the most reliable way + # to make sure instrumentation decorators know the + # setting when they run. + os.environ['TEST_INSTRUMENT_URL_COVERAGE'] = 'TRUE' # setup() needs to be called after coverage is started to get proper coverage reports of model # files, since part of setup is importing the models for all applications in INSTALLED_APPS. diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index 0579315924..4e667ad1c7 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import print_function from contextlib import contextmanager from typing import (cast, Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sized, Tuple, Union) @@ -195,6 +196,45 @@ class POSTRequestMock(object): self._log_data = {} # type: Dict[str, Any] self.META = {'PATH_INFO': 'test'} +INSTRUMENTING = os.environ.get('TEST_INSTRUMENT_URL_COVERAGE', '') == 'TRUE' +INSTRUMENTED_CALLS = [] + +def instrument_url(f): + if not INSTRUMENTING: + return f + else: + def wrapper(self, url, info={}, **kwargs): + start = time.time() + result = f(self, url, info, **kwargs) + delay = time.time() - start + test_name = self.id() + if '?' in url: + url, extra_info = url.split('?', 1) + else: + extra_info = '' + + INSTRUMENTED_CALLS.append(dict( + url=url, + status_code=result.status_code, + method=f.__name__, + delay=delay, + extra_info=extra_info, + info=info, + test_name=test_name, + kwargs=kwargs)) + return result + return wrapper + +def write_instrumentation_report(): + if INSTRUMENTING: + var_dir = 'var' # TODO make sure path is robust here + fn = os.path.join(var_dir, 'url_coverage.txt') + with open(fn, 'w') as f: + for call in INSTRUMENTED_CALLS: + line = ujson.dumps(call) + f.write(line + '\n') + print('URL coverage report is in %s' % (fn,)) + class AuthedTestCase(TestCase): ''' WRAPPER_COMMENT: @@ -209,6 +249,7 @@ class AuthedTestCase(TestCase): django_client to fool the regext. ''' + @instrument_url def client_patch(self, url, info={}, **kwargs): # type: (text_type, Dict[str, Any], **Any) -> HttpResponse """ @@ -218,6 +259,7 @@ class AuthedTestCase(TestCase): django_client = self.client # see WRAPPER_COMMENT return django_client.patch(url, encoded, **kwargs) + @instrument_url def client_patch_multipart(self, url, info={}, **kwargs): # type: (text_type, Dict[str, Any], **Any) -> HttpResponse """ @@ -236,22 +278,26 @@ class AuthedTestCase(TestCase): content_type=MULTIPART_CONTENT, **kwargs) + @instrument_url def client_put(self, url, info={}, **kwargs): # type: (text_type, Dict[str, Any], **Any) -> HttpResponse encoded = urllib.parse.urlencode(info) django_client = self.client # see WRAPPER_COMMENT return django_client.put(url, encoded, **kwargs) + @instrument_url def client_delete(self, url, info={}, **kwargs): # type: (text_type, Dict[str, Any], **Any) -> HttpResponse encoded = urllib.parse.urlencode(info) django_client = self.client # see WRAPPER_COMMENT return django_client.delete(url, encoded, **kwargs) + @instrument_url def client_post(self, url, info={}, **kwargs): django_client = self.client # see WRAPPER_COMMENT return django_client.post(url, info, **kwargs) + @instrument_url def client_get(self, url, info={}, **kwargs): django_client = self.client # see WRAPPER_COMMENT return django_client.get(url, info, **kwargs) diff --git a/zerver/lib/test_runner.py b/zerver/lib/test_runner.py index 4a64d3edc2..dc8e22ecf5 100644 --- a/zerver/lib/test_runner.py +++ b/zerver/lib/test_runner.py @@ -8,7 +8,9 @@ from django.test.signals import template_rendered from zerver.lib.cache import bounce_key_prefix_for_testing from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import get_all_templates +from zerver.lib.test_helpers import ( + get_all_templates, write_instrumentation_report, + ) import os import subprocess @@ -203,4 +205,6 @@ class Runner(DiscoverRunner): get_sqlalchemy_connection() failed = self.run_suite(suite, fatal_errors=kwargs.get('fatal_errors')) self.teardown_test_environment() + if not failed: + write_instrumentation_report() return failed