From ecadb33fbc21c3dd84dd6438c5419e4b07f6afdc Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Sat, 18 Feb 2017 16:26:52 -0800 Subject: [PATCH] test-backend: Add 100% test coverage assertions. This adds an assertion, when `test-backend` is run with `--coverage`, that we have 100% test coverage on a list of files that we expect to. There's a whitelist/blacklist, managed in tools/test-backend. Fixes #3363. --- docs/testing-with-django.md | 5 +- tools/test-backend | 109 ++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/docs/testing-with-django.md b/docs/testing-with-django.md index fed42320ed..4699a2e84d 100644 --- a/docs/testing-with-django.md +++ b/docs/testing-with-django.md @@ -242,4 +242,7 @@ typically simulate their behavior using mocks. modules. You can use the `--coverage` option to generate coverage reports, and new code should have 100% coverage, which generally requires testing not only the "happy path" but also error handling code and -edge cases. +edge cases. Note that `test-backend --coverage` will assert that +various specific files in the project have 100% test coverage and +throw an error if their coverage has fallen. One of our project goals +is to expand that checking to ever-larger parts of the codebase. diff --git a/tools/test-backend b/tools/test-backend index e187c695e3..f764d50e04 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -2,6 +2,7 @@ from __future__ import print_function from __future__ import absolute_import +import glob import optparse import os import sys @@ -15,6 +16,100 @@ import django from django.conf import settings from django.test.utils import get_runner +target_fully_covered = {path for target in [ + 'analytics/tests/*.py', + 'analytics/lib/*.py', + 'zerver/context_processors.py', + 'zerver/lib/alert_words.py', + 'zerver/lib/attachments.py', + 'zerver/lib/avatar_hash.py', + 'zerver/lib/context_managers.py', + 'zerver/lib/domains.py', + 'zerver/lib/emoji.py', + 'zerver/lib/i18n.py', + 'zerver/lib/mention.py', + 'zerver/lib/message.py', + 'zerver/lib/name_restrictions.py', + 'zerver/lib/retention.py', + 'zerver/lib/streams.py', + 'zerver/lib/users.py', + 'zerver/lib/webhooks/*.py', + 'zerver/views/*.py', + # Once we have a nice negative tests system, we can add these: + # 'zerver/webhooks/*/*.py', + # 'zerver/webhooks/*/*/*.py', + 'zproject/backends.py', + # Uncovered but in exclude list and we'd like to have included soon + 'confirmation/models.py', + 'zerver/decorator.py', + 'zerver/lib/actions.py', + 'zerver/lib/events.py', + 'zerver/lib/bugdown/__init__.py', + 'zerver/lib/events.py', + 'zerver/lib/integrations.py', + 'zerver/lib/message.py', + 'zerver/lib/narrow.py', + 'zerver/lib/notifications.py', + 'zerver/lib/push_notifications.py', + 'zerver/lib/request.py', + 'zerver/lib/upload.py', + 'zerver/lib/validator.py', + 'zerver/models.py', +] for path in glob.glob(target)} + +not_yet_fully_covered = { + # Goal is for analytics to have 100% coverage + 'analytics/lib/counts.py', + 'analytics/lib/fixtures.py', + 'analytics/lib/time_utils.py', + # Major lib files should have 100% coverage + 'confirmation/models.py', + 'zerver/decorator.py', + 'zerver/lib/actions.py', + 'zerver/lib/events.py', + 'zerver/lib/bugdown/__init__.py', + 'zerver/lib/events.py', + 'zerver/lib/integrations.py', + 'zerver/lib/message.py', + 'zerver/lib/narrow.py', + 'zerver/lib/notifications.py', + 'zerver/lib/push_notifications.py', + 'zerver/lib/request.py', + 'zerver/lib/upload.py', + 'zerver/lib/validator.py', + 'zerver/models.py', + # Test files should have full coverage; it's a bug in the test if + # they don't! + 'zerver/tests/test_auth_backends.py', + 'zerver/tests/test_bugdown.py', + 'zerver/tests/test_decorators.py', + 'zerver/tests/test_email_mirror.py', + 'zerver/tests/test_events.py', + 'zerver/tests/test_narrow.py', + 'zerver/tests/test_signup.py', + 'zerver/tests/test_subs.py', + 'zerver/tests/test_templates.py', + 'zerver/tests/test_tornado.py', + 'zerver/tests/test_urls.py', + 'zerver/tests/tests.py', + # Getting views file coverage to 100% is a major project goal + 'zerver/views/auth.py', + 'zerver/views/integrations.py', + 'zerver/views/messages.py', + 'zerver/views/report.py', + 'zerver/views/unsubscribe.py', + 'zerver/views/zephyr.py', + 'zerver/views/realm.py', + 'zerver/views/invite.py', + 'zerver/views/home.py', + 'zerver/views/registration.py', + 'zerver/views/events_register.py', + # Getting this to 100% is a major project goal. + 'zproject/backends.py', +} + +enforce_fully_covered = sorted(target_fully_covered - not_yet_fully_covered) + if __name__ == "__main__": TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.dirname(TOOLS_DIR)) @@ -185,6 +280,20 @@ if __name__ == "__main__": cov.report(show_missing=False) cov.html_report(directory='var/coverage') print("HTML report saved to var/coverage") + if full_suite and not failures and options.coverage: + # Assert that various files have full coverage + for path in enforce_fully_covered: + missing_lines = cov.analysis2(path)[3] + if len(missing_lines) > 0: + print("ERROR: %s no longer has complete backend test coverage" % (path,)) + print(" Lines missing coverage: %s" % (missing_lines,)) + print() + failures = True + if failures: + print("It looks like your changes lost 100% test coverage in one or more files") + print("Usually, the right fix for this is to add some tests.") + print("But also check out the include/exclude lists in tools/test-backend.") + print("To run this check locally, use `test-backend --coverage`.") if options.profile: prof.disable() prof.dump_stats("/tmp/profile.data")