test-backend: Add 100% test coverage assertions.

This adds an assertion, when `test-backend` is run with `--coverage`,
that we have 100% test coverage on a list of files that we expect to.
There's a whitelist/blacklist, managed in tools/test-backend.

Fixes #3363.

docs/testing-with-django.md

@@ -242,4 +242,7 @@ typically simulate their behavior using mocks.
 modules.  You can use the `--coverage` option to generate coverage
 reports, and new code should have 100% coverage, which generally requires
 testing not only the "happy path" but also error handling code and
-edge cases.
+edge cases.  Note that `test-backend --coverage` will assert that
+various specific files in the project have 100% test coverage and
+throw an error if their coverage has fallen.  One of our project goals
+is to expand that checking to ever-larger parts of the codebase.

tools/test-backend

@@ -2,6 +2,7 @@
 
 from __future__ import print_function
 from __future__ import absolute_import
+import glob
 import optparse
 import os
 import sys
@@ -15,6 +16,100 @@ import django
 from django.conf import settings
 from django.test.utils import get_runner
 
+target_fully_covered = {path for target in [
+    'analytics/tests/*.py',
+    'analytics/lib/*.py',
+    'zerver/context_processors.py',
+    'zerver/lib/alert_words.py',
+    'zerver/lib/attachments.py',
+    'zerver/lib/avatar_hash.py',
+    'zerver/lib/context_managers.py',
+    'zerver/lib/domains.py',
+    'zerver/lib/emoji.py',
+    'zerver/lib/i18n.py',
+    'zerver/lib/mention.py',
+    'zerver/lib/message.py',
+    'zerver/lib/name_restrictions.py',
+    'zerver/lib/retention.py',
+    'zerver/lib/streams.py',
+    'zerver/lib/users.py',
+    'zerver/lib/webhooks/*.py',
+    'zerver/views/*.py',
+    # Once we have a nice negative tests system, we can add these:
+    # 'zerver/webhooks/*/*.py',
+    # 'zerver/webhooks/*/*/*.py',
+    'zproject/backends.py',
+    # Uncovered but in exclude list and we'd like to have included soon
+    'confirmation/models.py',
+    'zerver/decorator.py',
+    'zerver/lib/actions.py',
+    'zerver/lib/events.py',
+    'zerver/lib/bugdown/__init__.py',
+    'zerver/lib/events.py',
+    'zerver/lib/integrations.py',
+    'zerver/lib/message.py',
+    'zerver/lib/narrow.py',
+    'zerver/lib/notifications.py',
+    'zerver/lib/push_notifications.py',
+    'zerver/lib/request.py',
+    'zerver/lib/upload.py',
+    'zerver/lib/validator.py',
+    'zerver/models.py',
+] for path in glob.glob(target)}
+
+not_yet_fully_covered = {
+    # Goal is for analytics to have 100% coverage
+    'analytics/lib/counts.py',
+    'analytics/lib/fixtures.py',
+    'analytics/lib/time_utils.py',
+    # Major lib files should have 100% coverage
+    'confirmation/models.py',
+    'zerver/decorator.py',
+    'zerver/lib/actions.py',
+    'zerver/lib/events.py',
+    'zerver/lib/bugdown/__init__.py',
+    'zerver/lib/events.py',
+    'zerver/lib/integrations.py',
+    'zerver/lib/message.py',
+    'zerver/lib/narrow.py',
+    'zerver/lib/notifications.py',
+    'zerver/lib/push_notifications.py',
+    'zerver/lib/request.py',
+    'zerver/lib/upload.py',
+    'zerver/lib/validator.py',
+    'zerver/models.py',
+    # Test files should have full coverage; it's a bug in the test if
+    # they don't!
+    'zerver/tests/test_auth_backends.py',
+    'zerver/tests/test_bugdown.py',
+    'zerver/tests/test_decorators.py',
+    'zerver/tests/test_email_mirror.py',
+    'zerver/tests/test_events.py',
+    'zerver/tests/test_narrow.py',
+    'zerver/tests/test_signup.py',
+    'zerver/tests/test_subs.py',
+    'zerver/tests/test_templates.py',
+    'zerver/tests/test_tornado.py',
+    'zerver/tests/test_urls.py',
+    'zerver/tests/tests.py',
+    # Getting views file coverage to 100% is a major project goal
+    'zerver/views/auth.py',
+    'zerver/views/integrations.py',
+    'zerver/views/messages.py',
+    'zerver/views/report.py',
+    'zerver/views/unsubscribe.py',
+    'zerver/views/zephyr.py',
+    'zerver/views/realm.py',
+    'zerver/views/invite.py',
+    'zerver/views/home.py',
+    'zerver/views/registration.py',
+    'zerver/views/events_register.py',
+    # Getting this to 100% is a major project goal.
+    'zproject/backends.py',
+}
+
+enforce_fully_covered = sorted(target_fully_covered - not_yet_fully_covered)
+
 if __name__ == "__main__":
     TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
     os.chdir(os.path.dirname(TOOLS_DIR))
@@ -185,6 +280,20 @@ if __name__ == "__main__":
             cov.report(show_missing=False)
         cov.html_report(directory='var/coverage')
         print("HTML report saved to var/coverage")
+    if full_suite and not failures and options.coverage:
+        # Assert that various files have full coverage
+        for path in enforce_fully_covered:
+            missing_lines = cov.analysis2(path)[3]
+            if len(missing_lines) > 0:
+                print("ERROR: %s no longer has complete backend test coverage" % (path,))
+                print("  Lines missing coverage: %s" % (missing_lines,))
+                print()
+                failures = True
+        if failures:
+            print("It looks like your changes lost 100% test coverage in one or more files")
+            print("Usually, the right fix for this is to add some tests.")
+            print("But also check out the include/exclude lists in tools/test-backend.")
+            print("To run this check locally, use `test-backend --coverage`.")
     if options.profile:
         prof.disable()
         prof.dump_stats("/tmp/profile.data")