Show templates rendered report.

Add two options to the `test-backend` script:

1. verbose
    If given the `test-backend` script will give detailed output.
2. no-shallow
    Default value is False. If given the `test-backend` script will
    fail if it finds a template which is shallow tested.

tools/test-backend

@@ -27,6 +27,14 @@ if __name__ == "__main__":
     parser.add_option('--profile', dest='profile',
                       action="store_true",
                       default=False, help='Profile test runtime.')
+    parser.add_option('--no-shallow', dest='no_shallow',
+                      action="store_true",
+                      default=False,
+                      help="Don't allow shallow testing of templates")
+    parser.add_option('--verbose', dest='verbose',
+                      action="store_true",
+                      default=False,
+                      help="Show detailed output")
 
     (options, args) = parser.parse_args()
     if len(args) == 0:
@@ -49,6 +57,19 @@ if __name__ == "__main__":
     test_runner = TestRunner()
     failures = test_runner.run_tests(suites, fatal_errors=options.fatal_errors)
 
+    templates_not_rendered = test_runner.get_shallow_tested_templates()
+    if templates_not_rendered:
+        missed_count = len(templates_not_rendered)
+        if options.no_shallow or options.verbose:
+            print("*** Shallow tested templates: {}".format(missed_count))
+
+        if options.verbose:
+            for template in templates_not_rendered:
+                print('--- {}'.format(template))
+
+        if options.no_shallow:
+            failures = True
+
     if options.coverage:
         cov.stop()
         cov.save()

zerver/lib/test_helpers.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from typing import Any, Callable, Generator, Iterable, Tuple
 
 from django.test import TestCase
+from django.template import loader
 
 from zerver.lib.initial_password import initial_password
 from zerver.lib.db import TimeTrackingCursor
@@ -367,3 +368,26 @@ class AuthedTestCase(TestCase):
 
     def get_last_message(self):
         return Message.objects.latest('id')
+
+def get_all_templates():
+    templates = []
+
+    relpath = os.path.relpath
+    isfile = os.path.isfile
+    path_exists = os.path.exists
+
+    is_valid_template = lambda p, n: not n.startswith('.') and isfile(p)
+
+    def process(template_dir, dirname, fnames):
+        for name in fnames:
+            path = os.path.join(dirname, name)
+            if is_valid_template(path, name):
+                templates.append(relpath(path, template_dir))
+
+    for engine in loader.engines.all():
+        template_dirs = [d for d in engine.template_dirs if path_exists(d)]
+        for template_dir in template_dirs:
+            template_dir = os.path.normpath(template_dir)
+            os.path.walk(template_dir, process, template_dir)
+
+    return templates

zerver/lib/test_runner.py

@@ -1,8 +1,13 @@
 from __future__ import print_function
+
+from typing import Any, Set
+
 from django.test.runner import DiscoverRunner
+from django.test.signals import template_rendered
 
 from zerver.lib.cache import bounce_key_prefix_for_testing
 from zerver.views.messages import get_sqlalchemy_connection
+from zerver.lib.test_helpers import get_all_templates
 
 import os
 import subprocess
@@ -111,8 +116,30 @@ def run_test(test):
 
 class Runner(DiscoverRunner):
     def __init__(self, *args, **kwargs):
+        # type: (*Any, **Any) -> None
         DiscoverRunner.__init__(self, *args, **kwargs)
 
+        # `templates_rendered` holds templates which were rendered
+        # in proper logical tests.
+        self.templates_rendered = set()  # type: Set[str]
+        # `shallow_tested_templates` holds templates which were rendered
+        # in `zerver.tests.test_templates`.
+        self.shallow_tested_templates = set()  # type: Set[str]
+        template_rendered.connect(self.on_template_rendered)
+
+    def on_template_rendered(self, sender, context, **kwargs):
+        if hasattr(sender, 'template'):
+            template_name = sender.template.name
+            if template_name not in self.templates_rendered:
+                if context.get('shallow_tested'):
+                    self.shallow_tested_templates.add(template_name)
+                else:
+                    self.templates_rendered.add(template_name)
+                    self.shallow_tested_templates.discard(template_name)
+
+    def get_shallow_tested_templates(self):
+        return self.shallow_tested_templates
+
     def run_suite(self, suite, fatal_errors=None):
         failed = False
         for test in suite: