doc tests: Create dedicated zephyr test.

This did not speed up the tests as much as I expected,
but it certainly makes the code easier to read, and
Tim is pretty confident that the zephyr logic is
fairly stable, so it's sufficient to test it on a
subset of representative urls.

zerver/tests/test_docs.py

@@ -84,12 +84,7 @@ class DocPageTest(ZulipTestCase):
             )
         return result
 
-    def _test_normal_path(
-        self,
-        *,
-        url: str,
-        expected_strings: Sequence[str],
-    ) -> None:
+    def _test(self, url: str, expected_strings: Sequence[str]) -> None:
         # Test the URL on the root subdomain
         self._check_basic_fetch(
             url=url,
@@ -123,46 +118,41 @@ class DocPageTest(ZulipTestCase):
                     result,
                 )
 
-    def _test_zephyr_path(
-        self,
-        *,
-        url: str,
-        expected_strings: Sequence[str],
-    ) -> None:
-        # Test the URL on the "zephyr" subdomain
+    def test_zephyr_disallows_robots(self) -> None:
+        sample_urls = [
+            "/apps/",
+            "/case-studies/end-point/",
+            "/communities/",
+            "/devlogin/",
+            "/devtools/",
+            "/emails/",
+            "/errors/404/",
+            "/errors/5xx/",
+            "/integrations/",
+            "/integrations/",
+            "/integrations/bots",
+            "/integrations/doc-html/asana",
+            "/integrations/doc/github",
+            "/team/",
+        ]
+
+        for url in sample_urls:
             self._check_basic_fetch(
                 url=url,
                 subdomain="zephyr",
-            expected_strings=expected_strings,
+                expected_strings=[],
                 allow_robots=False,
             )
 
-        if not self._is_landing_page(url):
-            return
-
+            if self._is_landing_page(url):
                 with self.settings(ROOT_DOMAIN_LANDING_PAGE=True):
-            # Test the URL on the "zephyr" subdomain with the landing page setting
                     self._check_basic_fetch(
                         url=url,
                         subdomain="zephyr",
-                expected_strings=expected_strings,
+                        expected_strings=[],
                         allow_robots=False,
                     )
 
-    def _test(
-        self,
-        url: str,
-        expected_strings: Sequence[str] = [],
-    ) -> None:
-        self._test_normal_path(
-            url=url,
-            expected_strings=expected_strings,
-        )
-        self._test_zephyr_path(
-            url=url,
-            expected_strings=expected_strings,
-        )
-
     def test_api_doc_endpoints(self) -> None:
         # We extract the set of /api/ endpoints to check by parsing
         # the /api/ page sidebar for links starting with /api/.
@@ -215,7 +205,7 @@ class DocPageTest(ZulipTestCase):
                 # TODO: Just fill out dictionary for all ~110 endpoints
                 expected_strings = []
 
-            self._test_normal_path(
+            self._test(
                 url=url,
                 expected_strings=expected_strings,
             )