mypy: Add types-beautifulsoup4.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

pyproject.toml

@@ -51,7 +51,6 @@ module = [
     "aioapns.*",
     "bitfield.*",
     "bmemcached.*",
-    "bs4.*",
     "bson.*",
     "cairosvg.*",
     "circuitbreaker.*",

requirements/dev.txt

@@ -1926,6 +1926,10 @@ typeguard==2.13.3 \
     --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \
     --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1
     # via testslide
+types-beautifulsoup4==4.10.11 \
+    --hash=sha256:29d1c71b378e75c722bc63845dde8d69832f5fa398c8c1322a5fd144ff6df714 \
+    --hash=sha256:2fad472ea689a457c97098069ad133ddffe45c1f758bc81273b43b5c3a2258da
+    # via -r requirements/mypy.in
 types-boto==2.49.6 \
     --hash=sha256:c23e97e5bed847fce1578c7f07e14d278b6f18bb2085b24467a9a6520848acf7 \
     --hash=sha256:d3ac75943063d43108b46140143c7fa9ae9d3f7007633d7119b63f722e3e02b0

requirements/mypy.in

@@ -8,6 +8,7 @@ backoff-stubs
 boto3-stubs[s3]
 lxml-stubs
 sqlalchemy[mypy]
+types-beautifulsoup4
 types-boto
 types-certifi
 types-chardet

requirements/mypy.txt

@@ -184,6 +184,10 @@ typed-ast==1.5.1 \
     --hash=sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb \
     --hash=sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e
     # via mypy
+types-beautifulsoup4==4.10.11 \
+    --hash=sha256:29d1c71b378e75c722bc63845dde8d69832f5fa398c8c1322a5fd144ff6df714 \
+    --hash=sha256:2fad472ea689a457c97098069ad133ddffe45c1f758bc81273b43b5c3a2258da
+    # via -r requirements/mypy.in
 types-boto==2.49.6 \
     --hash=sha256:c23e97e5bed847fce1578c7f07e14d278b6f18bb2085b24467a9a6520848acf7 \
     --hash=sha256:d3ac75943063d43108b46140143c7fa9ae9d3f7007633d7119b63f722e3e02b0

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 113
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "171.0"
+PROVISION_VERSION = "172.0"

zerver/lib/url_preview/parsers/generic.py

@@ -1,5 +1,7 @@
 from typing import Dict, Optional
 
+from bs4.element import Tag
+
 from zerver.lib.url_preview.parsers.base import BaseParser
 
 
@@ -22,7 +24,8 @@ class GenericParser(BaseParser):
     def _get_description(self) -> Optional[str]:
         soup = self._soup
         meta_description = soup.find("meta", attrs={"name": "description"})
-        if meta_description and meta_description.get("content", "") != "":
+        if isinstance(meta_description, Tag) and meta_description.get("content", "") != "":
+            assert isinstance(meta_description["content"], str)
             return meta_description["content"]
         first_h1 = soup.find("h1")
         if first_h1:
@@ -43,6 +46,7 @@ class GenericParser(BaseParser):
         first_h1 = soup.find("h1")
         if first_h1:
             first_image = first_h1.find_next_sibling("img", src=True)
-            if first_image and first_image["src"] != "":
+            if isinstance(first_image, Tag) and first_image["src"] != "":
+                assert isinstance(first_image["src"], str)
                 return first_image["src"]
         return None

zerver/tests/test_auth_backends.py

@@ -18,6 +18,7 @@ import orjson
 import requests
 import responses
 from bs4 import BeautifulSoup
+from bs4.element import Tag
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from django.conf import settings
 from django.contrib.auth import authenticate
@@ -796,8 +797,13 @@ class DesktopFlowTestingLib(ZulipTestCase):
         self.assertEqual(response.status_code, 200)
 
         soup = BeautifulSoup(response.content, "html.parser")
-        desktop_data = soup.find("input", value=True)["value"]
-        browser_url = soup.find("a", href=True)["href"]
+        input = soup.find("input", value=True)
+        assert isinstance(input, Tag)
+        desktop_data = input["value"]
+        assert isinstance(desktop_data, str)
+        a = soup.find("a", href=True)
+        assert isinstance(a, Tag)
+        browser_url = a["href"]
 
         self.assertEqual(browser_url, "/login/")
         decrypted_key = self.verify_desktop_data_and_return_key(desktop_data, desktop_flow_otp)

zerver/tests/test_middleware.py

@@ -6,6 +6,7 @@ from bs4 import BeautifulSoup
 
 from zerver.lib.realm_icon import get_realm_icon_url
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.utils import assert_is_not_none
 from zerver.middleware import is_slow_query, write_log_line
 from zerver.models import get_realm
 
@@ -70,10 +71,14 @@ class OpenGraphTest(ZulipTestCase):
         response = self.client_get(path)
         self.assertEqual(response.status_code, status_code)
         bs = BeautifulSoup(response.content, features="lxml")
-        open_graph_title = bs.select_one('meta[property="og:title"]').get("content")
+        open_graph_title = assert_is_not_none(bs.select_one('meta[property="og:title"]')).get(
+            "content"
+        )
         self.assertEqual(open_graph_title, title)
 
-        open_graph_description = bs.select_one('meta[property="og:description"]').get("content")
+        open_graph_description = assert_is_not_none(
+            bs.select_one('meta[property="og:description"]')
+        ).get("content")
         for substring in in_description:
             self.assertIn(substring, open_graph_description)
         for substring in not_in_description:
@@ -195,7 +200,9 @@ class OpenGraphTest(ZulipTestCase):
         self.assertEqual(response.status_code, 200)
 
         bs = BeautifulSoup(response.content, features="lxml")
-        open_graph_image = bs.select_one('meta[property="og:image"]').get("content")
+        open_graph_image = assert_is_not_none(bs.select_one('meta[property="og:image"]')).get(
+            "content"
+        )
         self.assertEqual(open_graph_image, f"{realm.uri}{realm_icon}")
 
     def test_login_page_realm_icon_absolute_url(self) -> None:
@@ -210,7 +217,9 @@ class OpenGraphTest(ZulipTestCase):
         self.assertEqual(response.status_code, 200)
 
         bs = BeautifulSoup(response.content, features="lxml")
-        open_graph_image = bs.select_one('meta[property="og:image"]').get("content")
+        open_graph_image = assert_is_not_none(bs.select_one('meta[property="og:image"]')).get(
+            "content"
+        )
         self.assertEqual(open_graph_image, icon_url)
 
     def test_no_realm_api_page_og_url(self) -> None:
@@ -218,6 +227,7 @@ class OpenGraphTest(ZulipTestCase):
         self.assertEqual(response.status_code, 200)
 
         bs = BeautifulSoup(response.content, features="lxml")
-        open_graph_url = bs.select_one('meta[property="og:url"]').get("content")
+        open_graph_url = assert_is_not_none(bs.select_one('meta[property="og:url"]')).get("content")
 
+        assert isinstance(open_graph_url, str)
         self.assertTrue(open_graph_url.endswith("/api/"))