requirements: Add django-stubs and configure plugin.

Note that django_stubs_ext is required to be placed within common.in
because we need the monkeypatched types in runtime; django-stubs
itself is for type checking only.

In the future, we would like to pin to a release instead of a git
revision, but several patches we've contributed upstream have not
appeared in a release yet.

We also remove the type annotation for RealmAuditLog.event_last_message_id
here instead of earlier because type checking fails otherwise.

Fixes #11560.

pyproject.toml

@@ -42,8 +42,10 @@ warn_unreachable = true
 # with this behavior.
 local_partial_types = true
 
+plugins = ["mypy_django_plugin.main"]
+
 [[tool.mypy.overrides]]
-module = ["zproject.configured_settings", "zproject.settings"]
+module = ["zproject.configured_settings", "zproject.settings", "zproject.default_settings"]
 no_implicit_reexport = false
 
 [[tool.mypy.overrides]]
@@ -59,7 +61,6 @@ module = [
     "defusedxml.*", # https://github.com/tiran/defusedxml/issues/46
     "digitalocean.*",
     "disposable_email_domains.*",
-    "django.*", # https://github.com/zulip/zulip/issues/11560
     "django_auth_ldap.*",
     "django_bmemcached.*",
     "django_cte.*",
@@ -93,3 +94,6 @@ module = [
     "two_factor.*",
 ]
 ignore_missing_imports = true
+
+[tool.django-stubs]
+django_settings_module = "zproject.settings"

requirements/common.in

@@ -194,3 +194,6 @@ soupsieve
 
 # Circuit-breaking for outgoing services
 circuitbreaker
+
+# Runtime monkeypatching of django-stubs generics
+https://github.com/typeddjango/django-stubs/archive/9bd8aed1e19f9da2c7d3a2879367a40847b57dea.zip#egg=django-stubs-ext==0.5.0+git&subdirectory=django_stubs_ext

requirements/dev.txt

@@ -466,6 +466,8 @@ django[argon2]==4.0.7 \
     #   django-phonenumber-field
     #   django-scim2
     #   django-sendfile2
+    #   django-stubs
+    #   django-stubs-ext
     #   django-two-factor-auth
 django-auth-ldap==4.1.0 \
     --hash=sha256:68870e7921e84b1a9867e268a9c8a3e573e8a0d95ea08bcf31be178f5826ff36 \
@@ -504,6 +506,14 @@ django-statsd-mozilla==0.4.0 \
     --hash=sha256:0d87cb63de8107279cbb748caad9aa74c6a44e7e96ccc5dbf07b89f77285a4b8 \
     --hash=sha256:81084f3d426f5184f0a0f1dbfe035cc26b66f041d2184559d916a228d856f0d3
     # via -r requirements/common.in
+https://github.com/typeddjango/django-stubs/archive/9bd8aed1e19f9da2c7d3a2879367a40847b57dea.zip#egg=django-stubs==1.12.0+git \
+    --hash=sha256:42340195b7e67a035f2399cf2718b0990ca40addf3ec2f0ac213887d7ef32c7b
+    # via -r requirements/mypy.in
+https://github.com/typeddjango/django-stubs/archive/9bd8aed1e19f9da2c7d3a2879367a40847b57dea.zip#egg=django-stubs-ext==0.5.0+git&subdirectory=django_stubs_ext \
+    --hash=sha256:42340195b7e67a035f2399cf2718b0990ca40addf3ec2f0ac213887d7ef32c7b
+    # via
+    #   -r requirements/common.in
+    #   django-stubs
 django-two-factor-auth[call,phonenumberslite,sms]==1.14.0 \
     --hash=sha256:b414ef51cc84335e0049e3f98ef89ac15a09187efa381f3acd321651afae95b3 \
     --hash=sha256:d18290f02766f400537b88937df06aa0a2d6d6d37937fbfb84896c4790a9e59b
@@ -1049,6 +1059,7 @@ mypy==0.971 \
     --hash=sha256:f2899a3cbd394da157194f913a931edfd4be5f274a88041c9dc2d9cdcb1c315c
     # via
     #   -r requirements/mypy.in
+    #   django-stubs
     #   sqlalchemy
 mypy-boto3-s3==1.24.36.post1 \
     --hash=sha256:30ae59b33c55f8b7b693170f9519ea5b91a2fbf31a73de79cdef57a27d784e5a \
@@ -2043,6 +2054,7 @@ tomli==2.0.1 \
     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
     # via
     #   black
+    #   django-stubs
     #   mypy
     #   pep517
 tornado==6.2 \
@@ -2156,10 +2168,16 @@ types-python-dateutil==2.8.19 \
     --hash=sha256:6284df1e4783d8fc6e587f0317a81333856b872a6669a282f8a325342bce7fa8 \
     --hash=sha256:bfd3eb39c7253aea4ba23b10f69b017d30b013662bb4be4ab48b20bbd763f309
     # via -r requirements/mypy.in
+types-pytz==2022.2.1.0 \
+    --hash=sha256:47cfb19c52b9f75896440541db392fd312a35b279c6307a531db71152ea63e2b \
+    --hash=sha256:50ead2254b524a3d4153bc65d00289b66898060d2938e586170dce918dbaf3b3
+    # via django-stubs
 types-pyyaml==6.0.11 \
     --hash=sha256:7f7da2fd11e9bc1e5e9eb3ea1be84f4849747017a59fc2eee0ea34ed1147c2e0 \
     --hash=sha256:8f890028123607379c63550179ddaec4517dc751f4c527a52bb61934bf495989
-    # via -r requirements/mypy.in
+    # via
+    #   -r requirements/mypy.in
+    #   django-stubs
 types-redis==4.3.19 \
     --hash=sha256:c74262197487a65e3f02db37d3f0e0f3bb2d5d370f7e6ba390814e865a45e56e \
     --hash=sha256:c881ffb94bd47dca21ed503328544a56bb315851575eebb27fb371bff735844a
@@ -2201,6 +2219,8 @@ typing-extensions==4.3.0 \
     #   -r requirements/common.in
     #   black
     #   boto3-stubs
+    #   django-stubs
+    #   django-stubs-ext
     #   libcst
     #   mypy
     #   mypy-boto3-s3

requirements/mypy.in

@@ -30,3 +30,5 @@ types-stripe
 types-zxcvbn
 
 importlib-metadata ; python_version < "3.10"  # for SQLAlchemy
+
+https://github.com/typeddjango/django-stubs/archive/9bd8aed1e19f9da2c7d3a2879367a40847b57dea.zip#egg=django-stubs==1.12.0+git

requirements/prod.txt

@@ -304,6 +304,7 @@ django[argon2]==4.0.7 \
     #   django-phonenumber-field
     #   django-scim2
     #   django-sendfile2
+    #   django-stubs-ext
     #   django-two-factor-auth
 django-auth-ldap==4.1.0 \
     --hash=sha256:68870e7921e84b1a9867e268a9c8a3e573e8a0d95ea08bcf31be178f5826ff36 \
@@ -342,6 +343,9 @@ django-statsd-mozilla==0.4.0 \
     --hash=sha256:0d87cb63de8107279cbb748caad9aa74c6a44e7e96ccc5dbf07b89f77285a4b8 \
     --hash=sha256:81084f3d426f5184f0a0f1dbfe035cc26b66f041d2184559d916a228d856f0d3
     # via -r requirements/common.in
+https://github.com/typeddjango/django-stubs/archive/9bd8aed1e19f9da2c7d3a2879367a40847b57dea.zip#egg=django-stubs-ext==0.5.0+git&subdirectory=django_stubs_ext \
+    --hash=sha256:42340195b7e67a035f2399cf2718b0990ca40addf3ec2f0ac213887d7ef32c7b
+    # via -r requirements/common.in
 django-two-factor-auth[call,phonenumberslite,sms]==1.14.0 \
     --hash=sha256:b414ef51cc84335e0049e3f98ef89ac15a09187efa381f3acd321651afae95b3 \
     --hash=sha256:d18290f02766f400537b88937df06aa0a2d6d6d37937fbfb84896c4790a9e59b
@@ -1391,6 +1395,7 @@ typing-extensions==4.3.0 \
     --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6
     # via
     #   -r requirements/common.in
+    #   django-stubs-ext
     #   mypy-boto3-s3
     #   mypy-boto3-ses
     #   mypy-boto3-sns

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 150
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (202, 0)
+PROVISION_VERSION = (203, 0)

zerver/lib/test_helpers.py

@@ -295,6 +295,14 @@ class HostRequestMock(HttpRequest):
     """A mock request object where get_host() works.  Useful for testing
     routes that use Zulip's subdomains feature"""
 
+    # The base class HttpRequest declares GET and POST as immutable
+    # QueryDict objects. The implementation of HostRequestMock
+    # requires POST to be mutable, and we have some use cases that
+    # modify GET, so GET and POST are both redeclared as mutable.
+
+    GET: QueryDict  # type: ignore[assignment] # See previous comment.
+    POST: QueryDict  # type: ignore[assignment] # See previous comment.
+
     def __init__(
         self,
         post_data: Dict[str, Any] = {},

zerver/models.py

@@ -132,7 +132,7 @@ class EmojiInfo(TypedDict):
 
 
 @models.Field.register_lookup
-class AndZero(models.Lookup):
+class AndZero(models.Lookup[int]):
     lookup_name = "andz"
 
     def as_sql(
@@ -144,7 +144,7 @@ class AndZero(models.Lookup):
 
 
 @models.Field.register_lookup
-class AndNonZero(models.Lookup):
+class AndNonZero(models.Lookup[int]):
     lookup_name = "andnz"
 
     def as_sql(
@@ -247,7 +247,7 @@ def clear_supported_auth_backends_cache() -> None:
     supported_backends = None
 
 
-class Realm(models.Model):
+class Realm(models.Model):  # type: ignore[django-manager-missing] # django-stubs cannot resolve the custom CTEManager yet https://github.com/typeddjango/django-stubs/issues/1023
     MAX_REALM_NAME_LENGTH = 40
     MAX_REALM_DESCRIPTION_LENGTH = 1000
     MAX_REALM_SUBDOMAIN_LENGTH = 40
@@ -1689,7 +1689,7 @@ class RealmUserDefault(UserBaseSettings):
     realm = models.OneToOneField(Realm, on_delete=CASCADE)
 
 
-class UserProfile(AbstractBaseUser, PermissionsMixin, UserBaseSettings):
+class UserProfile(AbstractBaseUser, PermissionsMixin, UserBaseSettings):  # type: ignore[django-manager-missing] # django-stubs cannot resolve the custom CTEManager yet https://github.com/typeddjango/django-stubs/issues/1023
     USERNAME_FIELD = "email"
     MAX_NAME_LENGTH = 100
     MIN_NAME_LENGTH = 2
@@ -2160,7 +2160,7 @@ class PasswordTooWeakError(Exception):
     pass
 
 
-class UserGroup(models.Model):
+class UserGroup(models.Model):  # type: ignore[django-manager-missing] # django-stubs cannot resolve the custom CTEManager yet https://github.com/typeddjango/django-stubs/issues/1023
     objects: CTEManager = CTEManager()
     id = models.AutoField(auto_created=True, primary_key=True, verbose_name="ID")
     name = models.CharField(max_length=100)
@@ -4450,7 +4450,7 @@ class RealmAuditLog(AbstractRealmAuditLog):
         null=True,
         on_delete=CASCADE,
     )
-    event_last_message_id: Optional[int] = models.IntegerField(null=True)
+    event_last_message_id = models.IntegerField(null=True)
 
     def __str__(self) -> str:
         if self.modified_user is not None:

zproject/backends.py

@@ -1402,7 +1402,7 @@ class ZulipRemoteUserBackend(RemoteUserBackend, ExternalAuthMethod):
 
     create_unknown_user = False
 
-    def authenticate(
+    def authenticate(  # type: ignore[override] # authenticate has an incompatible signature with ModelBackend and BaseBackend
         self,
         request: Optional[HttpRequest] = None,
         *,

zproject/settings.py

@@ -15,6 +15,13 @@
 # See https://zulip.readthedocs.io/en/latest/subsystems/settings.html for more information
 #
 ########################################################################
+import django_stubs_ext
+
+# Monkey-patch certain types that are declared as generic types
+# generic in django-stubs, but not (yet) as generic types in Django
+# itself. This is necessary to ensure type references like
+# django.db.models.Lookup[int] work correctly at runtime.
+django_stubs_ext.monkeypatch()
 
 from .configured_settings import *  # noqa: F401,F403 isort: skip
 from .computed_settings import *  # noqa: F401,F403 isort: skip