python: Replace functools.partial with type-safe returns.curry.partial.

The type annotation for functools.partial uses unchecked Any for all
the function parameters (both early and late).  returns.curry.partial
uses a mypy plugin to check the parameters safely.

https://returns.readthedocs.io/en/latest/pages/curry.html

Signed-off-by: Anders Kaseorg <anders@zulip.com>

pyproject.toml

@@ -42,7 +42,11 @@ warn_unreachable = true
 # with this behavior.
 local_partial_types = true
 
-plugins = ["mypy_django_plugin.main", "pydantic.mypy"]
+plugins = [
+    "mypy_django_plugin.main",
+    "pydantic.mypy",
+    "returns.contrib.mypy.returns_plugin",
+]
 
 [[tool.mypy.overrides]]
 module = [

requirements/common.in

@@ -8,6 +8,9 @@ Django[argon2]==4.2.*
 # needed for NotRequired, ParamSpec
 typing-extensions
 
+# For type-safe returns.curry.partial
+returns
+
 # Needed for rendering backend templates
 Jinja2
 

requirements/dev.txt

@@ -2442,6 +2442,10 @@ responses==0.23.3 \
     # via
     #   -r requirements/dev.in
     #   moto
+returns==0.22.0 \
+    --hash=sha256:c7bd85bd1e0041b44fe46c7e2f68fcc76a0546142c876229e395174bcd674f37 \
+    --hash=sha256:d38d6324692eeb29ec4bd698e1b859ec0ac79fb2c17bf0d302f92c8c42ef35c1
+    # via -r requirements/common.in
 rfc3339-validator==0.1.4 \
     --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \
     --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa
@@ -3059,6 +3063,7 @@ typing-extensions==4.7.1 \
     #   pyre-check
     #   pyre-extensions
     #   qrcode
+    #   returns
     #   rich
     #   semgrep
     #   sqlalchemy2-stubs

requirements/prod.txt

@@ -1921,6 +1921,10 @@ requests-oauthlib==1.3.1 \
     #   -r requirements/common.in
     #   python-twitter
     #   social-auth-core
+returns==0.22.0 \
+    --hash=sha256:c7bd85bd1e0041b44fe46c7e2f68fcc76a0546142c876229e395174bcd674f37 \
+    --hash=sha256:d38d6324692eeb29ec4bd698e1b859ec0ac79fb2c17bf0d302f92c8c42ef35c1
+    # via -r requirements/common.in
 rfc3339-validator==0.1.4 \
     --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \
     --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa
@@ -2173,6 +2177,7 @@ typing-extensions==4.7.1 \
     #   pydantic
     #   pydantic-core
     #   qrcode
+    #   returns
     #   stripe
     #   zulip
     #   zulip-bots

tools/semgrep.yml

@@ -250,3 +250,9 @@ rules:
     message: 'Use ".exists()" instead; it is more efficient'
     languages: [python]
     severity: ERROR
+
+  - id: functools-partial
+    pattern: functools.partial
+    message: "Replace functools.partial with returns.curry.partial for type safety"
+    languages: [python]
+    severity: ERROR

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 209
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (249, 0)
+PROVISION_VERSION = (249, 1)

zerver/data_import/import_util.py

@@ -4,7 +4,6 @@ import random
 import shutil
 from collections import defaultdict
 from concurrent.futures import ProcessPoolExecutor, as_completed
-from functools import partial
 from typing import (
     AbstractSet,
     Any,
@@ -25,6 +24,7 @@ import orjson
 import requests
 from django.forms.models import model_to_dict
 from django.utils.timezone import now as timezone_now
+from returns.curry import partial
 from typing_extensions import TypeAlias
 
 from zerver.data_import.sequencer import NEXT_ID

zerver/lib/test_runner.py

@@ -3,7 +3,6 @@ import os
 import random
 import shutil
 import unittest
-from functools import partial
 from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
 from unittest import TestSuite, runner
 from unittest.result import TestResult
@@ -14,6 +13,7 @@ from django.db import ProgrammingError, connections
 from django.test import runner as django_runner
 from django.test.runner import DiscoverRunner
 from django.test.signals import template_rendered
+from returns.curry import partial
 from typing_extensions import TypeAlias
 
 from scripts.lib.zulip_tools import (

zerver/views/video_calls.py

@@ -3,7 +3,6 @@ import json
 import random
 import secrets
 from base64 import b32encode
-from functools import partial
 from typing import Dict
 from urllib.parse import quote, urlencode, urljoin
 
@@ -21,6 +20,7 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
 from oauthlib.oauth2 import OAuth2Error
 from requests_oauthlib import OAuth2Session
+from returns.curry import partial
 
 from zerver.actions.video_calls import do_set_zoom_token
 from zerver.decorator import zulip_login_required

zerver/webhooks/bitbucket2/view.py

@@ -1,10 +1,10 @@
 # Webhooks for external integrations.
 import re
 import string
-from functools import partial
 from typing import Dict, List, Optional, Protocol
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import log_unsupported_webhook_event, webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/bitbucket3/view.py

@@ -1,8 +1,8 @@
 import string
-from functools import partial
 from typing import Dict, List, Optional, Protocol
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/clubhouse/view.py

@@ -1,7 +1,7 @@
-from functools import partial
 from typing import Callable, Dict, Iterable, Iterator, List, Optional
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/github/view.py

@@ -1,8 +1,8 @@
 import re
-from functools import partial
 from typing import Callable, Dict, Optional
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import log_unsupported_webhook_event, webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/gitlab/view.py

@@ -1,9 +1,9 @@
 import re
-from functools import partial
 from typing import Dict, List, Optional, Protocol, Union
 
 from django.http import HttpRequest, HttpResponse
 from pydantic import Json
+from returns.curry import partial
 
 from zerver.decorator import webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/groove/view.py

@@ -1,8 +1,8 @@
 # Webhooks for external integrations.
-from functools import partial
 from typing import Callable, Dict, Optional
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/webhooks/intercom/view.py

@@ -1,8 +1,8 @@
-from functools import partial
 from html.parser import HTMLParser
 from typing import Callable, Dict, List, Tuple
 
 from django.http import HttpRequest, HttpResponse
+from returns.curry import partial
 
 from zerver.decorator import return_success_on_head_request, webhook_view
 from zerver.lib.exceptions import UnsupportedWebhookEventTypeError

zerver/worker/queue_processors.py

@@ -4,7 +4,6 @@ import copy
 import datetime
 import email
 import email.policy
-import functools
 import logging
 import os
 import signal
@@ -43,6 +42,7 @@ from django.db.utils import IntegrityError
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
 from django.utils.translation import override as override_language
+from returns.curry import partial
 from sentry_sdk import add_breadcrumb, configure_scope
 from zulip_bots.lib import extract_query_without_mention
 
@@ -309,7 +309,7 @@ class QueueProcessingWorker(ABC):
                 try:
                     signal.signal(
                         signal.SIGALRM,
-                        functools.partial(self.timer_expired, self.MAX_CONSUME_SECONDS, events),
+                        partial(self.timer_expired, self.MAX_CONSUME_SECONDS, events),
                     )
                     try:
                         signal.alarm(self.MAX_CONSUME_SECONDS * len(events))
@@ -357,7 +357,7 @@ class QueueProcessingWorker(ABC):
         self.do_consume(consume_func, [event])
 
     def timer_expired(
-        self, limit: int, events: List[Dict[str, Any]], signal: int, frame: FrameType
+        self, limit: int, events: List[Dict[str, Any]], signal: int, frame: Optional[FrameType]
     ) -> None:
         raise WorkerTimeoutError(self.queue_name, limit, len(events))
 
@@ -911,7 +911,7 @@ class FetchLinksEmbedData(QueueProcessingWorker):
             do_update_embedded_data(message.sender, message, message.content, rendering_result)
 
     def timer_expired(
-        self, limit: int, events: List[Dict[str, Any]], signal: int, frame: FrameType
+        self, limit: int, events: List[Dict[str, Any]], signal: int, frame: Optional[FrameType]
     ) -> None:
         assert len(events) == 1
         event = events[0]