models: Extract zerver.models.linkifiers.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

tools/semgrep-py.yml

@@ -47,8 +47,8 @@ rules:
     patterns:
       - pattern-not: from zerver.lib.redis_utils import get_redis_client
       - pattern-not: from zerver.lib.utils import generate_api_key
-      - pattern-not: from zerver.models import filter_pattern_validator
+      - pattern-not: from zerver.models.linkifiers import filter_pattern_validator
-      - pattern-not: from zerver.models import url_template_validator
+      - pattern-not: from zerver.models.linkifiers import url_template_validator
       - pattern-not: from zerver.models import generate_email_token_for_stream
       - pattern-not: from zerver.models.realms import generate_realm_uuid_owner_secret
       - pattern-either:

zerver/actions/realm_linkifiers.py

@@ -7,14 +7,8 @@ from django.utils.translation import gettext as _
 
 from zerver.lib.exceptions import JsonableError
 from zerver.lib.types import LinkifierDict
-from zerver.models import (
+from zerver.models import Realm, RealmAuditLog, RealmFilter, UserProfile
-    Realm,
+from zerver.models.linkifiers import flush_linkifiers, linkifiers_for_realm
-    RealmAuditLog,
-    RealmFilter,
-    UserProfile,
-    flush_linkifiers,
-    linkifiers_for_realm,
-)
 from zerver.models.users import active_user_ids
 from zerver.tornado.django_api import send_event_on_commit
 

zerver/lib/events.py

@@ -82,9 +82,9 @@ from zerver.models import (
     custom_profile_fields_for_realm,
     get_default_stream_groups,
     get_realm_playgrounds,
-    linkifiers_for_realm,
 )
 from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.linkifiers import linkifiers_for_realm
 from zerver.models.realm_emoji import get_all_custom_emoji_for_realm
 from zerver.models.realms import get_realm_domains
 from zerver.tornado.django_api import get_user_events, request_event_queue

zerver/lib/markdown/__init__.py

@@ -74,7 +74,8 @@ from zerver.lib.timezone import common_timezones
 from zerver.lib.types import LinkifierDict
 from zerver.lib.url_encoding import encode_stream, hash_util_encode
 from zerver.lib.url_preview.types import UrlEmbedData, UrlOEmbedData
-from zerver.models import Message, Realm, linkifiers_for_realm
+from zerver.models import Message, Realm
+from zerver.models.linkifiers import linkifiers_for_realm
 from zerver.models.realm_emoji import EmojiInfo, get_name_keyed_dict_for_active_realm_emoji
 
 ReturnT = TypeVar("ReturnT")

zerver/management/commands/edit_linkifiers.py

@@ -7,7 +7,7 @@ from typing_extensions import override
 
 from zerver.actions.realm_linkifiers import do_add_linkifier, do_remove_linkifier
 from zerver.lib.management import ZulipBaseCommand
-from zerver.models import linkifiers_for_realm
+from zerver.models.linkifiers import linkifiers_for_realm
 
 
 class Command(ZulipBaseCommand):

zerver/migrations/0440_realmfilter_url_template.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations, models
 
-from zerver.models import url_template_validator
+from zerver.models.linkifiers import url_template_validator
 
 
 class Migration(migrations.Migration):

zerver/migrations/0442_remove_realmfilter_url_format_string.py

@@ -1,6 +1,6 @@
 from django.db import migrations, models
 
-from zerver.models import url_template_validator
+from zerver.models.linkifiers import url_template_validator
 
 
 class Migration(migrations.Migration):

zerver/migrations/0462_realmplayground_url_template.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations, models
 
-from zerver.models import url_template_validator
+from zerver.models.linkifiers import url_template_validator
 
 
 class Migration(migrations.Migration):

zerver/migrations/0464_remove_realmplayground_url_prefix.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations, models
 
-from zerver.models import url_template_validator
+from zerver.models.linkifiers import url_template_validator
 
 
 class Migration(migrations.Migration):

zerver/models/__init__.py

@@ -6,22 +6,9 @@ import secrets
 import time
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
-from typing import (
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypedDict, TypeVar, Union
-    Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Pattern,
-    Set,
-    Tuple,
-    TypedDict,
-    TypeVar,
-    Union,
-)
 
 import orjson
-import re2
 import uri_template
 from bitfield import BitField
 from bitfield.types import Bit, BitHandler
@@ -59,10 +46,7 @@ from zerver.lib.cache import (
     realm_alert_words_cache_key,
 )
 from zerver.lib.exceptions import RateLimitedError
-from zerver.lib.per_request_cache import (
+from zerver.lib.per_request_cache import return_same_value_during_entire_request
-    flush_per_request_cache,
-    return_same_value_during_entire_request,
-)
 from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.lib.types import (
     DefaultStreamDict,
@@ -70,7 +54,6 @@ from zerver.lib.types import (
     ExtendedValidator,
     FieldElement,
     GroupPermissionSetting,
-    LinkifierDict,
     ProfileDataElementBase,
     ProfileDataElementValue,
     RealmPlaygroundDict,
@@ -94,6 +77,8 @@ from zerver.models.groups import GroupGroupMembership as GroupGroupMembership
 from zerver.models.groups import SystemGroups
 from zerver.models.groups import UserGroup as UserGroup
 from zerver.models.groups import UserGroupMembership as UserGroupMembership
+from zerver.models.linkifiers import RealmFilter as RealmFilter
+from zerver.models.linkifiers import url_template_validator
 from zerver.models.realm_emoji import RealmEmoji as RealmEmoji
 from zerver.models.realms import Realm as Realm
 from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticationMethod
@@ -202,122 +187,6 @@ def get_recipient_ids(
     return to, recipient_type_str
 
 
-def filter_pattern_validator(value: str) -> Pattern[str]:
-    try:
-        # Do not write errors to stderr (this still raises exceptions)
-        options = re2.Options()
-        options.log_errors = False
-
-        regex = re2.compile(value, options=options)
-    except re2.error as e:
-        if len(e.args) >= 1:
-            if isinstance(e.args[0], str):  # nocoverage
-                raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0]))
-            if isinstance(e.args[0], bytes):
-                raise ValidationError(
-                    _("Bad regular expression: {regex}").format(regex=e.args[0].decode())
-                )
-        raise ValidationError(_("Unknown regular expression error"))  # nocoverage
-
-    return regex
-
-
-def url_template_validator(value: str) -> None:
-    """Validate as a URL template"""
-    if not uri_template.validate(value):
-        raise ValidationError(_("Invalid URL template."))
-
-
-class RealmFilter(models.Model):
-    """Realm-specific regular expressions to automatically linkify certain
-    strings inside the Markdown processor.  See "Custom filters" in the settings UI.
-    """
-
-    realm = models.ForeignKey(Realm, on_delete=CASCADE)
-    pattern = models.TextField()
-    url_template = models.TextField(validators=[url_template_validator])
-    # Linkifiers are applied in a message/topic in order; the processing order
-    # is important when there are overlapping patterns.
-    order = models.IntegerField(default=0)
-
-    class Meta:
-        unique_together = ("realm", "pattern")
-
-    @override
-    def __str__(self) -> str:
-        return f"{self.realm.string_id}: {self.pattern} {self.url_template}"
-
-    @override
-    def clean(self) -> None:
-        """Validate whether the set of parameters in the URL template
-        match the set of parameters in the regular expression.
-
-        Django's `full_clean` calls `clean_fields` followed by `clean` method
-        and stores all ValidationErrors from all stages to return as JSON.
-        """
-
-        # Extract variables present in the pattern
-        pattern = filter_pattern_validator(self.pattern)
-        group_set = set(pattern.groupindex.keys())
-
-        # Do not continue the check if the url template is invalid to begin with.
-        # The ValidationError for invalid template will only be raised by the validator
-        # set on the url_template field instead of here to avoid duplicates.
-        if not uri_template.validate(self.url_template):
-            return
-
-        # Extract variables used in the URL template.
-        template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names)
-
-        # Report patterns missing in linkifier pattern.
-        missing_in_pattern_set = template_variables_set - group_set
-        if len(missing_in_pattern_set) > 0:
-            name = min(missing_in_pattern_set)
-            raise ValidationError(
-                _("Group %(name)r in URL template is not present in linkifier pattern."),
-                params={"name": name},
-            )
-
-        missing_in_url_set = group_set - template_variables_set
-        # Report patterns missing in URL template.
-        if len(missing_in_url_set) > 0:
-            # We just report the first missing pattern here. Users can
-            # incrementally resolve errors if there are multiple
-            # missing patterns.
-            name = min(missing_in_url_set)
-            raise ValidationError(
-                _("Group %(name)r in linkifier pattern is not present in URL template."),
-                params={"name": name},
-            )
-
-
-def get_linkifiers_cache_key(realm_id: int) -> str:
-    return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}"
-
-
-@return_same_value_during_entire_request
-@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7)
-def linkifiers_for_realm(realm_id: int) -> List[LinkifierDict]:
-    return [
-        LinkifierDict(
-            pattern=linkifier.pattern,
-            url_template=linkifier.url_template,
-            id=linkifier.id,
-        )
-        for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order")
-    ]
-
-
-def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None:
-    realm_id = instance.realm_id
-    cache_delete(get_linkifiers_cache_key(realm_id))
-    flush_per_request_cache("linkifiers_for_realm")
-
-
-post_save.connect(flush_linkifiers, sender=RealmFilter)
-post_delete.connect(flush_linkifiers, sender=RealmFilter)
-
-
 class RealmPlayground(models.Model):
     """Server side storage model to store playground information needed by our
     'view code in playground' feature in code blocks.

zerver/models/linkifiers.py

@@ -0,0 +1,135 @@
+from typing import List, Pattern
+
+import re2
+import uri_template
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.signals import post_delete, post_save
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.lib import cache
+from zerver.lib.cache import cache_delete, cache_with_key
+from zerver.lib.per_request_cache import (
+    flush_per_request_cache,
+    return_same_value_during_entire_request,
+)
+from zerver.lib.types import LinkifierDict
+from zerver.models.realms import Realm
+
+
+def filter_pattern_validator(value: str) -> Pattern[str]:
+    try:
+        # Do not write errors to stderr (this still raises exceptions)
+        options = re2.Options()
+        options.log_errors = False
+
+        regex = re2.compile(value, options=options)
+    except re2.error as e:
+        if len(e.args) >= 1:
+            if isinstance(e.args[0], str):  # nocoverage
+                raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0]))
+            if isinstance(e.args[0], bytes):
+                raise ValidationError(
+                    _("Bad regular expression: {regex}").format(regex=e.args[0].decode())
+                )
+        raise ValidationError(_("Unknown regular expression error"))  # nocoverage
+
+    return regex
+
+
+def url_template_validator(value: str) -> None:
+    """Validate as a URL template"""
+    if not uri_template.validate(value):
+        raise ValidationError(_("Invalid URL template."))
+
+
+class RealmFilter(models.Model):
+    """Realm-specific regular expressions to automatically linkify certain
+    strings inside the Markdown processor.  See "Custom filters" in the settings UI.
+    """
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    pattern = models.TextField()
+    url_template = models.TextField(validators=[url_template_validator])
+    # Linkifiers are applied in a message/topic in order; the processing order
+    # is important when there are overlapping patterns.
+    order = models.IntegerField(default=0)
+
+    class Meta:
+        unique_together = ("realm", "pattern")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm.string_id}: {self.pattern} {self.url_template}"
+
+    @override
+    def clean(self) -> None:
+        """Validate whether the set of parameters in the URL template
+        match the set of parameters in the regular expression.
+
+        Django's `full_clean` calls `clean_fields` followed by `clean` method
+        and stores all ValidationErrors from all stages to return as JSON.
+        """
+
+        # Extract variables present in the pattern
+        pattern = filter_pattern_validator(self.pattern)
+        group_set = set(pattern.groupindex.keys())
+
+        # Do not continue the check if the url template is invalid to begin with.
+        # The ValidationError for invalid template will only be raised by the validator
+        # set on the url_template field instead of here to avoid duplicates.
+        if not uri_template.validate(self.url_template):
+            return
+
+        # Extract variables used in the URL template.
+        template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names)
+
+        # Report patterns missing in linkifier pattern.
+        missing_in_pattern_set = template_variables_set - group_set
+        if len(missing_in_pattern_set) > 0:
+            name = min(missing_in_pattern_set)
+            raise ValidationError(
+                _("Group %(name)r in URL template is not present in linkifier pattern."),
+                params={"name": name},
+            )
+
+        missing_in_url_set = group_set - template_variables_set
+        # Report patterns missing in URL template.
+        if len(missing_in_url_set) > 0:
+            # We just report the first missing pattern here. Users can
+            # incrementally resolve errors if there are multiple
+            # missing patterns.
+            name = min(missing_in_url_set)
+            raise ValidationError(
+                _("Group %(name)r in linkifier pattern is not present in URL template."),
+                params={"name": name},
+            )
+
+
+def get_linkifiers_cache_key(realm_id: int) -> str:
+    return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}"
+
+
+@return_same_value_during_entire_request
+@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7)
+def linkifiers_for_realm(realm_id: int) -> List[LinkifierDict]:
+    return [
+        LinkifierDict(
+            pattern=linkifier.pattern,
+            url_template=linkifier.url_template,
+            id=linkifier.id,
+        )
+        for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order")
+    ]
+
+
+def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None:
+    realm_id = instance.realm_id
+    cache_delete(get_linkifiers_cache_key(realm_id))
+    flush_per_request_cache("linkifiers_for_realm")
+
+
+post_save.connect(flush_linkifiers, sender=RealmFilter)
+post_delete.connect(flush_linkifiers, sender=RealmFilter)

zerver/tests/test_audit_log.py

@@ -83,9 +83,9 @@ from zerver.models import (
     UserProfile,
     get_realm_playgrounds,
     get_stream,
-    linkifiers_for_realm,
 )
 from zerver.models.groups import SystemGroups
+from zerver.models.linkifiers import linkifiers_for_realm
 from zerver.models.realm_emoji import EmojiInfo, get_all_custom_emoji_for_realm
 from zerver.models.realms import RealmDomainDict, get_realm, get_realm_domains
 

zerver/tests/test_markdown.py

@@ -69,9 +69,9 @@ from zerver.models import (
     UserProfile,
     get_client,
     get_stream,
-    linkifiers_for_realm,
 )
 from zerver.models.groups import SystemGroups
+from zerver.models.linkifiers import linkifiers_for_realm
 from zerver.models.realms import get_realm
 
 

zerver/tests/test_realm_linkifiers.py

@@ -6,7 +6,8 @@ from django.core.exceptions import ValidationError
 from typing_extensions import override
 
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.models import RealmAuditLog, RealmFilter, url_template_validator
+from zerver.models import RealmAuditLog, RealmFilter
+from zerver.models.linkifiers import url_template_validator
 
 
 class RealmFilterTest(ZulipTestCase):

zerver/views/realm_linkifiers.py

@@ -15,7 +15,8 @@ from zerver.lib.exceptions import JsonableError, ValidationFailureError
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
 from zerver.lib.validator import check_int, check_list
-from zerver.models import RealmFilter, UserProfile, linkifiers_for_realm
+from zerver.models import RealmFilter, UserProfile
+from zerver.models.linkifiers import linkifiers_for_realm
 
 
 # Custom realm linkifiers