diff --git a/tools/semgrep-py.yml b/tools/semgrep-py.yml index dad2e7bf91..3fd1030545 100644 --- a/tools/semgrep-py.yml +++ b/tools/semgrep-py.yml @@ -47,8 +47,8 @@ rules: patterns: - pattern-not: from zerver.lib.redis_utils import get_redis_client - pattern-not: from zerver.lib.utils import generate_api_key - - pattern-not: from zerver.models import filter_pattern_validator - - pattern-not: from zerver.models import url_template_validator + - pattern-not: from zerver.models.linkifiers import filter_pattern_validator + - pattern-not: from zerver.models.linkifiers import url_template_validator - pattern-not: from zerver.models import generate_email_token_for_stream - pattern-not: from zerver.models.realms import generate_realm_uuid_owner_secret - pattern-either: diff --git a/zerver/actions/realm_linkifiers.py b/zerver/actions/realm_linkifiers.py index a7f1e4315f..c5f243d0a8 100644 --- a/zerver/actions/realm_linkifiers.py +++ b/zerver/actions/realm_linkifiers.py @@ -7,14 +7,8 @@ from django.utils.translation import gettext as _ from zerver.lib.exceptions import JsonableError from zerver.lib.types import LinkifierDict -from zerver.models import ( - Realm, - RealmAuditLog, - RealmFilter, - UserProfile, - flush_linkifiers, - linkifiers_for_realm, -) +from zerver.models import Realm, RealmAuditLog, RealmFilter, UserProfile +from zerver.models.linkifiers import flush_linkifiers, linkifiers_for_realm from zerver.models.users import active_user_ids from zerver.tornado.django_api import send_event_on_commit diff --git a/zerver/lib/events.py b/zerver/lib/events.py index a6924acec2..4aaff68462 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -82,9 +82,9 @@ from zerver.models import ( custom_profile_fields_for_realm, get_default_stream_groups, get_realm_playgrounds, - linkifiers_for_realm, ) from zerver.models.constants import MAX_TOPIC_NAME_LENGTH +from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realm_emoji import get_all_custom_emoji_for_realm from zerver.models.realms import get_realm_domains from zerver.tornado.django_api import get_user_events, request_event_queue diff --git a/zerver/lib/markdown/__init__.py b/zerver/lib/markdown/__init__.py index 2eb2d81891..b3ff0cd630 100644 --- a/zerver/lib/markdown/__init__.py +++ b/zerver/lib/markdown/__init__.py @@ -74,7 +74,8 @@ from zerver.lib.timezone import common_timezones from zerver.lib.types import LinkifierDict from zerver.lib.url_encoding import encode_stream, hash_util_encode from zerver.lib.url_preview.types import UrlEmbedData, UrlOEmbedData -from zerver.models import Message, Realm, linkifiers_for_realm +from zerver.models import Message, Realm +from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realm_emoji import EmojiInfo, get_name_keyed_dict_for_active_realm_emoji ReturnT = TypeVar("ReturnT") diff --git a/zerver/management/commands/edit_linkifiers.py b/zerver/management/commands/edit_linkifiers.py index 233a63ba03..819f5dc0bf 100644 --- a/zerver/management/commands/edit_linkifiers.py +++ b/zerver/management/commands/edit_linkifiers.py @@ -7,7 +7,7 @@ from typing_extensions import override from zerver.actions.realm_linkifiers import do_add_linkifier, do_remove_linkifier from zerver.lib.management import ZulipBaseCommand -from zerver.models import linkifiers_for_realm +from zerver.models.linkifiers import linkifiers_for_realm class Command(ZulipBaseCommand): diff --git a/zerver/migrations/0440_realmfilter_url_template.py b/zerver/migrations/0440_realmfilter_url_template.py index 6f714d47de..15d4913c88 100644 --- a/zerver/migrations/0440_realmfilter_url_template.py +++ b/zerver/migrations/0440_realmfilter_url_template.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from zerver.models import url_template_validator +from zerver.models.linkifiers import url_template_validator class Migration(migrations.Migration): diff --git a/zerver/migrations/0442_remove_realmfilter_url_format_string.py b/zerver/migrations/0442_remove_realmfilter_url_format_string.py index e1256e5dc5..be7c6df8a9 100644 --- a/zerver/migrations/0442_remove_realmfilter_url_format_string.py +++ b/zerver/migrations/0442_remove_realmfilter_url_format_string.py @@ -1,6 +1,6 @@ from django.db import migrations, models -from zerver.models import url_template_validator +from zerver.models.linkifiers import url_template_validator class Migration(migrations.Migration): diff --git a/zerver/migrations/0462_realmplayground_url_template.py b/zerver/migrations/0462_realmplayground_url_template.py index 4b17a43ed4..72515b06f9 100644 --- a/zerver/migrations/0462_realmplayground_url_template.py +++ b/zerver/migrations/0462_realmplayground_url_template.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from zerver.models import url_template_validator +from zerver.models.linkifiers import url_template_validator class Migration(migrations.Migration): diff --git a/zerver/migrations/0464_remove_realmplayground_url_prefix.py b/zerver/migrations/0464_remove_realmplayground_url_prefix.py index 3b5358ff04..05e61aae59 100644 --- a/zerver/migrations/0464_remove_realmplayground_url_prefix.py +++ b/zerver/migrations/0464_remove_realmplayground_url_prefix.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from zerver.models import url_template_validator +from zerver.models.linkifiers import url_template_validator class Migration(migrations.Migration): diff --git a/zerver/models/__init__.py b/zerver/models/__init__.py index 587e73bfd4..101adf9275 100644 --- a/zerver/models/__init__.py +++ b/zerver/models/__init__.py @@ -6,22 +6,9 @@ import secrets import time from collections import defaultdict from datetime import datetime, timedelta, timezone -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Pattern, - Set, - Tuple, - TypedDict, - TypeVar, - Union, -) +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypedDict, TypeVar, Union import orjson -import re2 import uri_template from bitfield import BitField from bitfield.types import Bit, BitHandler @@ -59,10 +46,7 @@ from zerver.lib.cache import ( realm_alert_words_cache_key, ) from zerver.lib.exceptions import RateLimitedError -from zerver.lib.per_request_cache import ( - flush_per_request_cache, - return_same_value_during_entire_request, -) +from zerver.lib.per_request_cache import return_same_value_during_entire_request from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.types import ( DefaultStreamDict, @@ -70,7 +54,6 @@ from zerver.lib.types import ( ExtendedValidator, FieldElement, GroupPermissionSetting, - LinkifierDict, ProfileDataElementBase, ProfileDataElementValue, RealmPlaygroundDict, @@ -94,6 +77,8 @@ from zerver.models.groups import GroupGroupMembership as GroupGroupMembership from zerver.models.groups import SystemGroups from zerver.models.groups import UserGroup as UserGroup from zerver.models.groups import UserGroupMembership as UserGroupMembership +from zerver.models.linkifiers import RealmFilter as RealmFilter +from zerver.models.linkifiers import url_template_validator from zerver.models.realm_emoji import RealmEmoji as RealmEmoji from zerver.models.realms import Realm as Realm from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticationMethod @@ -202,122 +187,6 @@ def get_recipient_ids( return to, recipient_type_str -def filter_pattern_validator(value: str) -> Pattern[str]: - try: - # Do not write errors to stderr (this still raises exceptions) - options = re2.Options() - options.log_errors = False - - regex = re2.compile(value, options=options) - except re2.error as e: - if len(e.args) >= 1: - if isinstance(e.args[0], str): # nocoverage - raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0])) - if isinstance(e.args[0], bytes): - raise ValidationError( - _("Bad regular expression: {regex}").format(regex=e.args[0].decode()) - ) - raise ValidationError(_("Unknown regular expression error")) # nocoverage - - return regex - - -def url_template_validator(value: str) -> None: - """Validate as a URL template""" - if not uri_template.validate(value): - raise ValidationError(_("Invalid URL template.")) - - -class RealmFilter(models.Model): - """Realm-specific regular expressions to automatically linkify certain - strings inside the Markdown processor. See "Custom filters" in the settings UI. - """ - - realm = models.ForeignKey(Realm, on_delete=CASCADE) - pattern = models.TextField() - url_template = models.TextField(validators=[url_template_validator]) - # Linkifiers are applied in a message/topic in order; the processing order - # is important when there are overlapping patterns. - order = models.IntegerField(default=0) - - class Meta: - unique_together = ("realm", "pattern") - - @override - def __str__(self) -> str: - return f"{self.realm.string_id}: {self.pattern} {self.url_template}" - - @override - def clean(self) -> None: - """Validate whether the set of parameters in the URL template - match the set of parameters in the regular expression. - - Django's `full_clean` calls `clean_fields` followed by `clean` method - and stores all ValidationErrors from all stages to return as JSON. - """ - - # Extract variables present in the pattern - pattern = filter_pattern_validator(self.pattern) - group_set = set(pattern.groupindex.keys()) - - # Do not continue the check if the url template is invalid to begin with. - # The ValidationError for invalid template will only be raised by the validator - # set on the url_template field instead of here to avoid duplicates. - if not uri_template.validate(self.url_template): - return - - # Extract variables used in the URL template. - template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names) - - # Report patterns missing in linkifier pattern. - missing_in_pattern_set = template_variables_set - group_set - if len(missing_in_pattern_set) > 0: - name = min(missing_in_pattern_set) - raise ValidationError( - _("Group %(name)r in URL template is not present in linkifier pattern."), - params={"name": name}, - ) - - missing_in_url_set = group_set - template_variables_set - # Report patterns missing in URL template. - if len(missing_in_url_set) > 0: - # We just report the first missing pattern here. Users can - # incrementally resolve errors if there are multiple - # missing patterns. - name = min(missing_in_url_set) - raise ValidationError( - _("Group %(name)r in linkifier pattern is not present in URL template."), - params={"name": name}, - ) - - -def get_linkifiers_cache_key(realm_id: int) -> str: - return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}" - - -@return_same_value_during_entire_request -@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7) -def linkifiers_for_realm(realm_id: int) -> List[LinkifierDict]: - return [ - LinkifierDict( - pattern=linkifier.pattern, - url_template=linkifier.url_template, - id=linkifier.id, - ) - for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order") - ] - - -def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None: - realm_id = instance.realm_id - cache_delete(get_linkifiers_cache_key(realm_id)) - flush_per_request_cache("linkifiers_for_realm") - - -post_save.connect(flush_linkifiers, sender=RealmFilter) -post_delete.connect(flush_linkifiers, sender=RealmFilter) - - class RealmPlayground(models.Model): """Server side storage model to store playground information needed by our 'view code in playground' feature in code blocks. diff --git a/zerver/models/linkifiers.py b/zerver/models/linkifiers.py new file mode 100644 index 0000000000..fca07e4af6 --- /dev/null +++ b/zerver/models/linkifiers.py @@ -0,0 +1,135 @@ +from typing import List, Pattern + +import re2 +import uri_template +from django.core.exceptions import ValidationError +from django.db import models +from django.db.models import CASCADE +from django.db.models.signals import post_delete, post_save +from django.utils.translation import gettext as _ +from typing_extensions import override + +from zerver.lib import cache +from zerver.lib.cache import cache_delete, cache_with_key +from zerver.lib.per_request_cache import ( + flush_per_request_cache, + return_same_value_during_entire_request, +) +from zerver.lib.types import LinkifierDict +from zerver.models.realms import Realm + + +def filter_pattern_validator(value: str) -> Pattern[str]: + try: + # Do not write errors to stderr (this still raises exceptions) + options = re2.Options() + options.log_errors = False + + regex = re2.compile(value, options=options) + except re2.error as e: + if len(e.args) >= 1: + if isinstance(e.args[0], str): # nocoverage + raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0])) + if isinstance(e.args[0], bytes): + raise ValidationError( + _("Bad regular expression: {regex}").format(regex=e.args[0].decode()) + ) + raise ValidationError(_("Unknown regular expression error")) # nocoverage + + return regex + + +def url_template_validator(value: str) -> None: + """Validate as a URL template""" + if not uri_template.validate(value): + raise ValidationError(_("Invalid URL template.")) + + +class RealmFilter(models.Model): + """Realm-specific regular expressions to automatically linkify certain + strings inside the Markdown processor. See "Custom filters" in the settings UI. + """ + + realm = models.ForeignKey(Realm, on_delete=CASCADE) + pattern = models.TextField() + url_template = models.TextField(validators=[url_template_validator]) + # Linkifiers are applied in a message/topic in order; the processing order + # is important when there are overlapping patterns. + order = models.IntegerField(default=0) + + class Meta: + unique_together = ("realm", "pattern") + + @override + def __str__(self) -> str: + return f"{self.realm.string_id}: {self.pattern} {self.url_template}" + + @override + def clean(self) -> None: + """Validate whether the set of parameters in the URL template + match the set of parameters in the regular expression. + + Django's `full_clean` calls `clean_fields` followed by `clean` method + and stores all ValidationErrors from all stages to return as JSON. + """ + + # Extract variables present in the pattern + pattern = filter_pattern_validator(self.pattern) + group_set = set(pattern.groupindex.keys()) + + # Do not continue the check if the url template is invalid to begin with. + # The ValidationError for invalid template will only be raised by the validator + # set on the url_template field instead of here to avoid duplicates. + if not uri_template.validate(self.url_template): + return + + # Extract variables used in the URL template. + template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names) + + # Report patterns missing in linkifier pattern. + missing_in_pattern_set = template_variables_set - group_set + if len(missing_in_pattern_set) > 0: + name = min(missing_in_pattern_set) + raise ValidationError( + _("Group %(name)r in URL template is not present in linkifier pattern."), + params={"name": name}, + ) + + missing_in_url_set = group_set - template_variables_set + # Report patterns missing in URL template. + if len(missing_in_url_set) > 0: + # We just report the first missing pattern here. Users can + # incrementally resolve errors if there are multiple + # missing patterns. + name = min(missing_in_url_set) + raise ValidationError( + _("Group %(name)r in linkifier pattern is not present in URL template."), + params={"name": name}, + ) + + +def get_linkifiers_cache_key(realm_id: int) -> str: + return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}" + + +@return_same_value_during_entire_request +@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7) +def linkifiers_for_realm(realm_id: int) -> List[LinkifierDict]: + return [ + LinkifierDict( + pattern=linkifier.pattern, + url_template=linkifier.url_template, + id=linkifier.id, + ) + for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order") + ] + + +def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None: + realm_id = instance.realm_id + cache_delete(get_linkifiers_cache_key(realm_id)) + flush_per_request_cache("linkifiers_for_realm") + + +post_save.connect(flush_linkifiers, sender=RealmFilter) +post_delete.connect(flush_linkifiers, sender=RealmFilter) diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 7042965c83..c0855687b0 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -83,9 +83,9 @@ from zerver.models import ( UserProfile, get_realm_playgrounds, get_stream, - linkifiers_for_realm, ) from zerver.models.groups import SystemGroups +from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realm_emoji import EmojiInfo, get_all_custom_emoji_for_realm from zerver.models.realms import RealmDomainDict, get_realm, get_realm_domains diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index d82d6ff535..dd030f8946 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -69,9 +69,9 @@ from zerver.models import ( UserProfile, get_client, get_stream, - linkifiers_for_realm, ) from zerver.models.groups import SystemGroups +from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realms import get_realm diff --git a/zerver/tests/test_realm_linkifiers.py b/zerver/tests/test_realm_linkifiers.py index c4a7d3f406..dd32df67ac 100644 --- a/zerver/tests/test_realm_linkifiers.py +++ b/zerver/tests/test_realm_linkifiers.py @@ -6,7 +6,8 @@ from django.core.exceptions import ValidationError from typing_extensions import override from zerver.lib.test_classes import ZulipTestCase -from zerver.models import RealmAuditLog, RealmFilter, url_template_validator +from zerver.models import RealmAuditLog, RealmFilter +from zerver.models.linkifiers import url_template_validator class RealmFilterTest(ZulipTestCase): diff --git a/zerver/views/realm_linkifiers.py b/zerver/views/realm_linkifiers.py index 300e1d6f8c..5fa3eb9813 100644 --- a/zerver/views/realm_linkifiers.py +++ b/zerver/views/realm_linkifiers.py @@ -15,7 +15,8 @@ from zerver.lib.exceptions import JsonableError, ValidationFailureError from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_int, check_list -from zerver.models import RealmFilter, UserProfile, linkifiers_for_realm +from zerver.models import RealmFilter, UserProfile +from zerver.models.linkifiers import linkifiers_for_realm # Custom realm linkifiers