From 67fb485797af47fb90e9dedde0623692ecdae845 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 14 Dec 2023 17:43:18 -0800 Subject: [PATCH] models: Extract zerver.models.realm_emoji. Signed-off-by: Anders Kaseorg --- zerver/actions/realm_emoji.py | 10 +- zerver/lib/emoji.py | 7 +- zerver/lib/events.py | 2 +- zerver/lib/markdown/__init__.py | 9 +- zerver/models/__init__.py | 135 +---------------- zerver/models/realm_emoji.py | 142 ++++++++++++++++++ zerver/tests/test_audit_log.py | 3 +- .../tests/test_message_notification_emails.py | 10 +- zerver/views/realm_emoji.py | 3 +- 9 files changed, 156 insertions(+), 165 deletions(-) create mode 100644 zerver/models/realm_emoji.py diff --git a/zerver/actions/realm_emoji.py b/zerver/actions/realm_emoji.py index 318a6ac45c..35010cd92d 100644 --- a/zerver/actions/realm_emoji.py +++ b/zerver/actions/realm_emoji.py @@ -10,14 +10,8 @@ from zerver.lib.emoji import get_emoji_file_name from zerver.lib.exceptions import JsonableError from zerver.lib.pysa import mark_sanitized from zerver.lib.upload import upload_emoji_image -from zerver.models import ( - EmojiInfo, - Realm, - RealmAuditLog, - RealmEmoji, - UserProfile, - get_all_custom_emoji_for_realm, -) +from zerver.models import Realm, RealmAuditLog, RealmEmoji, UserProfile +from zerver.models.realm_emoji import EmojiInfo, get_all_custom_emoji_for_realm from zerver.models.users import active_user_ids from zerver.tornado.django_api import send_event_on_commit diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index 818fd72e0c..b14c564b59 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -9,11 +9,8 @@ from django.utils.translation import gettext as _ from zerver.lib.exceptions import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend -from zerver.models import ( - Reaction, - Realm, - RealmEmoji, - UserProfile, +from zerver.models import Reaction, Realm, RealmEmoji, UserProfile +from zerver.models.realm_emoji import ( get_all_custom_emoji_for_realm, get_name_keyed_dict_for_active_realm_emoji, ) diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 9c60f83f88..a6924acec2 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -80,12 +80,12 @@ from zerver.models import ( UserStatus, UserTopic, custom_profile_fields_for_realm, - get_all_custom_emoji_for_realm, get_default_stream_groups, get_realm_playgrounds, linkifiers_for_realm, ) from zerver.models.constants import MAX_TOPIC_NAME_LENGTH +from zerver.models.realm_emoji import get_all_custom_emoji_for_realm from zerver.models.realms import get_realm_domains from zerver.tornado.django_api import get_user_events, request_event_queue from zproject.backends import email_auth_enabled, password_auth_enabled diff --git a/zerver/lib/markdown/__init__.py b/zerver/lib/markdown/__init__.py index ff15043ecd..2eb2d81891 100644 --- a/zerver/lib/markdown/__init__.py +++ b/zerver/lib/markdown/__init__.py @@ -74,13 +74,8 @@ from zerver.lib.timezone import common_timezones from zerver.lib.types import LinkifierDict from zerver.lib.url_encoding import encode_stream, hash_util_encode from zerver.lib.url_preview.types import UrlEmbedData, UrlOEmbedData -from zerver.models import ( - EmojiInfo, - Message, - Realm, - get_name_keyed_dict_for_active_realm_emoji, - linkifiers_for_realm, -) +from zerver.models import Message, Realm, linkifiers_for_realm +from zerver.models.realm_emoji import EmojiInfo, get_name_keyed_dict_for_active_realm_emoji ReturnT = TypeVar("ReturnT") diff --git a/zerver/models/__init__.py b/zerver/models/__init__.py index 69e2757bec..587e73bfd4 100644 --- a/zerver/models/__init__.py +++ b/zerver/models/__init__.py @@ -32,7 +32,7 @@ from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchVectorField from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder -from django.core.validators import MinLengthValidator, RegexValidator +from django.core.validators import RegexValidator from django.db import models, transaction from django.db.backends.base.base import BaseDatabaseWrapper from django.db.models import CASCADE, Exists, F, OuterRef, Q, QuerySet @@ -49,7 +49,6 @@ from confirmation import settings as confirmation_settings from zerver.lib import cache from zerver.lib.cache import ( cache_delete, - cache_set, cache_with_key, flush_message, flush_muting_users_cache, @@ -95,6 +94,7 @@ from zerver.models.groups import GroupGroupMembership as GroupGroupMembership from zerver.models.groups import SystemGroups from zerver.models.groups import UserGroup as UserGroup from zerver.models.groups import UserGroupMembership as UserGroupMembership +from zerver.models.realm_emoji import RealmEmoji as RealmEmoji from zerver.models.realms import Realm as Realm from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticationMethod from zerver.models.realms import RealmDomain as RealmDomain @@ -104,15 +104,6 @@ from zerver.models.users import UserProfile as UserProfile from zerver.models.users import get_user_profile_by_id_in_realm -class EmojiInfo(TypedDict): - id: str - name: str - source_url: str - deactivated: bool - author_id: Optional[int] - still_url: Optional[str] - - @models.Field.register_lookup class AndZero(models.Lookup[int]): lookup_name = "andz" @@ -211,128 +202,6 @@ def get_recipient_ids( return to, recipient_type_str -def get_all_custom_emoji_for_realm_cache_key(realm_id: int) -> str: - return f"realm_emoji:{realm_id}" - - -class RealmEmoji(models.Model): - author = models.ForeignKey( - "UserProfile", - blank=True, - null=True, - on_delete=CASCADE, - ) - realm = models.ForeignKey(Realm, on_delete=CASCADE) - name = models.TextField( - validators=[ - MinLengthValidator(1), - # The second part of the regex (negative lookbehind) disallows names - # ending with one of the punctuation characters. - RegexValidator( - regex=r"^[0-9a-z.\-_]+(? str: - return f"{self.realm.string_id}: {self.id} {self.name} {self.deactivated} {self.file_name}" - - -def get_all_custom_emoji_for_realm_uncached(realm_id: int) -> Dict[str, EmojiInfo]: - # RealmEmoji objects with file_name=None are still in the process - # of being uploaded, and we expect to be cleaned up by a - # try/finally block if the upload fails, so it's correct to - # exclude them. - query = RealmEmoji.objects.filter(realm_id=realm_id).exclude( - file_name=None, - ) - d = {} - from zerver.lib.emoji import get_emoji_url - - for realm_emoji in query.all(): - author_id = realm_emoji.author_id - assert realm_emoji.file_name is not None - emoji_url = get_emoji_url(realm_emoji.file_name, realm_emoji.realm_id) - - emoji_dict: EmojiInfo = dict( - id=str(realm_emoji.id), - name=realm_emoji.name, - source_url=emoji_url, - deactivated=realm_emoji.deactivated, - author_id=author_id, - still_url=None, - ) - - if realm_emoji.is_animated: - # For animated emoji, we include still_url with a static - # version of the image, so that clients can display the - # emoji in a less distracting (not animated) fashion when - # desired. - emoji_dict["still_url"] = get_emoji_url( - realm_emoji.file_name, realm_emoji.realm_id, still=True - ) - - d[str(realm_emoji.id)] = emoji_dict - - return d - - -@cache_with_key(get_all_custom_emoji_for_realm_cache_key, timeout=3600 * 24 * 7) -def get_all_custom_emoji_for_realm(realm_id: int) -> Dict[str, EmojiInfo]: - return get_all_custom_emoji_for_realm_uncached(realm_id) - - -def get_name_keyed_dict_for_active_realm_emoji(realm_id: int) -> Dict[str, EmojiInfo]: - # It's important to use the cached version here. - realm_emojis = get_all_custom_emoji_for_realm(realm_id) - return {row["name"]: row for row in realm_emojis.values() if not row["deactivated"]} - - -def flush_realm_emoji(*, instance: RealmEmoji, **kwargs: object) -> None: - if instance.file_name is None: - # Because we construct RealmEmoji.file_name using the ID for - # the RealmEmoji object, it will always have file_name=None, - # and then it'll be updated with the actual filename as soon - # as the upload completes successfully. - # - # Doing nothing when file_name=None is the best option, since - # such an object shouldn't have been cached yet, and this - # function will be called again when file_name is set. - return - realm_id = instance.realm_id - cache_set( - get_all_custom_emoji_for_realm_cache_key(realm_id), - get_all_custom_emoji_for_realm_uncached(realm_id), - timeout=3600 * 24 * 7, - ) - - -post_save.connect(flush_realm_emoji, sender=RealmEmoji) -post_delete.connect(flush_realm_emoji, sender=RealmEmoji) - - def filter_pattern_validator(value: str) -> Pattern[str]: try: # Do not write errors to stderr (this still raises exceptions) diff --git a/zerver/models/realm_emoji.py b/zerver/models/realm_emoji.py new file mode 100644 index 0000000000..02766beeab --- /dev/null +++ b/zerver/models/realm_emoji.py @@ -0,0 +1,142 @@ +from typing import Dict, Optional, TypedDict + +from django.core.validators import MinLengthValidator, RegexValidator +from django.db import models +from django.db.models import CASCADE, Q +from django.db.models.signals import post_delete, post_save +from django.utils.translation import gettext_lazy +from typing_extensions import override + +from zerver.lib.cache import cache_set, cache_with_key +from zerver.models.realms import Realm + + +class EmojiInfo(TypedDict): + id: str + name: str + source_url: str + deactivated: bool + author_id: Optional[int] + still_url: Optional[str] + + +def get_all_custom_emoji_for_realm_cache_key(realm_id: int) -> str: + return f"realm_emoji:{realm_id}" + + +class RealmEmoji(models.Model): + author = models.ForeignKey( + "UserProfile", + blank=True, + null=True, + on_delete=CASCADE, + ) + realm = models.ForeignKey(Realm, on_delete=CASCADE) + name = models.TextField( + validators=[ + MinLengthValidator(1), + # The second part of the regex (negative lookbehind) disallows names + # ending with one of the punctuation characters. + RegexValidator( + regex=r"^[0-9a-z.\-_]+(? str: + return f"{self.realm.string_id}: {self.id} {self.name} {self.deactivated} {self.file_name}" + + +def get_all_custom_emoji_for_realm_uncached(realm_id: int) -> Dict[str, EmojiInfo]: + # RealmEmoji objects with file_name=None are still in the process + # of being uploaded, and we expect to be cleaned up by a + # try/finally block if the upload fails, so it's correct to + # exclude them. + query = RealmEmoji.objects.filter(realm_id=realm_id).exclude( + file_name=None, + ) + d = {} + from zerver.lib.emoji import get_emoji_url + + for realm_emoji in query.all(): + author_id = realm_emoji.author_id + assert realm_emoji.file_name is not None + emoji_url = get_emoji_url(realm_emoji.file_name, realm_emoji.realm_id) + + emoji_dict: EmojiInfo = dict( + id=str(realm_emoji.id), + name=realm_emoji.name, + source_url=emoji_url, + deactivated=realm_emoji.deactivated, + author_id=author_id, + still_url=None, + ) + + if realm_emoji.is_animated: + # For animated emoji, we include still_url with a static + # version of the image, so that clients can display the + # emoji in a less distracting (not animated) fashion when + # desired. + emoji_dict["still_url"] = get_emoji_url( + realm_emoji.file_name, realm_emoji.realm_id, still=True + ) + + d[str(realm_emoji.id)] = emoji_dict + + return d + + +@cache_with_key(get_all_custom_emoji_for_realm_cache_key, timeout=3600 * 24 * 7) +def get_all_custom_emoji_for_realm(realm_id: int) -> Dict[str, EmojiInfo]: + return get_all_custom_emoji_for_realm_uncached(realm_id) + + +def get_name_keyed_dict_for_active_realm_emoji(realm_id: int) -> Dict[str, EmojiInfo]: + # It's important to use the cached version here. + realm_emojis = get_all_custom_emoji_for_realm(realm_id) + return {row["name"]: row for row in realm_emojis.values() if not row["deactivated"]} + + +def flush_realm_emoji(*, instance: RealmEmoji, **kwargs: object) -> None: + if instance.file_name is None: + # Because we construct RealmEmoji.file_name using the ID for + # the RealmEmoji object, it will always have file_name=None, + # and then it'll be updated with the actual filename as soon + # as the upload completes successfully. + # + # Doing nothing when file_name=None is the best option, since + # such an object shouldn't have been cached yet, and this + # function will be called again when file_name is set. + return + realm_id = instance.realm_id + cache_set( + get_all_custom_emoji_for_realm_cache_key(realm_id), + get_all_custom_emoji_for_realm_uncached(realm_id), + timeout=3600 * 24 * 7, + ) + + +post_save.connect(flush_realm_emoji, sender=RealmEmoji) +post_delete.connect(flush_realm_emoji, sender=RealmEmoji) diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 6e601775e7..7042965c83 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -73,7 +73,6 @@ from zerver.lib.test_helpers import get_test_image_file from zerver.lib.types import LinkifierDict, RealmPlaygroundDict from zerver.lib.utils import assert_is_not_none from zerver.models import ( - EmojiInfo, Message, Realm, RealmAuditLog, @@ -82,12 +81,12 @@ from zerver.models import ( Subscription, UserGroup, UserProfile, - get_all_custom_emoji_for_realm, get_realm_playgrounds, get_stream, linkifiers_for_realm, ) from zerver.models.groups import SystemGroups +from zerver.models.realm_emoji import EmojiInfo, get_all_custom_emoji_for_realm from zerver.models.realms import RealmDomainDict, get_realm, get_realm_domains diff --git a/zerver/tests/test_message_notification_emails.py b/zerver/tests/test_message_notification_emails.py index c734219bbf..430c5d819d 100644 --- a/zerver/tests/test_message_notification_emails.py +++ b/zerver/tests/test_message_notification_emails.py @@ -27,14 +27,8 @@ from zerver.lib.email_notifications import ( ) from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - NotificationTriggers, - UserMessage, - UserProfile, - UserTopic, - get_name_keyed_dict_for_active_realm_emoji, - get_stream, -) +from zerver.models import NotificationTriggers, UserMessage, UserProfile, UserTopic, get_stream +from zerver.models.realm_emoji import get_name_keyed_dict_for_active_realm_emoji from zerver.models.realms import get_realm diff --git a/zerver/views/realm_emoji.py b/zerver/views/realm_emoji.py index 35bf25932c..6bd3d00a45 100644 --- a/zerver/views/realm_emoji.py +++ b/zerver/views/realm_emoji.py @@ -9,7 +9,8 @@ from zerver.lib.emoji import check_remove_custom_emoji, check_valid_emoji_name, from zerver.lib.exceptions import JsonableError, ResourceNotFoundError from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.models import RealmEmoji, UserProfile, get_all_custom_emoji_for_realm +from zerver.models import RealmEmoji, UserProfile +from zerver.models.realm_emoji import get_all_custom_emoji_for_realm def list_emoji(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: