From c9c819e1d7f1d44d4898b0e2faa30e542ab49889 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 15 Dec 2023 11:21:59 -0800 Subject: [PATCH] models: Extract zerver.models.scheduled_jobs. Signed-off-by: Anders Kaseorg --- analytics/tests/test_counts.py | 2 +- zerver/actions/message_send.py | 2 +- zerver/lib/email_notifications.py | 11 +- zerver/lib/notification_data.py | 3 +- zerver/lib/push_notifications.py | 2 +- zerver/lib/scheduled_messages.py | 5 +- zerver/lib/send_email.py | 3 +- zerver/lib/soft_deactivation.py | 2 +- ...468_rename_followup_day_email_templates.py | 2 +- zerver/models/__init__.py | 247 +---------------- zerver/models/scheduled_jobs.py | 255 ++++++++++++++++++ .../tests/test_message_edit_notifications.py | 3 +- .../tests/test_message_notification_emails.py | 3 +- zerver/tests/test_notification_data.py | 2 +- .../tests/test_outgoing_webhook_interfaces.py | 3 +- zerver/tests/test_push_notifications.py | 2 +- zerver/tests/test_queue_worker.py | 2 +- zerver/tests/test_service_bot_system.py | 3 +- zerver/tests/test_settings.py | 8 +- 19 files changed, 289 insertions(+), 271 deletions(-) create mode 100644 zerver/models/scheduled_jobs.py diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index c8ffd2b291..b30e9df0b1 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -68,7 +68,6 @@ from zerver.models import ( Client, Huddle, Message, - NotificationTriggers, PreregistrationUser, Realm, RealmAuditLog, @@ -80,6 +79,7 @@ from zerver.models import ( ) from zerver.models.clients import get_client from zerver.models.groups import SystemGroups +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.users import get_user, is_cross_realm_bot_email from zilencer.models import ( RemoteInstallationCount, diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.py index 01c7289e0a..b32ed2f996 100644 --- a/zerver/actions/message_send.py +++ b/zerver/actions/message_send.py @@ -94,7 +94,6 @@ from zerver.lib.widget import do_widget_post_save_actions from zerver.models import ( Client, Message, - NotificationTriggers, Realm, Recipient, Stream, @@ -107,6 +106,7 @@ from zerver.models import ( from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.recipients import get_huddle_user_ids +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream, get_stream_by_id_in_realm from zerver.models.users import get_system_bot, get_user_by_delivery_email, is_cross_realm_bot_email from zerver.tornado.django_api import send_event diff --git a/zerver/lib/email_notifications.py b/zerver/lib/email_notifications.py index dcaa74aa0e..56c4bb85b9 100644 --- a/zerver/lib/email_notifications.py +++ b/zerver/lib/email_notifications.py @@ -36,16 +36,9 @@ from zerver.lib.url_encoding import ( stream_narrow_url, topic_narrow_url, ) -from zerver.models import ( - Message, - NotificationTriggers, - Realm, - Recipient, - Stream, - UserMessage, - UserProfile, -) +from zerver.models import Message, Realm, Recipient, Stream, UserMessage, UserProfile from zerver.models.messages import get_context_for_message +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.users import get_user_profile_by_id if sys.version_info < (3, 9): # nocoverage diff --git a/zerver/lib/notification_data.py b/zerver/lib/notification_data.py index 29abe63dc0..86763f9eae 100644 --- a/zerver/lib/notification_data.py +++ b/zerver/lib/notification_data.py @@ -4,7 +4,8 @@ from typing import Any, Collection, Dict, List, Optional, Set from zerver.lib.mention import MentionData from zerver.lib.user_groups import get_user_group_direct_member_ids -from zerver.models import NotificationTriggers, UserGroup, UserProfile, UserTopic +from zerver.models import UserGroup, UserProfile, UserTopic +from zerver.models.scheduled_jobs import NotificationTriggers @dataclass diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index c3cc472f9b..7ccf41f232 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -57,7 +57,6 @@ from zerver.models import ( AbstractPushDeviceToken, ArchivedMessage, Message, - NotificationTriggers, PushDeviceToken, Realm, Recipient, @@ -67,6 +66,7 @@ from zerver.models import ( UserProfile, ) from zerver.models.realms import get_fake_email_domain +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.users import get_user_profile_by_id if TYPE_CHECKING: diff --git a/zerver/lib/scheduled_messages.py b/zerver/lib/scheduled_messages.py index 39fbc625c8..4f6b4219bf 100644 --- a/zerver/lib/scheduled_messages.py +++ b/zerver/lib/scheduled_messages.py @@ -3,11 +3,10 @@ from typing import List, Union from django.utils.translation import gettext as _ from zerver.lib.exceptions import ResourceNotFoundError -from zerver.models import ( +from zerver.models import ScheduledMessage, UserProfile +from zerver.models.scheduled_jobs import ( APIScheduledDirectMessageDict, APIScheduledStreamMessageDict, - ScheduledMessage, - UserProfile, ) diff --git a/zerver/lib/send_email.py b/zerver/lib/send_email.py index 8880bb7f8f..35b5c3ed28 100644 --- a/zerver/lib/send_email.py +++ b/zerver/lib/send_email.py @@ -29,7 +29,8 @@ from django.utils.translation import override as override_language from confirmation.models import generate_key from zerver.lib.logging_util import log_to_file -from zerver.models import EMAIL_TYPES, Realm, ScheduledEmail, UserProfile +from zerver.models import Realm, ScheduledEmail, UserProfile +from zerver.models.scheduled_jobs import EMAIL_TYPES from zerver.models.users import get_user_profile_by_id from zproject.email_backends import EmailLogBackEnd, get_forward_address diff --git a/zerver/lib/soft_deactivation.py b/zerver/lib/soft_deactivation.py index 07d13de98c..73da684937 100644 --- a/zerver/lib/soft_deactivation.py +++ b/zerver/lib/soft_deactivation.py @@ -14,7 +14,6 @@ from zerver.lib.queue import queue_json_publish from zerver.lib.utils import assert_is_not_none from zerver.models import ( Message, - NotificationTriggers, Realm, RealmAuditLog, Recipient, @@ -23,6 +22,7 @@ from zerver.models import ( UserMessage, UserProfile, ) +from zerver.models.scheduled_jobs import NotificationTriggers logger = logging.getLogger("zulip.soft_deactivation") log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH) diff --git a/zerver/migrations/0468_rename_followup_day_email_templates.py b/zerver/migrations/0468_rename_followup_day_email_templates.py index 6f77b0e9df..1d393adc76 100644 --- a/zerver/migrations/0468_rename_followup_day_email_templates.py +++ b/zerver/migrations/0468_rename_followup_day_email_templates.py @@ -6,7 +6,7 @@ from django.db.migrations.state import StateApps from django.db.models import F, Func, JSONField, TextField, Value from django.db.models.functions import Cast -# ScheduledMessage.type for onboarding emails from zerver/models/__init__.py +# ScheduledMessage.type for onboarding emails from zerver/models/scheduled_jobs.py WELCOME = 1 diff --git a/zerver/models/__init__.py b/zerver/models/__init__.py index 6d682987ae..d9f2df2710 100644 --- a/zerver/models/__init__.py +++ b/zerver/models/__init__.py @@ -1,10 +1,9 @@ # https://github.com/typeddjango/django-stubs/issues/1698 # mypy: disable-error-code="explicit-override" -from typing import Any, Callable, Dict, List, Tuple, TypedDict, TypeVar, Union +from typing import Any, Callable, Dict, List, Tuple, TypeVar, Union import orjson -from django.conf import settings from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder from django.db import models @@ -23,8 +22,6 @@ from zerver.lib.cache import ( realm_alert_words_automaton_cache_key, realm_alert_words_cache_key, ) -from zerver.lib.display_recipient import get_recipient_ids -from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.types import ( ExtendedFieldElement, ExtendedValidator, @@ -45,7 +42,6 @@ from zerver.lib.validator import ( validate_select_field, ) from zerver.models.clients import Client as Client -from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.drafts import Draft as Draft from zerver.models.groups import GroupGroupMembership as GroupGroupMembership from zerver.models.groups import UserGroup as UserGroup @@ -85,6 +81,13 @@ from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticatio from zerver.models.realms import RealmDomain as RealmDomain from zerver.models.recipients import Huddle as Huddle from zerver.models.recipients import Recipient as Recipient +from zerver.models.scheduled_jobs import AbstractScheduledJob as AbstractScheduledJob +from zerver.models.scheduled_jobs import MissedMessageEmailAddress as MissedMessageEmailAddress +from zerver.models.scheduled_jobs import ScheduledEmail as ScheduledEmail +from zerver.models.scheduled_jobs import ScheduledMessage as ScheduledMessage +from zerver.models.scheduled_jobs import ( + ScheduledMessageNotificationEmail as ScheduledMessageNotificationEmail, +) from zerver.models.streams import DefaultStream as DefaultStream from zerver.models.streams import DefaultStreamGroup as DefaultStreamGroup from zerver.models.streams import Stream as Stream @@ -151,240 +154,6 @@ def query_for_ids( return query -class AbstractScheduledJob(models.Model): - scheduled_timestamp = models.DateTimeField(db_index=True) - # JSON representation of arguments to consumer - data = models.TextField() - realm = models.ForeignKey(Realm, on_delete=CASCADE) - - class Meta: - abstract = True - - -class ScheduledEmail(AbstractScheduledJob): - # Exactly one of users or address should be set. These are - # duplicate values, used to efficiently filter the set of - # ScheduledEmails for use in clear_scheduled_emails; the - # recipients used for actually sending messages are stored in the - # data field of AbstractScheduledJob. - users = models.ManyToManyField(UserProfile) - # Just the address part of a full "name
" email address - address = models.EmailField(null=True, db_index=True) - - # Valid types are below - WELCOME = 1 - DIGEST = 2 - INVITATION_REMINDER = 3 - type = models.PositiveSmallIntegerField() - - @override - def __str__(self) -> str: - return f"{self.type} {self.address or list(self.users.all())} {self.scheduled_timestamp}" - - -class MissedMessageEmailAddress(models.Model): - message = models.ForeignKey(Message, on_delete=CASCADE) - user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) - email_token = models.CharField(max_length=34, unique=True, db_index=True) - - # Timestamp of when the missed message address generated. - timestamp = models.DateTimeField(db_index=True, default=timezone_now) - # Number of times the missed message address has been used. - times_used = models.PositiveIntegerField(default=0, db_index=True) - - @override - def __str__(self) -> str: - return settings.EMAIL_GATEWAY_PATTERN % (self.email_token,) - - def increment_times_used(self) -> None: - self.times_used += 1 - self.save(update_fields=["times_used"]) - - -class NotificationTriggers: - # "direct_message" is for 1:1 direct messages as well as huddles - DIRECT_MESSAGE = "direct_message" - MENTION = "mentioned" - TOPIC_WILDCARD_MENTION = "topic_wildcard_mentioned" - STREAM_WILDCARD_MENTION = "stream_wildcard_mentioned" - STREAM_PUSH = "stream_push_notify" - STREAM_EMAIL = "stream_email_notify" - FOLLOWED_TOPIC_PUSH = "followed_topic_push_notify" - FOLLOWED_TOPIC_EMAIL = "followed_topic_email_notify" - TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "topic_wildcard_mentioned_in_followed_topic" - STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "stream_wildcard_mentioned_in_followed_topic" - - -class ScheduledMessageNotificationEmail(models.Model): - """Stores planned outgoing message notification emails. They may be - processed earlier should Zulip choose to batch multiple messages - in a single email, but typically will be processed just after - scheduled_timestamp. - """ - - user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) - message = models.ForeignKey(Message, on_delete=CASCADE) - - EMAIL_NOTIFICATION_TRIGGER_CHOICES = [ - (NotificationTriggers.DIRECT_MESSAGE, "Direct message"), - (NotificationTriggers.MENTION, "Mention"), - (NotificationTriggers.TOPIC_WILDCARD_MENTION, "Topic wildcard mention"), - (NotificationTriggers.STREAM_WILDCARD_MENTION, "Stream wildcard mention"), - (NotificationTriggers.STREAM_EMAIL, "Stream notifications enabled"), - (NotificationTriggers.FOLLOWED_TOPIC_EMAIL, "Followed topic notifications enabled"), - ( - NotificationTriggers.TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC, - "Topic wildcard mention in followed topic", - ), - ( - NotificationTriggers.STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC, - "Stream wildcard mention in followed topic", - ), - ] - - trigger = models.TextField(choices=EMAIL_NOTIFICATION_TRIGGER_CHOICES) - mentioned_user_group = models.ForeignKey(UserGroup, null=True, on_delete=CASCADE) - - # Timestamp for when the notification should be processed and sent. - # Calculated from the time the event was received and the batching period. - scheduled_timestamp = models.DateTimeField(db_index=True) - - -class APIScheduledStreamMessageDict(TypedDict): - scheduled_message_id: int - to: int - type: str - content: str - rendered_content: str - topic: str - scheduled_delivery_timestamp: int - failed: bool - - -class APIScheduledDirectMessageDict(TypedDict): - scheduled_message_id: int - to: List[int] - type: str - content: str - rendered_content: str - scheduled_delivery_timestamp: int - failed: bool - - -class ScheduledMessage(models.Model): - sender = models.ForeignKey(UserProfile, on_delete=CASCADE) - recipient = models.ForeignKey(Recipient, on_delete=CASCADE) - subject = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH) - content = models.TextField() - rendered_content = models.TextField() - sending_client = models.ForeignKey(Client, on_delete=CASCADE) - stream = models.ForeignKey(Stream, null=True, on_delete=CASCADE) - realm = models.ForeignKey(Realm, on_delete=CASCADE) - scheduled_timestamp = models.DateTimeField(db_index=True) - read_by_sender = models.BooleanField() - delivered = models.BooleanField(default=False) - delivered_message = models.ForeignKey(Message, null=True, on_delete=CASCADE) - has_attachment = models.BooleanField(default=False, db_index=True) - - # Metadata for messages that failed to send when their scheduled - # moment arrived. - failed = models.BooleanField(default=False) - failure_message = models.TextField(null=True) - - SEND_LATER = 1 - REMIND = 2 - - DELIVERY_TYPES = ( - (SEND_LATER, "send_later"), - (REMIND, "remind"), - ) - - delivery_type = models.PositiveSmallIntegerField( - choices=DELIVERY_TYPES, - default=SEND_LATER, - ) - - class Meta: - indexes = [ - # We expect a large number of delivered scheduled messages - # to accumulate over time. This first index is for the - # deliver_scheduled_messages worker. - models.Index( - name="zerver_unsent_scheduled_messages_by_time", - fields=["scheduled_timestamp"], - condition=Q( - delivered=False, - failed=False, - ), - ), - # This index is for displaying scheduled messages to the - # user themself via the API; we don't filter failed - # messages since we will want to display those so that - # failures don't just disappear into a black hole. - models.Index( - name="zerver_realm_unsent_scheduled_messages_by_user", - fields=["realm_id", "sender", "delivery_type", "scheduled_timestamp"], - condition=Q( - delivered=False, - ), - ), - ] - - @override - def __str__(self) -> str: - return f"{self.recipient.label()} {self.subject} {self.sender!r} {self.scheduled_timestamp}" - - def topic_name(self) -> str: - return self.subject - - def set_topic_name(self, topic_name: str) -> None: - self.subject = topic_name - - def is_stream_message(self) -> bool: - return self.recipient.type == Recipient.STREAM - - def to_dict(self) -> Union[APIScheduledStreamMessageDict, APIScheduledDirectMessageDict]: - recipient, recipient_type_str = get_recipient_ids(self.recipient, self.sender.id) - - if recipient_type_str == "private": - # The topic for direct messages should always be an empty string. - assert self.topic_name() == "" - - return APIScheduledDirectMessageDict( - scheduled_message_id=self.id, - to=recipient, - type=recipient_type_str, - content=self.content, - rendered_content=self.rendered_content, - scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp), - failed=self.failed, - ) - - # The recipient for stream messages should always just be the unique stream ID. - assert len(recipient) == 1 - - return APIScheduledStreamMessageDict( - scheduled_message_id=self.id, - to=recipient[0], - type=recipient_type_str, - content=self.content, - rendered_content=self.rendered_content, - topic=self.topic_name(), - scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp), - failed=self.failed, - ) - - -EMAIL_TYPES = { - "account_registered": ScheduledEmail.WELCOME, - "onboarding_zulip_topics": ScheduledEmail.WELCOME, - "onboarding_zulip_guide": ScheduledEmail.WELCOME, - "onboarding_team_to_zulip": ScheduledEmail.WELCOME, - "digest": ScheduledEmail.DIGEST, - "invitation_reminder": ScheduledEmail.INVITATION_REMINDER, -} - - class AbstractRealmAuditLog(models.Model): """Defines fields common to RealmAuditLog and RemoteRealmAuditLog.""" diff --git a/zerver/models/scheduled_jobs.py b/zerver/models/scheduled_jobs.py new file mode 100644 index 0000000000..18d84a4668 --- /dev/null +++ b/zerver/models/scheduled_jobs.py @@ -0,0 +1,255 @@ +# https://github.com/typeddjango/django-stubs/issues/1698 +# mypy: disable-error-code="explicit-override" + +from typing import List, TypedDict, Union + +from django.conf import settings +from django.db import models +from django.db.models import CASCADE, Q +from django.utils.timezone import now as timezone_now +from typing_extensions import override + +from zerver.lib.display_recipient import get_recipient_ids +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.models.clients import Client +from zerver.models.constants import MAX_TOPIC_NAME_LENGTH +from zerver.models.groups import UserGroup +from zerver.models.messages import Message +from zerver.models.realms import Realm +from zerver.models.recipients import Recipient +from zerver.models.streams import Stream +from zerver.models.users import UserProfile + + +class AbstractScheduledJob(models.Model): + scheduled_timestamp = models.DateTimeField(db_index=True) + # JSON representation of arguments to consumer + data = models.TextField() + realm = models.ForeignKey(Realm, on_delete=CASCADE) + + class Meta: + abstract = True + + +class ScheduledEmail(AbstractScheduledJob): + # Exactly one of users or address should be set. These are + # duplicate values, used to efficiently filter the set of + # ScheduledEmails for use in clear_scheduled_emails; the + # recipients used for actually sending messages are stored in the + # data field of AbstractScheduledJob. + users = models.ManyToManyField(UserProfile) + # Just the address part of a full "name
" email address + address = models.EmailField(null=True, db_index=True) + + # Valid types are below + WELCOME = 1 + DIGEST = 2 + INVITATION_REMINDER = 3 + type = models.PositiveSmallIntegerField() + + @override + def __str__(self) -> str: + return f"{self.type} {self.address or list(self.users.all())} {self.scheduled_timestamp}" + + +class MissedMessageEmailAddress(models.Model): + message = models.ForeignKey(Message, on_delete=CASCADE) + user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) + email_token = models.CharField(max_length=34, unique=True, db_index=True) + + # Timestamp of when the missed message address generated. + timestamp = models.DateTimeField(db_index=True, default=timezone_now) + # Number of times the missed message address has been used. + times_used = models.PositiveIntegerField(default=0, db_index=True) + + @override + def __str__(self) -> str: + return settings.EMAIL_GATEWAY_PATTERN % (self.email_token,) + + def increment_times_used(self) -> None: + self.times_used += 1 + self.save(update_fields=["times_used"]) + + +class NotificationTriggers: + # "direct_message" is for 1:1 direct messages as well as huddles + DIRECT_MESSAGE = "direct_message" + MENTION = "mentioned" + TOPIC_WILDCARD_MENTION = "topic_wildcard_mentioned" + STREAM_WILDCARD_MENTION = "stream_wildcard_mentioned" + STREAM_PUSH = "stream_push_notify" + STREAM_EMAIL = "stream_email_notify" + FOLLOWED_TOPIC_PUSH = "followed_topic_push_notify" + FOLLOWED_TOPIC_EMAIL = "followed_topic_email_notify" + TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "topic_wildcard_mentioned_in_followed_topic" + STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "stream_wildcard_mentioned_in_followed_topic" + + +class ScheduledMessageNotificationEmail(models.Model): + """Stores planned outgoing message notification emails. They may be + processed earlier should Zulip choose to batch multiple messages + in a single email, but typically will be processed just after + scheduled_timestamp. + """ + + user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) + message = models.ForeignKey(Message, on_delete=CASCADE) + + EMAIL_NOTIFICATION_TRIGGER_CHOICES = [ + (NotificationTriggers.DIRECT_MESSAGE, "Direct message"), + (NotificationTriggers.MENTION, "Mention"), + (NotificationTriggers.TOPIC_WILDCARD_MENTION, "Topic wildcard mention"), + (NotificationTriggers.STREAM_WILDCARD_MENTION, "Stream wildcard mention"), + (NotificationTriggers.STREAM_EMAIL, "Stream notifications enabled"), + (NotificationTriggers.FOLLOWED_TOPIC_EMAIL, "Followed topic notifications enabled"), + ( + NotificationTriggers.TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC, + "Topic wildcard mention in followed topic", + ), + ( + NotificationTriggers.STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC, + "Stream wildcard mention in followed topic", + ), + ] + + trigger = models.TextField(choices=EMAIL_NOTIFICATION_TRIGGER_CHOICES) + mentioned_user_group = models.ForeignKey(UserGroup, null=True, on_delete=CASCADE) + + # Timestamp for when the notification should be processed and sent. + # Calculated from the time the event was received and the batching period. + scheduled_timestamp = models.DateTimeField(db_index=True) + + +class APIScheduledStreamMessageDict(TypedDict): + scheduled_message_id: int + to: int + type: str + content: str + rendered_content: str + topic: str + scheduled_delivery_timestamp: int + failed: bool + + +class APIScheduledDirectMessageDict(TypedDict): + scheduled_message_id: int + to: List[int] + type: str + content: str + rendered_content: str + scheduled_delivery_timestamp: int + failed: bool + + +class ScheduledMessage(models.Model): + sender = models.ForeignKey(UserProfile, on_delete=CASCADE) + recipient = models.ForeignKey(Recipient, on_delete=CASCADE) + subject = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH) + content = models.TextField() + rendered_content = models.TextField() + sending_client = models.ForeignKey(Client, on_delete=CASCADE) + stream = models.ForeignKey(Stream, null=True, on_delete=CASCADE) + realm = models.ForeignKey(Realm, on_delete=CASCADE) + scheduled_timestamp = models.DateTimeField(db_index=True) + read_by_sender = models.BooleanField() + delivered = models.BooleanField(default=False) + delivered_message = models.ForeignKey(Message, null=True, on_delete=CASCADE) + has_attachment = models.BooleanField(default=False, db_index=True) + + # Metadata for messages that failed to send when their scheduled + # moment arrived. + failed = models.BooleanField(default=False) + failure_message = models.TextField(null=True) + + SEND_LATER = 1 + REMIND = 2 + + DELIVERY_TYPES = ( + (SEND_LATER, "send_later"), + (REMIND, "remind"), + ) + + delivery_type = models.PositiveSmallIntegerField( + choices=DELIVERY_TYPES, + default=SEND_LATER, + ) + + class Meta: + indexes = [ + # We expect a large number of delivered scheduled messages + # to accumulate over time. This first index is for the + # deliver_scheduled_messages worker. + models.Index( + name="zerver_unsent_scheduled_messages_by_time", + fields=["scheduled_timestamp"], + condition=Q( + delivered=False, + failed=False, + ), + ), + # This index is for displaying scheduled messages to the + # user themself via the API; we don't filter failed + # messages since we will want to display those so that + # failures don't just disappear into a black hole. + models.Index( + name="zerver_realm_unsent_scheduled_messages_by_user", + fields=["realm_id", "sender", "delivery_type", "scheduled_timestamp"], + condition=Q( + delivered=False, + ), + ), + ] + + @override + def __str__(self) -> str: + return f"{self.recipient.label()} {self.subject} {self.sender!r} {self.scheduled_timestamp}" + + def topic_name(self) -> str: + return self.subject + + def set_topic_name(self, topic_name: str) -> None: + self.subject = topic_name + + def is_stream_message(self) -> bool: + return self.recipient.type == Recipient.STREAM + + def to_dict(self) -> Union[APIScheduledStreamMessageDict, APIScheduledDirectMessageDict]: + recipient, recipient_type_str = get_recipient_ids(self.recipient, self.sender.id) + + if recipient_type_str == "private": + # The topic for direct messages should always be an empty string. + assert self.topic_name() == "" + + return APIScheduledDirectMessageDict( + scheduled_message_id=self.id, + to=recipient, + type=recipient_type_str, + content=self.content, + rendered_content=self.rendered_content, + scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp), + failed=self.failed, + ) + + # The recipient for stream messages should always just be the unique stream ID. + assert len(recipient) == 1 + + return APIScheduledStreamMessageDict( + scheduled_message_id=self.id, + to=recipient[0], + type=recipient_type_str, + content=self.content, + rendered_content=self.rendered_content, + topic=self.topic_name(), + scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp), + failed=self.failed, + ) + + +EMAIL_TYPES = { + "account_registered": ScheduledEmail.WELCOME, + "onboarding_zulip_topics": ScheduledEmail.WELCOME, + "onboarding_zulip_guide": ScheduledEmail.WELCOME, + "onboarding_team_to_zulip": ScheduledEmail.WELCOME, + "digest": ScheduledEmail.DIGEST, + "invitation_reminder": ScheduledEmail.INVITATION_REMINDER, +} diff --git a/zerver/tests/test_message_edit_notifications.py b/zerver/tests/test_message_edit_notifications.py index e857681e01..69daf9dc5d 100644 --- a/zerver/tests/test_message_edit_notifications.py +++ b/zerver/tests/test_message_edit_notifications.py @@ -8,7 +8,8 @@ from zerver.actions.user_topics import do_set_user_topic_visibility_policy from zerver.lib.push_notifications import get_apns_badge_count, get_apns_badge_count_future from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import mock_queue_publish -from zerver.models import NotificationTriggers, Subscription, UserPresence, UserTopic +from zerver.models import Subscription, UserPresence, UserTopic +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream from zerver.tornado.event_queue import maybe_enqueue_notifications diff --git a/zerver/tests/test_message_notification_emails.py b/zerver/tests/test_message_notification_emails.py index 825334455b..35557c1c6f 100644 --- a/zerver/tests/test_message_notification_emails.py +++ b/zerver/tests/test_message_notification_emails.py @@ -27,9 +27,10 @@ from zerver.lib.email_notifications import ( ) from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import ZulipTestCase -from zerver.models import NotificationTriggers, UserMessage, UserProfile, UserTopic +from zerver.models import UserMessage, UserProfile, UserTopic from zerver.models.realm_emoji import get_name_keyed_dict_for_active_realm_emoji from zerver.models.realms import get_realm +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream diff --git a/zerver/tests/test_notification_data.py b/zerver/tests/test_notification_data.py index c6ce2320bd..3367b0383a 100644 --- a/zerver/tests/test_notification_data.py +++ b/zerver/tests/test_notification_data.py @@ -2,7 +2,7 @@ from zerver.actions.user_groups import check_add_user_group from zerver.lib.mention import MentionBackend, MentionData from zerver.lib.notification_data import UserMessageNotificationsData, get_user_group_mentions_data from zerver.lib.test_classes import ZulipTestCase -from zerver.models import NotificationTriggers +from zerver.models.scheduled_jobs import NotificationTriggers class TestNotificationData(ZulipTestCase): diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py index cd3e29a4ce..27b72d9595 100644 --- a/zerver/tests/test_outgoing_webhook_interfaces.py +++ b/zerver/tests/test_outgoing_webhook_interfaces.py @@ -12,8 +12,9 @@ from zerver.lib.outgoing_webhook import get_service_interface_class, process_suc from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import TOPIC_NAME -from zerver.models import SLACK_INTERFACE, Message, NotificationTriggers +from zerver.models import SLACK_INTERFACE, Message from zerver.models.realms import get_realm +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream from zerver.models.users import get_user from zerver.openapi.openapi import validate_against_openapi_schema diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 45c901fa09..5a624e3e6b 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -80,7 +80,6 @@ from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.user_counts import realm_user_count_by_role from zerver.models import ( Message, - NotificationTriggers, PushDeviceToken, Realm, RealmAuditLog, @@ -93,6 +92,7 @@ from zerver.models import ( ) from zerver.models.clients import get_client from zerver.models.realms import get_realm +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream from zilencer.models import RemoteZulipServerAuditLog from zilencer.views import DevicesToCleanUpDict diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 8677f6d931..2e54549ba0 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -26,7 +26,6 @@ from zerver.lib.send_email import EmailNotDeliveredError, FromAddress from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import mock_queue_publish from zerver.models import ( - NotificationTriggers, PreregistrationUser, ScheduledMessageNotificationEmail, UserActivity, @@ -34,6 +33,7 @@ from zerver.models import ( ) from zerver.models.clients import get_client from zerver.models.realms import get_realm +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.streams import get_stream from zerver.tornado.event_queue import build_offline_notification from zerver.worker import queue_processors diff --git a/zerver/tests/test_service_bot_system.py b/zerver/tests/test_service_bot_system.py index 131b30d803..ce1a0805fb 100644 --- a/zerver/tests/test_service_bot_system.py +++ b/zerver/tests/test_service_bot_system.py @@ -15,8 +15,9 @@ from zerver.lib.bot_storage import StateError from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import mock_queue_publish from zerver.lib.validator import check_string -from zerver.models import NotificationTriggers, Recipient, UserProfile +from zerver.models import Recipient, UserProfile from zerver.models.realms import get_realm +from zerver.models.scheduled_jobs import NotificationTriggers BOT_TYPE_TO_QUEUE_NAME = { UserProfile.OUTGOING_WEBHOOK_BOT: "outgoing_webhooks", diff --git a/zerver/tests/test_settings.py b/zerver/tests/test_settings.py index a8c9a44b43..2862388040 100644 --- a/zerver/tests/test_settings.py +++ b/zerver/tests/test_settings.py @@ -11,12 +11,8 @@ from zerver.lib.initial_password import initial_password from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file, ratelimit_rule from zerver.lib.users import get_all_api_keys -from zerver.models import ( - Draft, - NotificationTriggers, - ScheduledMessageNotificationEmail, - UserProfile, -) +from zerver.models import Draft, ScheduledMessageNotificationEmail, UserProfile +from zerver.models.scheduled_jobs import NotificationTriggers from zerver.models.users import get_user_profile_by_api_key if TYPE_CHECKING: