From eda000899b35c69e475fb311646cbd1201f9d319 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 14 Apr 2022 14:55:52 -0700 Subject: [PATCH] actions: Split out zerver.actions.message_edit. Signed-off-by: Anders Kaseorg --- static/js/message_edit.js | 4 +- static/shared/js/resolved_topic.js | 2 +- zerver/actions/message_edit.py | 1095 +++++++++++++++++++++++ zerver/lib/actions.py | 1091 +--------------------- zerver/tests/test_events.py | 8 +- zerver/tests/test_example.py | 4 +- zerver/tests/test_link_embed.py | 4 +- zerver/tests/test_message_edit.py | 24 +- zerver/tests/test_message_fetch.py | 3 +- zerver/tests/test_push_notifications.py | 2 +- zerver/tests/test_retention.py | 3 +- zerver/views/message_edit.py | 2 +- zerver/views/streams.py | 2 +- zerver/worker/queue_processors.py | 4 +- 14 files changed, 1133 insertions(+), 1115 deletions(-) create mode 100644 zerver/actions/message_edit.py diff --git a/static/js/message_edit.js b/static/js/message_edit.js index 736db7d1ab..06571d4e28 100644 --- a/static/js/message_edit.js +++ b/static/js/message_edit.js @@ -401,7 +401,7 @@ function edit_message($row, raw_content) { // been able to click it at the time the mouse entered the message_row. Also // a buffer in case their computer is slow, or stalled for a second, etc // If you change this number also change edit_limit_buffer in - // zerver.lib.actions.check_update_message + // zerver.actions.message_edit.check_update_message const seconds_left_buffer = 5; const editability = get_editability(message, seconds_left_buffer); const max_file_upload_size = page_params.max_file_upload_size_mib; @@ -534,7 +534,7 @@ function edit_message($row, raw_content) { ) { // Give them at least 10 seconds. // If you change this number also change edit_limit_buffer in - // zerver.lib.actions.check_update_message + // zerver.actions.message_edit.check_update_message const min_seconds_to_edit = 10; let seconds_left = page_params.realm_message_content_edit_limit_seconds + diff --git a/static/shared/js/resolved_topic.js b/static/shared/js/resolved_topic.js index a71286c08d..b43694ff28 100644 --- a/static/shared/js/resolved_topic.js +++ b/static/shared/js/resolved_topic.js @@ -8,7 +8,7 @@ export const RESOLVED_TOPIC_PREFIX = "✔ "; */ // The class has the same characters as RESOLVED_TOPIC_PREFIX. // It's designed to remove a weird "✔ ✔✔ " prefix, if present. -// Compare maybe_send_resolve_topic_notifications in zerver/lib/actions.py. +// Compare maybe_send_resolve_topic_notifications in zerver/actions/message_edit.py. const RESOLVED_TOPIC_PREFIX_RE = /^✔ [ ✔]*/; export function is_resolved(topic_name) { diff --git a/zerver/actions/message_edit.py b/zerver/actions/message_edit.py new file mode 100644 index 0000000000..c710a6f02a --- /dev/null +++ b/zerver/actions/message_edit.py @@ -0,0 +1,1095 @@ +import datetime +from typing import Any, Dict, Iterable, List, Optional, Set + +from django.conf import settings +from django.db import transaction +from django.utils.timezone import now as timezone_now +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy +from django.utils.translation import override as override_language +from typing_extensions import TypedDict + +from zerver.actions.message_flags import do_update_mobile_push_notification +from zerver.actions.message_send import ( + filter_presence_idle_user_ids, + get_recipient_info, + internal_send_stream_message, + render_incoming_message, +) +from zerver.actions.uploads import check_attachment_reference_change +from zerver.actions.user_topics import do_mute_topic, do_unmute_topic +from zerver.lib import retention as retention +from zerver.lib.exceptions import JsonableError +from zerver.lib.markdown import MessageRenderingResult, topic_links +from zerver.lib.markdown import version as markdown_version +from zerver.lib.mention import MentionBackend, MentionData, silent_mention_syntax_for_user +from zerver.lib.message import ( + access_message, + bulk_access_messages, + normalize_body, + truncate_topic, + update_to_dict_cache, + wildcard_mention_allowed, +) +from zerver.lib.queue import queue_json_publish +from zerver.lib.retention import move_messages_to_archive +from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id +from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.streams import access_stream_by_id, check_stream_access_based_on_stream_post_policy +from zerver.lib.string_validation import check_stream_topic +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.topic import ( + ORIG_TOPIC, + RESOLVED_TOPIC_PREFIX, + TOPIC_LINKS, + TOPIC_NAME, + messages_for_topic, + save_message_for_edit_use_case, + update_edit_history, + update_messages_for_topic_edit, +) +from zerver.lib.types import EditHistoryEvent +from zerver.lib.user_message import UserMessageLite, bulk_insert_ums +from zerver.lib.user_topics import get_users_muting_topic, remove_topic_mute +from zerver.lib.widget import is_widget_message +from zerver.models import ( + ArchivedAttachment, + Attachment, + Message, + Reaction, + Realm, + Stream, + UserMessage, + UserProfile, + get_stream_by_id_in_realm, + get_system_bot, +) +from zerver.tornado.django_api import send_event + + +def subscriber_info(user_id: int) -> Dict[str, Any]: + return {"id": user_id, "flags": ["read"]} + + +def validate_message_edit_payload( + message: Message, + stream_id: Optional[int], + topic_name: Optional[str], + propagate_mode: Optional[str], + content: Optional[str], +) -> None: + """ + Checks that the data sent is well-formed. Does not handle editability, permissions etc. + """ + if topic_name is None and content is None and stream_id is None: + raise JsonableError(_("Nothing to change")) + + if not message.is_stream_message(): + if stream_id is not None: + raise JsonableError(_("Private messages cannot be moved to streams.")) + if topic_name is not None: + raise JsonableError(_("Private messages cannot have topics.")) + + if propagate_mode != "change_one" and topic_name is None and stream_id is None: + raise JsonableError(_("Invalid propagate_mode without topic edit")) + + if topic_name is not None: + check_stream_topic(topic_name) + + if stream_id is not None and content is not None: + raise JsonableError(_("Cannot change message content while changing stream")) + + # Right now, we prevent users from editing widgets. + if content is not None and is_widget_message(message): + raise JsonableError(_("Widgets cannot be edited.")) + + +def can_edit_content_or_topic( + message: Message, + user_profile: UserProfile, + is_no_topic_msg: bool, + content: Optional[str] = None, + topic_name: Optional[str] = None, +) -> bool: + # You have permission to edit the message (both content and topic) if you sent it. + if message.sender_id == user_profile.id: + return True + + # You cannot edit the content of message sent by someone else. + if content is not None: + return False + + assert topic_name is not None + + # The following cases are the various reasons a user might be + # allowed to edit topics. + + # We allow anyone to edit (no topic) messages to help tend them. + if is_no_topic_msg: + return True + + # The can_edit_topic_of_any_message helper returns whether the user can edit the topic + # or not based on edit_topic_policy setting and the user's role. + if user_profile.can_edit_topic_of_any_message(): + return True + + return False + + +class MessageUpdateUserInfoResult(TypedDict): + message_user_ids: Set[int] + mention_user_ids: Set[int] + + +def maybe_send_resolve_topic_notifications( + *, + user_profile: UserProfile, + stream: Stream, + old_topic: str, + new_topic: str, + changed_messages: List[Message], +) -> None: + # Note that topics will have already been stripped in check_update_message. + # + # This logic is designed to treat removing a weird "✔ ✔✔ " + # prefix as unresolving the topic. + if old_topic.lstrip(RESOLVED_TOPIC_PREFIX) != new_topic.lstrip(RESOLVED_TOPIC_PREFIX): + return + + topic_resolved: bool = new_topic.startswith(RESOLVED_TOPIC_PREFIX) and not old_topic.startswith( + RESOLVED_TOPIC_PREFIX + ) + topic_unresolved: bool = old_topic.startswith( + RESOLVED_TOPIC_PREFIX + ) and not new_topic.startswith(RESOLVED_TOPIC_PREFIX) + + if not topic_resolved and not topic_unresolved: + # If there's some other weird topic that does not toggle the + # state of "topic starts with RESOLVED_TOPIC_PREFIX", we do + # nothing. Any other logic could result in cases where we send + # these notifications in a non-alternating fashion. + # + # Note that it is still possible for an individual topic to + # have multiple "This topic was marked as resolved" + # notifications in a row: one can send new messages to the + # pre-resolve topic and then resolve the topic created that + # way to get multiple in the resolved topic. And then an + # administrator can the messages in between. We consider this + # to be a fundamental risk of irresponsible message deletion, + # not a bug with the "resolve topics" feature. + return + + # Compute the users who either sent or reacted to messages that + # were moved via the "resolve topic' action. Only those users + # should be eligible for this message being managed as unread. + affected_participant_ids = {message.sender_id for message in changed_messages} | set( + Reaction.objects.filter(message__in=changed_messages).values_list( + "user_profile_id", flat=True + ) + ) + sender = get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id) + user_mention = silent_mention_syntax_for_user(user_profile) + with override_language(stream.realm.default_language): + if topic_resolved: + notification_string = _("{user} has marked this topic as resolved.") + elif topic_unresolved: + notification_string = _("{user} has marked this topic as unresolved.") + + internal_send_stream_message( + sender, + stream, + new_topic, + notification_string.format( + user=user_mention, + ), + limit_unread_user_ids=affected_participant_ids, + ) + + +def send_message_moved_breadcrumbs( + user_profile: UserProfile, + old_stream: Stream, + old_topic: str, + old_thread_notification_string: Optional[str], + new_stream: Stream, + new_topic: Optional[str], + new_thread_notification_string: Optional[str], + changed_messages_count: int, +) -> None: + # Since moving content between streams is highly disruptive, + # it's worth adding a couple tombstone messages showing what + # happened. + sender = get_system_bot(settings.NOTIFICATION_BOT, old_stream.realm_id) + + if new_topic is None: + new_topic = old_topic + + user_mention = silent_mention_syntax_for_user(user_profile) + old_topic_link = f"#**{old_stream.name}>{old_topic}**" + new_topic_link = f"#**{new_stream.name}>{new_topic}**" + + if new_thread_notification_string is not None: + with override_language(new_stream.realm.default_language): + internal_send_stream_message( + sender, + new_stream, + new_topic, + new_thread_notification_string.format( + old_location=old_topic_link, + user=user_mention, + changed_messages_count=changed_messages_count, + ), + ) + + if old_thread_notification_string is not None: + with override_language(old_stream.realm.default_language): + # Send a notification to the old stream that the topic was moved. + internal_send_stream_message( + sender, + old_stream, + old_topic, + old_thread_notification_string.format( + user=user_mention, + new_location=new_topic_link, + changed_messages_count=changed_messages_count, + ), + ) + + +def get_user_info_for_message_updates(message_id: int) -> MessageUpdateUserInfoResult: + + # We exclude UserMessage.flags.historical rows since those + # users did not receive the message originally, and thus + # probably are not relevant for reprocessed alert_words, + # mentions and similar rendering features. This may be a + # decision we change in the future. + query = UserMessage.objects.filter( + message=message_id, + flags=~UserMessage.flags.historical, + ).values("user_profile_id", "flags") + rows = list(query) + + message_user_ids = {row["user_profile_id"] for row in rows} + + mask = UserMessage.flags.mentioned | UserMessage.flags.wildcard_mentioned + + mention_user_ids = {row["user_profile_id"] for row in rows if int(row["flags"]) & mask} + + return dict( + message_user_ids=message_user_ids, + mention_user_ids=mention_user_ids, + ) + + +def update_user_message_flags( + rendering_result: MessageRenderingResult, ums: Iterable[UserMessage] +) -> None: + wildcard = rendering_result.mentions_wildcard + mentioned_ids = rendering_result.mentions_user_ids + ids_with_alert_words = rendering_result.user_ids_with_alert_words + changed_ums: Set[UserMessage] = set() + + def update_flag(um: UserMessage, should_set: bool, flag: int) -> None: + if should_set: + if not (um.flags & flag): + um.flags |= flag + changed_ums.add(um) + else: + if um.flags & flag: + um.flags &= ~flag + changed_ums.add(um) + + for um in ums: + has_alert_word = um.user_profile_id in ids_with_alert_words + update_flag(um, has_alert_word, UserMessage.flags.has_alert_word) + + mentioned = um.user_profile_id in mentioned_ids + update_flag(um, mentioned, UserMessage.flags.mentioned) + + update_flag(um, wildcard, UserMessage.flags.wildcard_mentioned) + + for um in changed_ums: + um.save(update_fields=["flags"]) + + +def do_update_embedded_data( + user_profile: UserProfile, + message: Message, + content: Optional[str], + rendering_result: MessageRenderingResult, +) -> None: + timestamp = timezone_now() + event: Dict[str, Any] = { + "type": "update_message", + "user_id": None, + "edit_timestamp": datetime_to_timestamp(timestamp), + "message_id": message.id, + "rendering_only": True, + } + changed_messages = [message] + rendered_content: Optional[str] = None + + ums = UserMessage.objects.filter(message=message.id) + + if content is not None: + update_user_message_flags(rendering_result, ums) + rendered_content = rendering_result.rendered_content + message.rendered_content = rendered_content + message.rendered_content_version = markdown_version + event["content"] = content + event["rendered_content"] = rendered_content + + message.save(update_fields=["content", "rendered_content"]) + + event["message_ids"] = update_to_dict_cache(changed_messages) + + def user_info(um: UserMessage) -> Dict[str, Any]: + return { + "id": um.user_profile_id, + "flags": um.flags_list(), + } + + send_event(user_profile.realm, event, list(map(user_info, ums))) + + +class DeleteMessagesEvent(TypedDict, total=False): + type: str + message_ids: List[int] + message_type: str + topic: str + stream_id: int + + +# We use transaction.atomic to support select_for_update in the attachment codepath. +@transaction.atomic(savepoint=False) +def do_update_message( + user_profile: UserProfile, + target_message: Message, + new_stream: Optional[Stream], + topic_name: Optional[str], + propagate_mode: str, + send_notification_to_old_thread: bool, + send_notification_to_new_thread: bool, + content: Optional[str], + rendering_result: Optional[MessageRenderingResult], + prior_mention_user_ids: Set[int], + mention_data: Optional[MentionData] = None, +) -> int: + """ + The main function for message editing. A message edit event can + modify: + * the message's content (in which case the caller will have + set both content and rendered_content), + * the topic, in which case the caller will have set topic_name + * or both message's content and the topic + * or stream and/or topic, in which case the caller will have set + new_stream and/or topic_name. + + With topic edits, propagate_mode determines whether other message + also have their topics edited. + """ + timestamp = timezone_now() + target_message.last_edit_time = timestamp + + event: Dict[str, Any] = { + "type": "update_message", + "user_id": user_profile.id, + "edit_timestamp": datetime_to_timestamp(timestamp), + "message_id": target_message.id, + "rendering_only": False, + } + + edit_history_event: EditHistoryEvent = { + "user_id": user_profile.id, + "timestamp": event["edit_timestamp"], + } + + changed_messages = [target_message] + + realm = user_profile.realm + + stream_being_edited = None + if target_message.is_stream_message(): + stream_id = target_message.recipient.type_id + stream_being_edited = get_stream_by_id_in_realm(stream_id, realm) + event["stream_name"] = stream_being_edited.name + event["stream_id"] = stream_being_edited.id + + ums = UserMessage.objects.filter(message=target_message.id) + + if content is not None: + assert rendering_result is not None + + # mention_data is required if there's a content edit. + assert mention_data is not None + + # add data from group mentions to mentions_user_ids. + for group_id in rendering_result.mentions_user_group_ids: + members = mention_data.get_group_members(group_id) + rendering_result.mentions_user_ids.update(members) + + update_user_message_flags(rendering_result, ums) + + # One could imagine checking realm.allow_edit_history here and + # modifying the events based on that setting, but doing so + # doesn't really make sense. We need to send the edit event + # to clients regardless, and a client already had access to + # the original/pre-edit content of the message anyway. That + # setting must be enforced on the client side, and making a + # change here simply complicates the logic for clients parsing + # edit history events. + event["orig_content"] = target_message.content + event["orig_rendered_content"] = target_message.rendered_content + edit_history_event["prev_content"] = target_message.content + edit_history_event["prev_rendered_content"] = target_message.rendered_content + edit_history_event[ + "prev_rendered_content_version" + ] = target_message.rendered_content_version + target_message.content = content + target_message.rendered_content = rendering_result.rendered_content + target_message.rendered_content_version = markdown_version + event["content"] = content + event["rendered_content"] = rendering_result.rendered_content + event["prev_rendered_content_version"] = target_message.rendered_content_version + event["is_me_message"] = Message.is_status_message( + content, rendering_result.rendered_content + ) + + # target_message.has_image and target_message.has_link will have been + # already updated by Markdown rendering in the caller. + target_message.has_attachment = check_attachment_reference_change( + target_message, rendering_result + ) + + if target_message.is_stream_message(): + if topic_name is not None: + new_topic_name = topic_name + else: + new_topic_name = target_message.topic_name() + + stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget( + stream_id=stream_id, + topic_name=new_topic_name, + ) + else: + stream_topic = None + + info = get_recipient_info( + realm_id=realm.id, + recipient=target_message.recipient, + sender_id=target_message.sender_id, + stream_topic=stream_topic, + possible_wildcard_mention=mention_data.message_has_wildcards(), + ) + + event["online_push_user_ids"] = list(info["online_push_user_ids"]) + event["pm_mention_push_disabled_user_ids"] = list(info["pm_mention_push_disabled_user_ids"]) + event["pm_mention_email_disabled_user_ids"] = list( + info["pm_mention_email_disabled_user_ids"] + ) + event["stream_push_user_ids"] = list(info["stream_push_user_ids"]) + event["stream_email_user_ids"] = list(info["stream_email_user_ids"]) + event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"]) + event["prior_mention_user_ids"] = list(prior_mention_user_ids) + event["presence_idle_user_ids"] = filter_presence_idle_user_ids(info["active_user_ids"]) + event["all_bot_user_ids"] = list(info["all_bot_user_ids"]) + if rendering_result.mentions_wildcard: + event["wildcard_mention_user_ids"] = list(info["wildcard_mention_user_ids"]) + else: + event["wildcard_mention_user_ids"] = [] + + do_update_mobile_push_notification( + target_message, + prior_mention_user_ids, + rendering_result.mentions_user_ids, + info["stream_push_user_ids"], + ) + + if topic_name is not None or new_stream is not None: + orig_topic_name = target_message.topic_name() + event["propagate_mode"] = propagate_mode + + if new_stream is not None: + assert content is None + assert target_message.is_stream_message() + assert stream_being_edited is not None + + edit_history_event["prev_stream"] = stream_being_edited.id + edit_history_event["stream"] = new_stream.id + event[ORIG_TOPIC] = orig_topic_name + target_message.recipient_id = new_stream.recipient_id + + event["new_stream_id"] = new_stream.id + event["propagate_mode"] = propagate_mode + + # When messages are moved from one stream to another, some + # users may lose access to those messages, including guest + # users and users not subscribed to the new stream (if it is a + # private stream). For those users, their experience is as + # though the messages were deleted, and we should send a + # delete_message event to them instead. + + subs_to_old_stream = get_active_subscriptions_for_stream_id( + stream_id, include_deactivated_users=True + ).select_related("user_profile") + subs_to_new_stream = list( + get_active_subscriptions_for_stream_id( + new_stream.id, include_deactivated_users=True + ).select_related("user_profile") + ) + + old_stream_sub_ids = [user.user_profile_id for user in subs_to_old_stream] + new_stream_sub_ids = [user.user_profile_id for user in subs_to_new_stream] + + # Get users who aren't subscribed to the new_stream. + subs_losing_usermessages = [ + sub for sub in subs_to_old_stream if sub.user_profile_id not in new_stream_sub_ids + ] + # Users who can longer access the message without some action + # from administrators. + subs_losing_access = [ + sub + for sub in subs_losing_usermessages + if sub.user_profile.is_guest or not new_stream.is_public() + ] + ums = ums.exclude( + user_profile_id__in=[sub.user_profile_id for sub in subs_losing_usermessages] + ) + + subs_gaining_usermessages = [] + if not new_stream.is_history_public_to_subscribers(): + # For private streams, with history not public to subscribers, + # We find out users who are not present in the msgs' old stream + # and create new UserMessage for these users so that they can + # access this message. + subs_gaining_usermessages += [ + user_id for user_id in new_stream_sub_ids if user_id not in old_stream_sub_ids + ] + + if topic_name is not None: + topic_name = truncate_topic(topic_name) + target_message.set_topic_name(topic_name) + + # These fields have legacy field names. + event[ORIG_TOPIC] = orig_topic_name + event[TOPIC_NAME] = topic_name + event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id, topic_name) + edit_history_event["prev_topic"] = orig_topic_name + edit_history_event["topic"] = topic_name + + update_edit_history(target_message, timestamp, edit_history_event) + + delete_event_notify_user_ids: List[int] = [] + if propagate_mode in ["change_later", "change_all"]: + assert topic_name is not None or new_stream is not None + assert stream_being_edited is not None + + # Other messages should only get topic/stream fields in their edit history. + topic_only_edit_history_event: EditHistoryEvent = { + "user_id": edit_history_event["user_id"], + "timestamp": edit_history_event["timestamp"], + } + if topic_name is not None: + topic_only_edit_history_event["prev_topic"] = edit_history_event["prev_topic"] + topic_only_edit_history_event["topic"] = edit_history_event["topic"] + if new_stream is not None: + topic_only_edit_history_event["prev_stream"] = edit_history_event["prev_stream"] + topic_only_edit_history_event["stream"] = edit_history_event["stream"] + + messages_list = update_messages_for_topic_edit( + acting_user=user_profile, + edited_message=target_message, + propagate_mode=propagate_mode, + orig_topic_name=orig_topic_name, + topic_name=topic_name, + new_stream=new_stream, + old_stream=stream_being_edited, + edit_history_event=topic_only_edit_history_event, + last_edit_time=timestamp, + ) + changed_messages += messages_list + + if new_stream is not None: + assert stream_being_edited is not None + changed_message_ids = [msg.id for msg in changed_messages] + + if subs_gaining_usermessages: + ums_to_create = [] + for message_id in changed_message_ids: + for user_profile_id in subs_gaining_usermessages: + # The fact that the user didn't have a UserMessage originally means we can infer that the user + # was not mentioned in the original message (even if mention syntax was present, it would not + # take effect for a user who was not subscribed). If we were editing the message's content, we + # would rerender the message and then use the new stream's data to determine whether this is + # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this + # mention syntax an actual mention" decision made during the original rendering for implementation + # simplicity. As a result, the only flag to consider applying here is read. + um = UserMessageLite( + user_profile_id=user_profile_id, + message_id=message_id, + flags=UserMessage.flags.read, + ) + ums_to_create.append(um) + bulk_insert_ums(ums_to_create) + + # Delete UserMessage objects for users who will no + # longer have access to these messages. Note: This could be + # very expensive, since it's N guest users x M messages. + UserMessage.objects.filter( + user_profile_id__in=[sub.user_profile_id for sub in subs_losing_usermessages], + message_id__in=changed_message_ids, + ).delete() + + delete_event: DeleteMessagesEvent = { + "type": "delete_message", + "message_ids": changed_message_ids, + "message_type": "stream", + "stream_id": stream_being_edited.id, + "topic": orig_topic_name, + } + delete_event_notify_user_ids = [sub.user_profile_id for sub in subs_losing_access] + send_event(user_profile.realm, delete_event, delete_event_notify_user_ids) + + # Reset the Attachment.is_*_public caches for all messages + # moved to another stream with different access permissions. + if new_stream.invite_only != stream_being_edited.invite_only: + Attachment.objects.filter(messages__in=changed_message_ids).update( + is_realm_public=None, + ) + ArchivedAttachment.objects.filter(messages__in=changed_message_ids).update( + is_realm_public=None, + ) + + if new_stream.is_web_public != stream_being_edited.is_web_public: + Attachment.objects.filter(messages__in=changed_message_ids).update( + is_web_public=None, + ) + ArchivedAttachment.objects.filter(messages__in=changed_message_ids).update( + is_web_public=None, + ) + + # This does message.save(update_fields=[...]) + save_message_for_edit_use_case(message=target_message) + + realm_id: Optional[int] = None + if stream_being_edited is not None: + realm_id = stream_being_edited.realm_id + + event["message_ids"] = update_to_dict_cache(changed_messages, realm_id) + + def user_info(um: UserMessage) -> Dict[str, Any]: + return { + "id": um.user_profile_id, + "flags": um.flags_list(), + } + + # The following blocks arranges that users who are subscribed to a + # stream and can see history from before they subscribed get + # live-update when old messages are edited (e.g. if the user does + # a topic edit themself). + # + # We still don't send an update event to users who are not + # subscribed to this stream and don't have a UserMessage row. This + # means if a non-subscriber is viewing the narrow, they won't get + # a real-time updates. This is a balance between sending + # message-edit notifications for every public stream to every user + # in the organization (too expansive, and also not what we do for + # newly sent messages anyway) and having magical live-updates + # where possible. + users_to_be_notified = list(map(user_info, ums)) + if stream_being_edited is not None: + if stream_being_edited.is_history_public_to_subscribers: + subscriptions = get_active_subscriptions_for_stream_id( + stream_id, include_deactivated_users=False + ) + # We exclude long-term idle users, since they by + # definition have no active clients. + subscriptions = subscriptions.exclude(user_profile__long_term_idle=True) + # Remove duplicates by excluding the id of users already + # in users_to_be_notified list. This is the case where a + # user both has a UserMessage row and is a current + # Subscriber + subscriptions = subscriptions.exclude( + user_profile_id__in=[um.user_profile_id for um in ums] + ) + + if new_stream is not None: + assert delete_event_notify_user_ids is not None + subscriptions = subscriptions.exclude( + user_profile_id__in=delete_event_notify_user_ids + ) + + # All users that are subscribed to the stream must be + # notified when a message is edited + subscriber_ids = set(subscriptions.values_list("user_profile_id", flat=True)) + + if new_stream is not None: + # TODO: Guest users don't see the new moved topic + # unless breadcrumb message for new stream is + # enabled. Excluding these users from receiving this + # event helps us avoid a error traceback for our + # clients. We should figure out a way to inform the + # guest users of this new topic if sending a 'message' + # event for these messages is not an option. + # + # Don't send this event to guest subs who are not + # subscribers of the old stream but are subscribed to + # the new stream; clients will be confused. + old_stream_unsubbed_guests = [ + sub + for sub in subs_to_new_stream + if sub.user_profile.is_guest and sub.user_profile_id not in subscriber_ids + ] + subscriptions = subscriptions.exclude( + user_profile_id__in=[sub.user_profile_id for sub in old_stream_unsubbed_guests] + ) + subscriber_ids = set(subscriptions.values_list("user_profile_id", flat=True)) + + users_to_be_notified += list(map(subscriber_info, sorted(list(subscriber_ids)))) + + # UserTopic updates and the content of notifications depend on + # whether we've moved the entire topic, or just part of it. We + # make that determination here. + moved_all_visible_messages = False + if topic_name is not None or new_stream is not None: + assert stream_being_edited is not None + + if propagate_mode == "change_all": + moved_all_visible_messages = True + else: + # With other propagate modes, if the user in fact moved + # all messages in the stream, we want to explain it was a + # full-topic move. + # + # For security model reasons, we don't want to allow a + # user to take any action that would leak information + # about older messages they cannot access (E.g. the only + # remaining messages are in a stream without shared + # history). The bulk_access_messages call below addresses + # that concern. + # + # bulk_access_messages is inefficient for this task, since + # we just want to do the exists() version of this + # query. But it's nice to reuse code, and this bulk + # operation is likely cheaper than a `GET /messages` + # unless the topic has thousands of messages of history. + unmoved_messages = messages_for_topic( + stream_being_edited.recipient_id, + orig_topic_name, + ) + visible_unmoved_messages = bulk_access_messages( + user_profile, unmoved_messages, stream=stream_being_edited + ) + moved_all_visible_messages = len(visible_unmoved_messages) == 0 + + # Migrate muted topic configuration in the following circumstances: + # + # * If propagate_mode is change_all, do so unconditionally. + # + # * If propagate_mode is change_later or change_one, do so when + # the acting user has moved the entire topic (as visible to them). + # + # This rule corresponds to checking moved_all_visible_messages. + # + # We may want more complex behavior in cases where one appears to + # be merging topics (E.g. there are existing messages in the + # target topic). + if moved_all_visible_messages: + assert stream_being_edited is not None + assert topic_name is not None or new_stream is not None + + for muting_user in get_users_muting_topic(stream_being_edited.id, orig_topic_name): + # TODO: Ideally, this would be a bulk update operation, + # because we are doing database operations in a loop here. + # + # This loop is only acceptable in production because it is + # rare for more than a few users to have muted an + # individual topic that is being moved; as of this + # writing, no individual topic in Zulip Cloud had been + # muted by more than 100 users. + + if new_stream is not None and muting_user.id in delete_event_notify_user_ids: + # If the messages are being moved to a stream the user + # cannot access, then we treat this as the + # messages/topic being deleted for this user. This is + # important for security reasons; we don't want to + # give users a UserTopic row in a stream they cannot + # access. Unmute the topic for such users. + do_unmute_topic(muting_user, stream_being_edited, orig_topic_name) + else: + # Otherwise, we move the muted topic record for the user. + # We call remove_topic_mute rather than do_unmute_topic to + # avoid sending two events with new muted topics in + # immediate succession; this is correct only because + # muted_topics events always send the full set of topics. + remove_topic_mute(muting_user, stream_being_edited.id, orig_topic_name) + do_mute_topic( + muting_user, + new_stream if new_stream is not None else stream_being_edited, + topic_name if topic_name is not None else orig_topic_name, + ignore_duplicate=True, + ) + + send_event(user_profile.realm, event, users_to_be_notified) + + if len(changed_messages) > 0 and new_stream is not None and stream_being_edited is not None: + # Notify users that the topic was moved. + changed_messages_count = len(changed_messages) + + old_thread_notification_string = None + if send_notification_to_old_thread: + if moved_all_visible_messages: + old_thread_notification_string = gettext_lazy( + "This topic was moved to {new_location} by {user}." + ) + elif changed_messages_count == 1: + old_thread_notification_string = gettext_lazy( + "A message was moved from this topic to {new_location} by {user}." + ) + else: + old_thread_notification_string = gettext_lazy( + "{changed_messages_count} messages were moved from this topic to {new_location} by {user}." + ) + + new_thread_notification_string = None + if send_notification_to_new_thread: + if moved_all_visible_messages: + new_thread_notification_string = gettext_lazy( + "This topic was moved here from {old_location} by {user}." + ) + elif changed_messages_count == 1: + new_thread_notification_string = gettext_lazy( + "A message was moved here from {old_location} by {user}." + ) + else: + new_thread_notification_string = gettext_lazy( + "{changed_messages_count} messages were moved here from {old_location} by {user}." + ) + + send_message_moved_breadcrumbs( + user_profile, + stream_being_edited, + orig_topic_name, + old_thread_notification_string, + new_stream, + topic_name, + new_thread_notification_string, + changed_messages_count, + ) + + if ( + topic_name is not None + and new_stream is None + and content is None + and len(changed_messages) > 0 + ): + assert stream_being_edited is not None + maybe_send_resolve_topic_notifications( + user_profile=user_profile, + stream=stream_being_edited, + old_topic=orig_topic_name, + new_topic=topic_name, + changed_messages=changed_messages, + ) + + return len(changed_messages) + + +def check_update_message( + user_profile: UserProfile, + message_id: int, + stream_id: Optional[int] = None, + topic_name: Optional[str] = None, + propagate_mode: str = "change_one", + send_notification_to_old_thread: bool = True, + send_notification_to_new_thread: bool = True, + content: Optional[str] = None, +) -> int: + """This will update a message given the message id and user profile. + It checks whether the user profile has the permission to edit the message + and raises a JsonableError if otherwise. + It returns the number changed. + """ + message, ignored_user_message = access_message(user_profile, message_id) + + if not user_profile.realm.allow_message_editing: + raise JsonableError(_("Your organization has turned off message editing")) + + # The zerver/views/message_edit.py call point already strips this + # via REQ_topic; so we can delete this line if we arrange a + # contract where future callers in the embedded bots system strip + # use REQ_topic as well (or otherwise are guaranteed to strip input). + if topic_name is not None: + topic_name = topic_name.strip() + if topic_name == message.topic_name(): + topic_name = None + + validate_message_edit_payload(message, stream_id, topic_name, propagate_mode, content) + + is_no_topic_msg = message.topic_name() == "(no topic)" + + if content is not None or topic_name is not None: + if not can_edit_content_or_topic( + message, user_profile, is_no_topic_msg, content, topic_name + ): + raise JsonableError(_("You don't have permission to edit this message")) + + # If there is a change to the content, check that it hasn't been too long + # Allow an extra 20 seconds since we potentially allow editing 15 seconds + # past the limit, and in case there are network issues, etc. The 15 comes + # from (min_seconds_to_edit + seconds_left_buffer) in message_edit.js; if + # you change this value also change those two parameters in message_edit.js. + edit_limit_buffer = 20 + if content is not None and user_profile.realm.message_content_edit_limit_seconds > 0: + deadline_seconds = user_profile.realm.message_content_edit_limit_seconds + edit_limit_buffer + if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds): + raise JsonableError(_("The time limit for editing this message has passed")) + + # If there is a change to the topic, check that the user is allowed to + # edit it and that it has not been too long. If this is not the user who + # sent the message, they are not the admin, and the time limit for editing + # topics is passed, raise an error. + if ( + topic_name is not None + and message.sender != user_profile + and not user_profile.is_realm_admin + and not user_profile.is_moderator + and not is_no_topic_msg + ): + deadline_seconds = Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS + edit_limit_buffer + if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds): + raise JsonableError(_("The time limit for editing this message's topic has passed")) + + rendering_result = None + links_for_embed: Set[str] = set() + prior_mention_user_ids: Set[int] = set() + mention_data: Optional[MentionData] = None + if content is not None: + if content.rstrip() == "": + content = "(deleted)" + content = normalize_body(content) + + mention_backend = MentionBackend(user_profile.realm_id) + mention_data = MentionData( + mention_backend=mention_backend, + content=content, + ) + user_info = get_user_info_for_message_updates(message.id) + prior_mention_user_ids = user_info["mention_user_ids"] + + # We render the message using the current user's realm; since + # the cross-realm bots never edit messages, this should be + # always correct. + # Note: If rendering fails, the called code will raise a JsonableError. + rendering_result = render_incoming_message( + message, + content, + user_info["message_user_ids"], + user_profile.realm, + mention_data=mention_data, + ) + links_for_embed |= rendering_result.links_for_preview + + if message.is_stream_message() and rendering_result.mentions_wildcard: + stream = access_stream_by_id(user_profile, message.recipient.type_id)[0] + if not wildcard_mention_allowed(message.sender, stream): + raise JsonableError( + _("You do not have permission to use wildcard mentions in this stream.") + ) + + new_stream = None + number_changed = 0 + + if stream_id is not None: + assert message.is_stream_message() + if not user_profile.can_move_messages_between_streams(): + raise JsonableError(_("You don't have permission to move this message")) + try: + access_stream_by_id(user_profile, message.recipient.type_id) + except JsonableError: + raise JsonableError( + _( + "You don't have permission to move this message due to missing access to its stream" + ) + ) + + new_stream = access_stream_by_id(user_profile, stream_id, require_active=True)[0] + check_stream_access_based_on_stream_post_policy(user_profile, new_stream) + + number_changed = do_update_message( + user_profile, + message, + new_stream, + topic_name, + propagate_mode, + send_notification_to_old_thread, + send_notification_to_new_thread, + content, + rendering_result, + prior_mention_user_ids, + mention_data, + ) + + if links_for_embed: + event_data = { + "message_id": message.id, + "message_content": message.content, + # The choice of `user_profile.realm_id` rather than + # `sender.realm_id` must match the decision made in the + # `render_incoming_message` call earlier in this function. + "message_realm_id": user_profile.realm_id, + "urls": list(links_for_embed), + } + queue_json_publish("embed_links", event_data) + + return number_changed + + +def do_delete_messages(realm: Realm, messages: Iterable[Message]) -> None: + # messages in delete_message event belong to the same topic + # or is a single private message, as any other behaviour is not possible with + # the current callers to this method. + messages = list(messages) + message_ids = [message.id for message in messages] + if not message_ids: + return + + event: DeleteMessagesEvent = { + "type": "delete_message", + "message_ids": message_ids, + } + + sample_message = messages[0] + message_type = "stream" + users_to_notify = [] + if not sample_message.is_stream_message(): + assert len(messages) == 1 + message_type = "private" + ums = UserMessage.objects.filter(message_id__in=message_ids) + users_to_notify = [um.user_profile_id for um in ums] + archiving_chunk_size = retention.MESSAGE_BATCH_SIZE + + if message_type == "stream": + stream_id = sample_message.recipient.type_id + event["stream_id"] = stream_id + event["topic"] = sample_message.topic_name() + subscriptions = get_active_subscriptions_for_stream_id( + stream_id, include_deactivated_users=False + ) + # We exclude long-term idle users, since they by definition have no active clients. + subscriptions = subscriptions.exclude(user_profile__long_term_idle=True) + users_to_notify = list(subscriptions.values_list("user_profile_id", flat=True)) + archiving_chunk_size = retention.STREAM_MESSAGE_BATCH_SIZE + + move_messages_to_archive(message_ids, realm=realm, chunk_size=archiving_chunk_size) + + event["message_type"] = message_type + transaction.on_commit(lambda: send_event(realm, event, users_to_notify)) + + +def do_delete_messages_by_sender(user: UserProfile) -> None: + message_ids = list( + Message.objects.filter(sender=user).values_list("id", flat=True).order_by("id") + ) + if message_ids: + move_messages_to_archive(message_ids, chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE) diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 1fe07629ed..36fe9208e2 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -1,6 +1,6 @@ import datetime import logging -from typing import Any, Dict, Iterable, List, Optional, Sequence, Set +from typing import Any, Dict, List, Optional, Sequence import orjson from django.conf import settings @@ -8,79 +8,29 @@ from django.core.exceptions import ValidationError from django.db import transaction from django.utils.timezone import now as timezone_now from django.utils.translation import gettext as _ -from django.utils.translation import gettext_lazy -from django.utils.translation import override as override_language -from typing_extensions import TypedDict from confirmation.models import Confirmation, create_confirmation_link, generate_key from zerver.actions.custom_profile_fields import do_remove_realm_custom_profile_fields -from zerver.actions.message_flags import do_update_mobile_push_notification -from zerver.actions.message_send import ( - filter_presence_idle_user_ids, - get_recipient_info, - internal_send_stream_message, - render_incoming_message, -) -from zerver.actions.uploads import check_attachment_reference_change +from zerver.actions.message_edit import do_delete_messages_by_sender +from zerver.actions.message_send import internal_send_stream_message from zerver.actions.user_groups import update_users_in_full_members_system_group from zerver.actions.user_settings import do_delete_avatar_image, send_user_email_update_event -from zerver.actions.user_topics import do_mute_topic, do_unmute_topic -from zerver.lib import retention as retention from zerver.lib.bulk_create import create_users from zerver.lib.cache import flush_user_profile from zerver.lib.create_user import get_display_email_address from zerver.lib.email_validation import email_reserved_for_system_bots_error -from zerver.lib.exceptions import JsonableError -from zerver.lib.markdown import MessageRenderingResult, topic_links -from zerver.lib.markdown import version as markdown_version -from zerver.lib.mention import MentionBackend, MentionData, silent_mention_syntax_for_user -from zerver.lib.message import ( - access_message, - bulk_access_messages, - normalize_body, - truncate_topic, - update_first_visible_message_id, - update_to_dict_cache, - wildcard_mention_allowed, -) -from zerver.lib.queue import queue_json_publish -from zerver.lib.retention import move_messages_to_archive +from zerver.lib.message import update_first_visible_message_id from zerver.lib.send_email import FromAddress, send_email_to_admins from zerver.lib.server_initialization import create_internal_realm, server_initialized from zerver.lib.sessions import delete_user_sessions -from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id -from zerver.lib.stream_topic import StreamTopicTarget -from zerver.lib.streams import ( - access_stream_by_id, - check_stream_access_based_on_stream_post_policy, - ensure_stream, - get_signups_stream, -) -from zerver.lib.string_validation import check_stream_topic -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - ORIG_TOPIC, - RESOLVED_TOPIC_PREFIX, - TOPIC_LINKS, - TOPIC_NAME, - filter_by_topic_name_via_message, - messages_for_topic, - save_message_for_edit_use_case, - update_edit_history, - update_messages_for_topic_edit, -) -from zerver.lib.types import EditHistoryEvent +from zerver.lib.streams import ensure_stream, get_signups_stream +from zerver.lib.topic import filter_by_topic_name_via_message from zerver.lib.user_counts import realm_user_count_by_role from zerver.lib.user_groups import create_system_user_groups_for_realm -from zerver.lib.user_message import UserMessageLite, bulk_insert_ums -from zerver.lib.user_topics import get_users_muting_topic, remove_topic_mute -from zerver.lib.widget import is_widget_message from zerver.models import ( - ArchivedAttachment, Attachment, DefaultStream, Message, - Reaction, Realm, RealmAuditLog, RealmDomain, @@ -92,7 +42,6 @@ from zerver.models import ( active_user_ids, get_realm, get_realm_domains, - get_stream_by_id_in_realm, get_system_bot, is_cross_realm_bot_email, ) @@ -102,10 +51,6 @@ if settings.BILLING_ENABLED: from corporate.lib.stripe import downgrade_now_without_creating_additional_invoices -def subscriber_info(user_id: int) -> Dict[str, Any]: - return {"id": user_id, "flags": ["read"]} - - def active_humans_in_realm(realm: Realm) -> Sequence[UserProfile]: return UserProfile.objects.filter(realm=realm, is_active=True, is_bot=False) @@ -500,221 +445,6 @@ def do_scrub_realm(realm: Realm, *, acting_user: Optional[UserProfile]) -> None: ) -def validate_message_edit_payload( - message: Message, - stream_id: Optional[int], - topic_name: Optional[str], - propagate_mode: Optional[str], - content: Optional[str], -) -> None: - """ - Checks that the data sent is well-formed. Does not handle editability, permissions etc. - """ - if topic_name is None and content is None and stream_id is None: - raise JsonableError(_("Nothing to change")) - - if not message.is_stream_message(): - if stream_id is not None: - raise JsonableError(_("Private messages cannot be moved to streams.")) - if topic_name is not None: - raise JsonableError(_("Private messages cannot have topics.")) - - if propagate_mode != "change_one" and topic_name is None and stream_id is None: - raise JsonableError(_("Invalid propagate_mode without topic edit")) - - if topic_name is not None: - check_stream_topic(topic_name) - - if stream_id is not None and content is not None: - raise JsonableError(_("Cannot change message content while changing stream")) - - # Right now, we prevent users from editing widgets. - if content is not None and is_widget_message(message): - raise JsonableError(_("Widgets cannot be edited.")) - - -def can_edit_content_or_topic( - message: Message, - user_profile: UserProfile, - is_no_topic_msg: bool, - content: Optional[str] = None, - topic_name: Optional[str] = None, -) -> bool: - # You have permission to edit the message (both content and topic) if you sent it. - if message.sender_id == user_profile.id: - return True - - # You cannot edit the content of message sent by someone else. - if content is not None: - return False - - assert topic_name is not None - - # The following cases are the various reasons a user might be - # allowed to edit topics. - - # We allow anyone to edit (no topic) messages to help tend them. - if is_no_topic_msg: - return True - - # The can_edit_topic_of_any_message helper returns whether the user can edit the topic - # or not based on edit_topic_policy setting and the user's role. - if user_profile.can_edit_topic_of_any_message(): - return True - - return False - - -def check_update_message( - user_profile: UserProfile, - message_id: int, - stream_id: Optional[int] = None, - topic_name: Optional[str] = None, - propagate_mode: str = "change_one", - send_notification_to_old_thread: bool = True, - send_notification_to_new_thread: bool = True, - content: Optional[str] = None, -) -> int: - """This will update a message given the message id and user profile. - It checks whether the user profile has the permission to edit the message - and raises a JsonableError if otherwise. - It returns the number changed. - """ - message, ignored_user_message = access_message(user_profile, message_id) - - if not user_profile.realm.allow_message_editing: - raise JsonableError(_("Your organization has turned off message editing")) - - # The zerver/views/message_edit.py call point already strips this - # via REQ_topic; so we can delete this line if we arrange a - # contract where future callers in the embedded bots system strip - # use REQ_topic as well (or otherwise are guaranteed to strip input). - if topic_name is not None: - topic_name = topic_name.strip() - if topic_name == message.topic_name(): - topic_name = None - - validate_message_edit_payload(message, stream_id, topic_name, propagate_mode, content) - - is_no_topic_msg = message.topic_name() == "(no topic)" - - if content is not None or topic_name is not None: - if not can_edit_content_or_topic( - message, user_profile, is_no_topic_msg, content, topic_name - ): - raise JsonableError(_("You don't have permission to edit this message")) - - # If there is a change to the content, check that it hasn't been too long - # Allow an extra 20 seconds since we potentially allow editing 15 seconds - # past the limit, and in case there are network issues, etc. The 15 comes - # from (min_seconds_to_edit + seconds_left_buffer) in message_edit.js; if - # you change this value also change those two parameters in message_edit.js. - edit_limit_buffer = 20 - if content is not None and user_profile.realm.message_content_edit_limit_seconds > 0: - deadline_seconds = user_profile.realm.message_content_edit_limit_seconds + edit_limit_buffer - if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds): - raise JsonableError(_("The time limit for editing this message has passed")) - - # If there is a change to the topic, check that the user is allowed to - # edit it and that it has not been too long. If this is not the user who - # sent the message, they are not the admin, and the time limit for editing - # topics is passed, raise an error. - if ( - topic_name is not None - and message.sender != user_profile - and not user_profile.is_realm_admin - and not user_profile.is_moderator - and not is_no_topic_msg - ): - deadline_seconds = Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS + edit_limit_buffer - if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds): - raise JsonableError(_("The time limit for editing this message's topic has passed")) - - rendering_result = None - links_for_embed: Set[str] = set() - prior_mention_user_ids: Set[int] = set() - mention_data: Optional[MentionData] = None - if content is not None: - if content.rstrip() == "": - content = "(deleted)" - content = normalize_body(content) - - mention_backend = MentionBackend(user_profile.realm_id) - mention_data = MentionData( - mention_backend=mention_backend, - content=content, - ) - user_info = get_user_info_for_message_updates(message.id) - prior_mention_user_ids = user_info["mention_user_ids"] - - # We render the message using the current user's realm; since - # the cross-realm bots never edit messages, this should be - # always correct. - # Note: If rendering fails, the called code will raise a JsonableError. - rendering_result = render_incoming_message( - message, - content, - user_info["message_user_ids"], - user_profile.realm, - mention_data=mention_data, - ) - links_for_embed |= rendering_result.links_for_preview - - if message.is_stream_message() and rendering_result.mentions_wildcard: - stream = access_stream_by_id(user_profile, message.recipient.type_id)[0] - if not wildcard_mention_allowed(message.sender, stream): - raise JsonableError( - _("You do not have permission to use wildcard mentions in this stream.") - ) - - new_stream = None - number_changed = 0 - - if stream_id is not None: - assert message.is_stream_message() - if not user_profile.can_move_messages_between_streams(): - raise JsonableError(_("You don't have permission to move this message")) - try: - access_stream_by_id(user_profile, message.recipient.type_id) - except JsonableError: - raise JsonableError( - _( - "You don't have permission to move this message due to missing access to its stream" - ) - ) - - new_stream = access_stream_by_id(user_profile, stream_id, require_active=True)[0] - check_stream_access_based_on_stream_post_policy(user_profile, new_stream) - - number_changed = do_update_message( - user_profile, - message, - new_stream, - topic_name, - propagate_mode, - send_notification_to_old_thread, - send_notification_to_new_thread, - content, - rendering_result, - prior_mention_user_ids, - mention_data, - ) - - if links_for_embed: - event_data = { - "message_id": message.id, - "message_content": message.content, - # The choice of `user_profile.realm_id` rather than - # `sender.realm_id` must match the decision made in the - # `render_incoming_message` call earlier in this function. - "message_realm_id": user_profile.realm_id, - "urls": list(links_for_embed), - } - queue_json_publish("embed_links", event_data) - - return number_changed - - @transaction.atomic(durable=True) def do_change_realm_org_type( realm: Realm, @@ -950,815 +680,6 @@ def do_create_realm( return realm -class MessageUpdateUserInfoResult(TypedDict): - message_user_ids: Set[int] - mention_user_ids: Set[int] - - -def maybe_send_resolve_topic_notifications( - *, - user_profile: UserProfile, - stream: Stream, - old_topic: str, - new_topic: str, - changed_messages: List[Message], -) -> None: - # Note that topics will have already been stripped in check_update_message. - # - # This logic is designed to treat removing a weird "✔ ✔✔ " - # prefix as unresolving the topic. - if old_topic.lstrip(RESOLVED_TOPIC_PREFIX) != new_topic.lstrip(RESOLVED_TOPIC_PREFIX): - return - - topic_resolved: bool = new_topic.startswith(RESOLVED_TOPIC_PREFIX) and not old_topic.startswith( - RESOLVED_TOPIC_PREFIX - ) - topic_unresolved: bool = old_topic.startswith( - RESOLVED_TOPIC_PREFIX - ) and not new_topic.startswith(RESOLVED_TOPIC_PREFIX) - - if not topic_resolved and not topic_unresolved: - # If there's some other weird topic that does not toggle the - # state of "topic starts with RESOLVED_TOPIC_PREFIX", we do - # nothing. Any other logic could result in cases where we send - # these notifications in a non-alternating fashion. - # - # Note that it is still possible for an individual topic to - # have multiple "This topic was marked as resolved" - # notifications in a row: one can send new messages to the - # pre-resolve topic and then resolve the topic created that - # way to get multiple in the resolved topic. And then an - # administrator can the messages in between. We consider this - # to be a fundamental risk of irresponsible message deletion, - # not a bug with the "resolve topics" feature. - return - - # Compute the users who either sent or reacted to messages that - # were moved via the "resolve topic' action. Only those users - # should be eligible for this message being managed as unread. - affected_participant_ids = {message.sender_id for message in changed_messages} | set( - Reaction.objects.filter(message__in=changed_messages).values_list( - "user_profile_id", flat=True - ) - ) - sender = get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id) - user_mention = silent_mention_syntax_for_user(user_profile) - with override_language(stream.realm.default_language): - if topic_resolved: - notification_string = _("{user} has marked this topic as resolved.") - elif topic_unresolved: - notification_string = _("{user} has marked this topic as unresolved.") - - internal_send_stream_message( - sender, - stream, - new_topic, - notification_string.format( - user=user_mention, - ), - limit_unread_user_ids=affected_participant_ids, - ) - - -def send_message_moved_breadcrumbs( - user_profile: UserProfile, - old_stream: Stream, - old_topic: str, - old_thread_notification_string: Optional[str], - new_stream: Stream, - new_topic: Optional[str], - new_thread_notification_string: Optional[str], - changed_messages_count: int, -) -> None: - # Since moving content between streams is highly disruptive, - # it's worth adding a couple tombstone messages showing what - # happened. - sender = get_system_bot(settings.NOTIFICATION_BOT, old_stream.realm_id) - - if new_topic is None: - new_topic = old_topic - - user_mention = silent_mention_syntax_for_user(user_profile) - old_topic_link = f"#**{old_stream.name}>{old_topic}**" - new_topic_link = f"#**{new_stream.name}>{new_topic}**" - - if new_thread_notification_string is not None: - with override_language(new_stream.realm.default_language): - internal_send_stream_message( - sender, - new_stream, - new_topic, - new_thread_notification_string.format( - old_location=old_topic_link, - user=user_mention, - changed_messages_count=changed_messages_count, - ), - ) - - if old_thread_notification_string is not None: - with override_language(old_stream.realm.default_language): - # Send a notification to the old stream that the topic was moved. - internal_send_stream_message( - sender, - old_stream, - old_topic, - old_thread_notification_string.format( - user=user_mention, - new_location=new_topic_link, - changed_messages_count=changed_messages_count, - ), - ) - - -def get_user_info_for_message_updates(message_id: int) -> MessageUpdateUserInfoResult: - - # We exclude UserMessage.flags.historical rows since those - # users did not receive the message originally, and thus - # probably are not relevant for reprocessed alert_words, - # mentions and similar rendering features. This may be a - # decision we change in the future. - query = UserMessage.objects.filter( - message=message_id, - flags=~UserMessage.flags.historical, - ).values("user_profile_id", "flags") - rows = list(query) - - message_user_ids = {row["user_profile_id"] for row in rows} - - mask = UserMessage.flags.mentioned | UserMessage.flags.wildcard_mentioned - - mention_user_ids = {row["user_profile_id"] for row in rows if int(row["flags"]) & mask} - - return dict( - message_user_ids=message_user_ids, - mention_user_ids=mention_user_ids, - ) - - -def update_user_message_flags( - rendering_result: MessageRenderingResult, ums: Iterable[UserMessage] -) -> None: - wildcard = rendering_result.mentions_wildcard - mentioned_ids = rendering_result.mentions_user_ids - ids_with_alert_words = rendering_result.user_ids_with_alert_words - changed_ums: Set[UserMessage] = set() - - def update_flag(um: UserMessage, should_set: bool, flag: int) -> None: - if should_set: - if not (um.flags & flag): - um.flags |= flag - changed_ums.add(um) - else: - if um.flags & flag: - um.flags &= ~flag - changed_ums.add(um) - - for um in ums: - has_alert_word = um.user_profile_id in ids_with_alert_words - update_flag(um, has_alert_word, UserMessage.flags.has_alert_word) - - mentioned = um.user_profile_id in mentioned_ids - update_flag(um, mentioned, UserMessage.flags.mentioned) - - update_flag(um, wildcard, UserMessage.flags.wildcard_mentioned) - - for um in changed_ums: - um.save(update_fields=["flags"]) - - -def do_update_embedded_data( - user_profile: UserProfile, - message: Message, - content: Optional[str], - rendering_result: MessageRenderingResult, -) -> None: - timestamp = timezone_now() - event: Dict[str, Any] = { - "type": "update_message", - "user_id": None, - "edit_timestamp": datetime_to_timestamp(timestamp), - "message_id": message.id, - "rendering_only": True, - } - changed_messages = [message] - rendered_content: Optional[str] = None - - ums = UserMessage.objects.filter(message=message.id) - - if content is not None: - update_user_message_flags(rendering_result, ums) - rendered_content = rendering_result.rendered_content - message.rendered_content = rendered_content - message.rendered_content_version = markdown_version - event["content"] = content - event["rendered_content"] = rendered_content - - message.save(update_fields=["content", "rendered_content"]) - - event["message_ids"] = update_to_dict_cache(changed_messages) - - def user_info(um: UserMessage) -> Dict[str, Any]: - return { - "id": um.user_profile_id, - "flags": um.flags_list(), - } - - send_event(user_profile.realm, event, list(map(user_info, ums))) - - -class DeleteMessagesEvent(TypedDict, total=False): - type: str - message_ids: List[int] - message_type: str - topic: str - stream_id: int - - -# We use transaction.atomic to support select_for_update in the attachment codepath. -@transaction.atomic(savepoint=False) -def do_update_message( - user_profile: UserProfile, - target_message: Message, - new_stream: Optional[Stream], - topic_name: Optional[str], - propagate_mode: str, - send_notification_to_old_thread: bool, - send_notification_to_new_thread: bool, - content: Optional[str], - rendering_result: Optional[MessageRenderingResult], - prior_mention_user_ids: Set[int], - mention_data: Optional[MentionData] = None, -) -> int: - """ - The main function for message editing. A message edit event can - modify: - * the message's content (in which case the caller will have - set both content and rendered_content), - * the topic, in which case the caller will have set topic_name - * or both message's content and the topic - * or stream and/or topic, in which case the caller will have set - new_stream and/or topic_name. - - With topic edits, propagate_mode determines whether other message - also have their topics edited. - """ - timestamp = timezone_now() - target_message.last_edit_time = timestamp - - event: Dict[str, Any] = { - "type": "update_message", - "user_id": user_profile.id, - "edit_timestamp": datetime_to_timestamp(timestamp), - "message_id": target_message.id, - "rendering_only": False, - } - - edit_history_event: EditHistoryEvent = { - "user_id": user_profile.id, - "timestamp": event["edit_timestamp"], - } - - changed_messages = [target_message] - - realm = user_profile.realm - - stream_being_edited = None - if target_message.is_stream_message(): - stream_id = target_message.recipient.type_id - stream_being_edited = get_stream_by_id_in_realm(stream_id, realm) - event["stream_name"] = stream_being_edited.name - event["stream_id"] = stream_being_edited.id - - ums = UserMessage.objects.filter(message=target_message.id) - - if content is not None: - assert rendering_result is not None - - # mention_data is required if there's a content edit. - assert mention_data is not None - - # add data from group mentions to mentions_user_ids. - for group_id in rendering_result.mentions_user_group_ids: - members = mention_data.get_group_members(group_id) - rendering_result.mentions_user_ids.update(members) - - update_user_message_flags(rendering_result, ums) - - # One could imagine checking realm.allow_edit_history here and - # modifying the events based on that setting, but doing so - # doesn't really make sense. We need to send the edit event - # to clients regardless, and a client already had access to - # the original/pre-edit content of the message anyway. That - # setting must be enforced on the client side, and making a - # change here simply complicates the logic for clients parsing - # edit history events. - event["orig_content"] = target_message.content - event["orig_rendered_content"] = target_message.rendered_content - edit_history_event["prev_content"] = target_message.content - edit_history_event["prev_rendered_content"] = target_message.rendered_content - edit_history_event[ - "prev_rendered_content_version" - ] = target_message.rendered_content_version - target_message.content = content - target_message.rendered_content = rendering_result.rendered_content - target_message.rendered_content_version = markdown_version - event["content"] = content - event["rendered_content"] = rendering_result.rendered_content - event["prev_rendered_content_version"] = target_message.rendered_content_version - event["is_me_message"] = Message.is_status_message( - content, rendering_result.rendered_content - ) - - # target_message.has_image and target_message.has_link will have been - # already updated by Markdown rendering in the caller. - target_message.has_attachment = check_attachment_reference_change( - target_message, rendering_result - ) - - if target_message.is_stream_message(): - if topic_name is not None: - new_topic_name = topic_name - else: - new_topic_name = target_message.topic_name() - - stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget( - stream_id=stream_id, - topic_name=new_topic_name, - ) - else: - stream_topic = None - - info = get_recipient_info( - realm_id=realm.id, - recipient=target_message.recipient, - sender_id=target_message.sender_id, - stream_topic=stream_topic, - possible_wildcard_mention=mention_data.message_has_wildcards(), - ) - - event["online_push_user_ids"] = list(info["online_push_user_ids"]) - event["pm_mention_push_disabled_user_ids"] = list(info["pm_mention_push_disabled_user_ids"]) - event["pm_mention_email_disabled_user_ids"] = list( - info["pm_mention_email_disabled_user_ids"] - ) - event["stream_push_user_ids"] = list(info["stream_push_user_ids"]) - event["stream_email_user_ids"] = list(info["stream_email_user_ids"]) - event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"]) - event["prior_mention_user_ids"] = list(prior_mention_user_ids) - event["presence_idle_user_ids"] = filter_presence_idle_user_ids(info["active_user_ids"]) - event["all_bot_user_ids"] = list(info["all_bot_user_ids"]) - if rendering_result.mentions_wildcard: - event["wildcard_mention_user_ids"] = list(info["wildcard_mention_user_ids"]) - else: - event["wildcard_mention_user_ids"] = [] - - do_update_mobile_push_notification( - target_message, - prior_mention_user_ids, - rendering_result.mentions_user_ids, - info["stream_push_user_ids"], - ) - - if topic_name is not None or new_stream is not None: - orig_topic_name = target_message.topic_name() - event["propagate_mode"] = propagate_mode - - if new_stream is not None: - assert content is None - assert target_message.is_stream_message() - assert stream_being_edited is not None - - edit_history_event["prev_stream"] = stream_being_edited.id - edit_history_event["stream"] = new_stream.id - event[ORIG_TOPIC] = orig_topic_name - target_message.recipient_id = new_stream.recipient_id - - event["new_stream_id"] = new_stream.id - event["propagate_mode"] = propagate_mode - - # When messages are moved from one stream to another, some - # users may lose access to those messages, including guest - # users and users not subscribed to the new stream (if it is a - # private stream). For those users, their experience is as - # though the messages were deleted, and we should send a - # delete_message event to them instead. - - subs_to_old_stream = get_active_subscriptions_for_stream_id( - stream_id, include_deactivated_users=True - ).select_related("user_profile") - subs_to_new_stream = list( - get_active_subscriptions_for_stream_id( - new_stream.id, include_deactivated_users=True - ).select_related("user_profile") - ) - - old_stream_sub_ids = [user.user_profile_id for user in subs_to_old_stream] - new_stream_sub_ids = [user.user_profile_id for user in subs_to_new_stream] - - # Get users who aren't subscribed to the new_stream. - subs_losing_usermessages = [ - sub for sub in subs_to_old_stream if sub.user_profile_id not in new_stream_sub_ids - ] - # Users who can longer access the message without some action - # from administrators. - subs_losing_access = [ - sub - for sub in subs_losing_usermessages - if sub.user_profile.is_guest or not new_stream.is_public() - ] - ums = ums.exclude( - user_profile_id__in=[sub.user_profile_id for sub in subs_losing_usermessages] - ) - - subs_gaining_usermessages = [] - if not new_stream.is_history_public_to_subscribers(): - # For private streams, with history not public to subscribers, - # We find out users who are not present in the msgs' old stream - # and create new UserMessage for these users so that they can - # access this message. - subs_gaining_usermessages += [ - user_id for user_id in new_stream_sub_ids if user_id not in old_stream_sub_ids - ] - - if topic_name is not None: - topic_name = truncate_topic(topic_name) - target_message.set_topic_name(topic_name) - - # These fields have legacy field names. - event[ORIG_TOPIC] = orig_topic_name - event[TOPIC_NAME] = topic_name - event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id, topic_name) - edit_history_event["prev_topic"] = orig_topic_name - edit_history_event["topic"] = topic_name - - update_edit_history(target_message, timestamp, edit_history_event) - - delete_event_notify_user_ids: List[int] = [] - if propagate_mode in ["change_later", "change_all"]: - assert topic_name is not None or new_stream is not None - assert stream_being_edited is not None - - # Other messages should only get topic/stream fields in their edit history. - topic_only_edit_history_event: EditHistoryEvent = { - "user_id": edit_history_event["user_id"], - "timestamp": edit_history_event["timestamp"], - } - if topic_name is not None: - topic_only_edit_history_event["prev_topic"] = edit_history_event["prev_topic"] - topic_only_edit_history_event["topic"] = edit_history_event["topic"] - if new_stream is not None: - topic_only_edit_history_event["prev_stream"] = edit_history_event["prev_stream"] - topic_only_edit_history_event["stream"] = edit_history_event["stream"] - - messages_list = update_messages_for_topic_edit( - acting_user=user_profile, - edited_message=target_message, - propagate_mode=propagate_mode, - orig_topic_name=orig_topic_name, - topic_name=topic_name, - new_stream=new_stream, - old_stream=stream_being_edited, - edit_history_event=topic_only_edit_history_event, - last_edit_time=timestamp, - ) - changed_messages += messages_list - - if new_stream is not None: - assert stream_being_edited is not None - changed_message_ids = [msg.id for msg in changed_messages] - - if subs_gaining_usermessages: - ums_to_create = [] - for message_id in changed_message_ids: - for user_profile_id in subs_gaining_usermessages: - # The fact that the user didn't have a UserMessage originally means we can infer that the user - # was not mentioned in the original message (even if mention syntax was present, it would not - # take effect for a user who was not subscribed). If we were editing the message's content, we - # would rerender the message and then use the new stream's data to determine whether this is - # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this - # mention syntax an actual mention" decision made during the original rendering for implementation - # simplicity. As a result, the only flag to consider applying here is read. - um = UserMessageLite( - user_profile_id=user_profile_id, - message_id=message_id, - flags=UserMessage.flags.read, - ) - ums_to_create.append(um) - bulk_insert_ums(ums_to_create) - - # Delete UserMessage objects for users who will no - # longer have access to these messages. Note: This could be - # very expensive, since it's N guest users x M messages. - UserMessage.objects.filter( - user_profile_id__in=[sub.user_profile_id for sub in subs_losing_usermessages], - message_id__in=changed_message_ids, - ).delete() - - delete_event: DeleteMessagesEvent = { - "type": "delete_message", - "message_ids": changed_message_ids, - "message_type": "stream", - "stream_id": stream_being_edited.id, - "topic": orig_topic_name, - } - delete_event_notify_user_ids = [sub.user_profile_id for sub in subs_losing_access] - send_event(user_profile.realm, delete_event, delete_event_notify_user_ids) - - # Reset the Attachment.is_*_public caches for all messages - # moved to another stream with different access permissions. - if new_stream.invite_only != stream_being_edited.invite_only: - Attachment.objects.filter(messages__in=changed_message_ids).update( - is_realm_public=None, - ) - ArchivedAttachment.objects.filter(messages__in=changed_message_ids).update( - is_realm_public=None, - ) - - if new_stream.is_web_public != stream_being_edited.is_web_public: - Attachment.objects.filter(messages__in=changed_message_ids).update( - is_web_public=None, - ) - ArchivedAttachment.objects.filter(messages__in=changed_message_ids).update( - is_web_public=None, - ) - - # This does message.save(update_fields=[...]) - save_message_for_edit_use_case(message=target_message) - - realm_id: Optional[int] = None - if stream_being_edited is not None: - realm_id = stream_being_edited.realm_id - - event["message_ids"] = update_to_dict_cache(changed_messages, realm_id) - - def user_info(um: UserMessage) -> Dict[str, Any]: - return { - "id": um.user_profile_id, - "flags": um.flags_list(), - } - - # The following blocks arranges that users who are subscribed to a - # stream and can see history from before they subscribed get - # live-update when old messages are edited (e.g. if the user does - # a topic edit themself). - # - # We still don't send an update event to users who are not - # subscribed to this stream and don't have a UserMessage row. This - # means if a non-subscriber is viewing the narrow, they won't get - # a real-time updates. This is a balance between sending - # message-edit notifications for every public stream to every user - # in the organization (too expansive, and also not what we do for - # newly sent messages anyway) and having magical live-updates - # where possible. - users_to_be_notified = list(map(user_info, ums)) - if stream_being_edited is not None: - if stream_being_edited.is_history_public_to_subscribers: - subscriptions = get_active_subscriptions_for_stream_id( - stream_id, include_deactivated_users=False - ) - # We exclude long-term idle users, since they by - # definition have no active clients. - subscriptions = subscriptions.exclude(user_profile__long_term_idle=True) - # Remove duplicates by excluding the id of users already - # in users_to_be_notified list. This is the case where a - # user both has a UserMessage row and is a current - # Subscriber - subscriptions = subscriptions.exclude( - user_profile_id__in=[um.user_profile_id for um in ums] - ) - - if new_stream is not None: - assert delete_event_notify_user_ids is not None - subscriptions = subscriptions.exclude( - user_profile_id__in=delete_event_notify_user_ids - ) - - # All users that are subscribed to the stream must be - # notified when a message is edited - subscriber_ids = set(subscriptions.values_list("user_profile_id", flat=True)) - - if new_stream is not None: - # TODO: Guest users don't see the new moved topic - # unless breadcrumb message for new stream is - # enabled. Excluding these users from receiving this - # event helps us avoid a error traceback for our - # clients. We should figure out a way to inform the - # guest users of this new topic if sending a 'message' - # event for these messages is not an option. - # - # Don't send this event to guest subs who are not - # subscribers of the old stream but are subscribed to - # the new stream; clients will be confused. - old_stream_unsubbed_guests = [ - sub - for sub in subs_to_new_stream - if sub.user_profile.is_guest and sub.user_profile_id not in subscriber_ids - ] - subscriptions = subscriptions.exclude( - user_profile_id__in=[sub.user_profile_id for sub in old_stream_unsubbed_guests] - ) - subscriber_ids = set(subscriptions.values_list("user_profile_id", flat=True)) - - users_to_be_notified += list(map(subscriber_info, sorted(list(subscriber_ids)))) - - # UserTopic updates and the content of notifications depend on - # whether we've moved the entire topic, or just part of it. We - # make that determination here. - moved_all_visible_messages = False - if topic_name is not None or new_stream is not None: - assert stream_being_edited is not None - - if propagate_mode == "change_all": - moved_all_visible_messages = True - else: - # With other propagate modes, if the user in fact moved - # all messages in the stream, we want to explain it was a - # full-topic move. - # - # For security model reasons, we don't want to allow a - # user to take any action that would leak information - # about older messages they cannot access (E.g. the only - # remaining messages are in a stream without shared - # history). The bulk_access_messages call below addresses - # that concern. - # - # bulk_access_messages is inefficient for this task, since - # we just want to do the exists() version of this - # query. But it's nice to reuse code, and this bulk - # operation is likely cheaper than a `GET /messages` - # unless the topic has thousands of messages of history. - unmoved_messages = messages_for_topic( - stream_being_edited.recipient_id, - orig_topic_name, - ) - visible_unmoved_messages = bulk_access_messages( - user_profile, unmoved_messages, stream=stream_being_edited - ) - moved_all_visible_messages = len(visible_unmoved_messages) == 0 - - # Migrate muted topic configuration in the following circumstances: - # - # * If propagate_mode is change_all, do so unconditionally. - # - # * If propagate_mode is change_later or change_one, do so when - # the acting user has moved the entire topic (as visible to them). - # - # This rule corresponds to checking moved_all_visible_messages. - # - # We may want more complex behavior in cases where one appears to - # be merging topics (E.g. there are existing messages in the - # target topic). - if moved_all_visible_messages: - assert stream_being_edited is not None - assert topic_name is not None or new_stream is not None - - for muting_user in get_users_muting_topic(stream_being_edited.id, orig_topic_name): - # TODO: Ideally, this would be a bulk update operation, - # because we are doing database operations in a loop here. - # - # This loop is only acceptable in production because it is - # rare for more than a few users to have muted an - # individual topic that is being moved; as of this - # writing, no individual topic in Zulip Cloud had been - # muted by more than 100 users. - - if new_stream is not None and muting_user.id in delete_event_notify_user_ids: - # If the messages are being moved to a stream the user - # cannot access, then we treat this as the - # messages/topic being deleted for this user. This is - # important for security reasons; we don't want to - # give users a UserTopic row in a stream they cannot - # access. Unmute the topic for such users. - do_unmute_topic(muting_user, stream_being_edited, orig_topic_name) - else: - # Otherwise, we move the muted topic record for the user. - # We call remove_topic_mute rather than do_unmute_topic to - # avoid sending two events with new muted topics in - # immediate succession; this is correct only because - # muted_topics events always send the full set of topics. - remove_topic_mute(muting_user, stream_being_edited.id, orig_topic_name) - do_mute_topic( - muting_user, - new_stream if new_stream is not None else stream_being_edited, - topic_name if topic_name is not None else orig_topic_name, - ignore_duplicate=True, - ) - - send_event(user_profile.realm, event, users_to_be_notified) - - if len(changed_messages) > 0 and new_stream is not None and stream_being_edited is not None: - # Notify users that the topic was moved. - changed_messages_count = len(changed_messages) - - old_thread_notification_string = None - if send_notification_to_old_thread: - if moved_all_visible_messages: - old_thread_notification_string = gettext_lazy( - "This topic was moved to {new_location} by {user}." - ) - elif changed_messages_count == 1: - old_thread_notification_string = gettext_lazy( - "A message was moved from this topic to {new_location} by {user}." - ) - else: - old_thread_notification_string = gettext_lazy( - "{changed_messages_count} messages were moved from this topic to {new_location} by {user}." - ) - - new_thread_notification_string = None - if send_notification_to_new_thread: - if moved_all_visible_messages: - new_thread_notification_string = gettext_lazy( - "This topic was moved here from {old_location} by {user}." - ) - elif changed_messages_count == 1: - new_thread_notification_string = gettext_lazy( - "A message was moved here from {old_location} by {user}." - ) - else: - new_thread_notification_string = gettext_lazy( - "{changed_messages_count} messages were moved here from {old_location} by {user}." - ) - - send_message_moved_breadcrumbs( - user_profile, - stream_being_edited, - orig_topic_name, - old_thread_notification_string, - new_stream, - topic_name, - new_thread_notification_string, - changed_messages_count, - ) - - if ( - topic_name is not None - and new_stream is None - and content is None - and len(changed_messages) > 0 - ): - assert stream_being_edited is not None - maybe_send_resolve_topic_notifications( - user_profile=user_profile, - stream=stream_being_edited, - old_topic=orig_topic_name, - new_topic=topic_name, - changed_messages=changed_messages, - ) - - return len(changed_messages) - - -def do_delete_messages(realm: Realm, messages: Iterable[Message]) -> None: - # messages in delete_message event belong to the same topic - # or is a single private message, as any other behaviour is not possible with - # the current callers to this method. - messages = list(messages) - message_ids = [message.id for message in messages] - if not message_ids: - return - - event: DeleteMessagesEvent = { - "type": "delete_message", - "message_ids": message_ids, - } - - sample_message = messages[0] - message_type = "stream" - users_to_notify = [] - if not sample_message.is_stream_message(): - assert len(messages) == 1 - message_type = "private" - ums = UserMessage.objects.filter(message_id__in=message_ids) - users_to_notify = [um.user_profile_id for um in ums] - archiving_chunk_size = retention.MESSAGE_BATCH_SIZE - - if message_type == "stream": - stream_id = sample_message.recipient.type_id - event["stream_id"] = stream_id - event["topic"] = sample_message.topic_name() - subscriptions = get_active_subscriptions_for_stream_id( - stream_id, include_deactivated_users=False - ) - # We exclude long-term idle users, since they by definition have no active clients. - subscriptions = subscriptions.exclude(user_profile__long_term_idle=True) - users_to_notify = list(subscriptions.values_list("user_profile_id", flat=True)) - archiving_chunk_size = retention.STREAM_MESSAGE_BATCH_SIZE - - move_messages_to_archive(message_ids, realm=realm, chunk_size=archiving_chunk_size) - - event["message_type"] = message_type - transaction.on_commit(lambda: send_event(realm, event, users_to_notify)) - - -def do_delete_messages_by_sender(user: UserProfile) -> None: - message_ids = list( - Message.objects.filter(sender=user).values_list("id", flat=True).order_by("id") - ) - if message_ids: - move_messages_to_archive(message_ids, chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE) - - def email_not_system_bot(email: str) -> None: if is_cross_realm_bot_email(email): msg = email_reserved_for_system_bots_error(email) diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 23e62569e1..d2221cdb2e 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -45,6 +45,11 @@ from zerver.actions.invites import ( do_revoke_multi_use_invite, do_revoke_user_invite, ) +from zerver.actions.message_edit import ( + do_delete_messages, + do_update_embedded_data, + do_update_message, +) from zerver.actions.message_flags import do_update_message_flags from zerver.actions.muted_users import do_mute_user, do_unmute_user from zerver.actions.presence import do_update_user_presence, do_update_user_status @@ -99,7 +104,6 @@ from zerver.lib.actions import ( do_change_realm_domain, do_change_realm_plan_type, do_deactivate_realm, - do_delete_messages, do_remove_realm_domain, do_set_realm_authentication_methods, do_set_realm_message_editing, @@ -107,8 +111,6 @@ from zerver.lib.actions import ( do_set_realm_property, do_set_realm_signup_notifications_stream, do_set_realm_user_default_setting, - do_update_embedded_data, - do_update_message, ) from zerver.lib.drafts import do_create_drafts, do_delete_draft, do_edit_draft from zerver.lib.event_schema import ( diff --git a/zerver/tests/test_example.py b/zerver/tests/test_example.py index 8f028b4729..1474bbe190 100644 --- a/zerver/tests/test_example.py +++ b/zerver/tests/test_example.py @@ -494,7 +494,7 @@ class TestMocking(ZulipTestCase): # that is beyond the limit. # # Notice how mock.patch() is used here to do exactly the above mentioned. - # mock.patch() here makes any calls to `timezone_now` in `zerver.lib.actions` + # mock.patch() here makes any calls to `timezone_now` in `zerver.actions.message_edit` # to return the value passed to `return_value` in the its context. # You can also use mock.patch() as a decorator depending on the # requirements. Read more at the documentation link provided above. @@ -504,7 +504,7 @@ class TestMocking(ZulipTestCase): ) # There's a buffer time applied to the limit, hence the extra 100s. with mock.patch( - "zerver.lib.actions.timezone_now", + "zerver.actions.message_edit.timezone_now", return_value=time_beyond_edit_limit, ): result = self.client_patch( diff --git a/zerver/tests/test_link_embed.py b/zerver/tests/test_link_embed.py index be5f7f2216..e188363ec5 100644 --- a/zerver/tests/test_link_embed.py +++ b/zerver/tests/test_link_embed.py @@ -389,7 +389,7 @@ class PreviewTestCase(ZulipTestCase): url = "http://test.org/" self.create_mock_response(url) - with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched: + with mock_queue_publish("zerver.actions.message_edit.queue_json_publish") as patched: result = self.client_patch( "/json/messages/" + str(msg_id), { @@ -509,7 +509,7 @@ class PreviewTestCase(ZulipTestCase): ) with mock_queue_publish( - "zerver.lib.actions.queue_json_publish", wraps=wrapped_queue_json_publish + "zerver.actions.message_edit.queue_json_publish", wraps=wrapped_queue_json_publish ): result = self.client_patch( "/json/messages/" + str(msg_id), diff --git a/zerver/tests/test_message_edit.py b/zerver/tests/test_message_edit.py index adc5963c9d..9a785ff345 100644 --- a/zerver/tests/test_message_edit.py +++ b/zerver/tests/test_message_edit.py @@ -8,18 +8,16 @@ from django.db import IntegrityError from django.http import HttpResponse from django.utils.timezone import now as timezone_now +from zerver.actions.message_edit import ( + check_update_message, + do_delete_messages, + do_update_message, + get_user_info_for_message_updates, +) from zerver.actions.reactions import do_add_reaction from zerver.actions.streams import do_change_stream_post_policy, do_deactivate_stream from zerver.actions.users import do_change_user_role -from zerver.lib.actions import ( - check_update_message, - do_change_realm_plan_type, - do_delete_messages, - do_set_realm_property, - do_update_message, - get_topic_messages, - get_user_info_for_message_updates, -) +from zerver.lib.actions import do_change_realm_plan_type, do_set_realm_property, get_topic_messages from zerver.lib.message import MessageDict, has_message_access, messages_for_ids from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import cache_tries_captured, queries_captured @@ -1163,7 +1161,7 @@ class EditMessageTest(EditMessageTestCase): message.save() do_edit_message_assert_success(id_, "D", "cordelia") - @mock.patch("zerver.lib.actions.send_event") + @mock.patch("zerver.actions.message_edit.send_event") def test_edit_topic_public_history_stream(self, mock_send_event: mock.MagicMock) -> None: stream_name = "Macbeth" hamlet = self.example_user("hamlet") @@ -1245,7 +1243,7 @@ class EditMessageTest(EditMessageTestCase): users_to_be_notified = list(map(notify, [hamlet.id])) do_update_message_topic_success(hamlet, message, "Change again", users_to_be_notified) - @mock.patch("zerver.lib.actions.send_event") + @mock.patch("zerver.actions.message_edit.send_event") def test_edit_muted_topic(self, mock_send_event: mock.MagicMock) -> None: stream_name = "Stream 123" stream = self.make_stream(stream_name) @@ -1487,7 +1485,7 @@ class EditMessageTest(EditMessageTestCase): self.assertFalse(topic_is_muted(cordelia, new_public_stream.id, "final topic name")) self.assertFalse(topic_is_muted(aaron, new_public_stream.id, "final topic name")) - @mock.patch("zerver.lib.actions.send_event") + @mock.patch("zerver.actions.message_edit.send_event") def test_wildcard_mention(self, mock_send_event: mock.MagicMock) -> None: stream_name = "Macbeth" hamlet = self.example_user("hamlet") @@ -2889,7 +2887,7 @@ class DeleteMessageTest(ZulipTestCase): message = self.get_last_message() with self.tornado_redirected_to_list([], expected_num_events=1): - with mock.patch("zerver.lib.actions.send_event") as m: + with mock.patch("zerver.actions.message_edit.send_event") as m: m.side_effect = AssertionError( "Events should be sent only after the transaction commits." ) diff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.py index 8269d96fc3..750a285618 100644 --- a/zerver/tests/test_message_fetch.py +++ b/zerver/tests/test_message_fetch.py @@ -13,9 +13,10 @@ from sqlalchemy.types import Integer from analytics.lib.counts import COUNT_STATS from analytics.models import RealmCount +from zerver.actions.message_edit import do_update_message from zerver.actions.uploads import do_claim_attachments from zerver.actions.users import do_deactivate_user -from zerver.lib.actions import do_set_realm_property, do_update_message +from zerver.lib.actions import do_set_realm_property from zerver.lib.avatar import avatar_url from zerver.lib.exceptions import JsonableError from zerver.lib.mention import MentionBackend, MentionData diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 2782c8b76c..2db795f67f 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -23,9 +23,9 @@ from requests.models import PreparedRequest from analytics.lib.counts import CountStat, LoggingCountStat from analytics.models import InstallationCount, RealmCount +from zerver.actions.message_edit import do_delete_messages from zerver.actions.message_flags import do_mark_stream_messages_as_read, do_update_message_flags from zerver.actions.user_settings import do_regenerate_api_key -from zerver.lib.actions import do_delete_messages from zerver.lib.avatar import absolute_avatar_url from zerver.lib.exceptions import JsonableError from zerver.lib.push_notifications import ( diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 71a03bf90c..5f6abf49cd 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -5,9 +5,10 @@ from unittest import mock from django.conf import settings from django.utils.timezone import now as timezone_now +from zerver.actions.message_edit import do_delete_messages from zerver.actions.message_send import internal_send_private_message from zerver.actions.submessage import do_add_submessage -from zerver.lib.actions import do_create_realm, do_delete_messages, do_set_realm_property +from zerver.lib.actions import do_create_realm, do_set_realm_property from zerver.lib.retention import ( archive_messages, clean_archived_data, diff --git a/zerver/views/message_edit.py b/zerver/views/message_edit.py index 06da18e679..8df61b7da4 100644 --- a/zerver/views/message_edit.py +++ b/zerver/views/message_edit.py @@ -8,8 +8,8 @@ from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import gettext as _ +from zerver.actions.message_edit import check_update_message, do_delete_messages from zerver.context_processors import get_valid_realm_from_request -from zerver.lib.actions import check_update_message, do_delete_messages from zerver.lib.exceptions import JsonableError from zerver.lib.html_diff import highlight_html_differences from zerver.lib.message import access_message, access_web_public_message, messages_for_ids diff --git a/zerver/views/streams.py b/zerver/views/streams.py index 75b64e8db9..7cb598460d 100644 --- a/zerver/views/streams.py +++ b/zerver/views/streams.py @@ -21,6 +21,7 @@ from zerver.actions.default_streams import ( do_remove_streams_from_default_stream_group, get_default_streams_for_realm, ) +from zerver.actions.message_edit import do_delete_messages from zerver.actions.message_send import ( do_send_messages, internal_prep_private_message, @@ -45,7 +46,6 @@ from zerver.decorator import ( require_post, require_realm_admin, ) -from zerver.lib.actions import do_delete_messages from zerver.lib.exceptions import ( ErrorCode, JsonableError, diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index ae1f0ac0f1..b8e4018138 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -47,13 +47,13 @@ from sentry_sdk import add_breadcrumb, configure_scope from zulip_bots.lib import extract_query_without_mention from zerver.actions.invites import do_send_confirmation_email +from zerver.actions.message_edit import do_update_embedded_data from zerver.actions.message_flags import do_mark_stream_messages_as_read from zerver.actions.message_send import internal_send_private_message, render_incoming_message from zerver.actions.presence import do_update_user_presence from zerver.actions.realm_export import notify_realm_export from zerver.actions.user_activity import do_update_user_activity, do_update_user_activity_interval from zerver.context_processors import common_context -from zerver.lib.actions import do_update_embedded_data from zerver.lib.bot_lib import EmbeddedBotHandler, EmbeddedBotQuitException, get_bot_handler from zerver.lib.context_managers import lockfile from zerver.lib.db import reset_queries @@ -865,7 +865,7 @@ class FetchLinksEmbedData(QueueProcessingWorker): message = Message.objects.get(id=event["message_id"]) # If the message changed, we will run this task after updating the message - # in zerver.lib.actions.check_update_message + # in zerver.actions.message_edit.check_update_message if message.content != event["message_content"]: return if message.content is not None: