actions: Handle cross-realm messages in do_scrub_realm.

This is necessary to properly scrub all data from the realm.

zerver/actions/realm_settings.py

@@ -14,6 +14,7 @@ from zerver.actions.message_delete import do_delete_messages_by_sender
 from zerver.actions.user_groups import update_users_in_full_members_system_group
 from zerver.actions.user_settings import do_delete_avatar_image
 from zerver.lib.message import parse_message_time_limit_setting, update_first_visible_message_id
+from zerver.lib.retention import move_messages_to_archive
 from zerver.lib.send_email import FromAddress, send_email_to_admins
 from zerver.lib.sessions import delete_user_sessions
 from zerver.lib.upload import delete_message_attachments
@@ -21,15 +22,19 @@ from zerver.lib.user_counts import realm_user_count_by_role
 from zerver.models import (
     ArchivedAttachment,
     Attachment,
+    Message,
     Realm,
     RealmAuditLog,
     RealmAuthenticationMethod,
     RealmReactivationStatus,
     RealmUserDefault,
+    Recipient,
     ScheduledEmail,
     Stream,
+    Subscription,
     UserProfile,
     active_user_ids,
+    get_realm,
 )
 from zerver.tornado.django_api import send_event
 
@@ -374,6 +379,32 @@ def do_scrub_realm(realm: Realm, *, acting_user: Optional[UserProfile]) -> None:
         user.delivery_email = scrubbed_email
         user.save(update_fields=["full_name", "email", "delivery_email"])
 
+    internal_realm = get_realm(settings.SYSTEM_BOT_REALM)
+    # We could more simply obtain the Message list by just doing
+    # Message.objects.filter(sender__realm=internal_realm, realm=realm), but it's
+    # more secure against bugs that may cause Message.realm to be incorrect for some
+    # cross-realm messages to also determine the actual Recipients - to prevent
+    # deletion of excessive messages.
+    all_recipient_ids_in_realm = (
+        list(Stream.objects.filter(realm=realm).values_list("recipient_id", flat=True))
+        + list(UserProfile.objects.filter(realm=realm).values_list("recipient_id", flat=True))
+        + list(
+            Subscription.objects.filter(
+                recipient__type=Recipient.HUDDLE, user_profile__realm=realm
+            ).values_list("recipient_id", flat=True)
+        )
+    )
+    cross_realm_bot_message_ids = list(
+        Message.objects.filter(
+            # Filtering by both message.recipient and message.realm is more robust for ensuring
+            # no messages belonging to another realm will be deleted due to some bugs.
+            sender__realm=internal_realm,
+            recipient_id__in=all_recipient_ids_in_realm,
+            realm=realm,
+        ).values_list("id", flat=True)
+    )
+    move_messages_to_archive(cross_realm_bot_message_ids)
+
     do_remove_realm_custom_profile_fields(realm)
     do_delete_all_realm_attachments(realm)
 

zerver/tests/test_realm.py

@@ -11,6 +11,11 @@ from django.utils.timezone import now as timezone_now
 
 from confirmation.models import Confirmation, create_confirmation_link
 from zerver.actions.create_realm import do_change_realm_subdomain, do_create_realm
+from zerver.actions.message_send import (
+    internal_send_huddle_message,
+    internal_send_private_message,
+    internal_send_stream_message,
+)
 from zerver.actions.realm_settings import (
     do_add_deactivated_redirect,
     do_change_realm_org_type,
@@ -45,6 +50,7 @@ from zerver.models import (
     UserProfile,
     get_realm,
     get_stream,
+    get_system_bot,
     get_user_profile_by_id,
 )
 
@@ -1455,6 +1461,7 @@ class ScrubRealmTest(ZulipTestCase):
     def test_scrub_realm(self) -> None:
         zulip = get_realm("zulip")
         lear = get_realm("lear")
+        internal_realm = get_realm(settings.SYSTEM_BOT_REALM)
 
         hamlet = self.example_user("hamlet")
         iago = self.example_user("iago")
@@ -1463,6 +1470,8 @@ class ScrubRealmTest(ZulipTestCase):
         cordelia = self.lear_user("cordelia")
         king = self.lear_user("king")
 
+        notification_bot = get_system_bot(settings.NOTIFICATION_BOT, internal_realm.id)
+
         create_stream_if_needed(lear, "Shakespeare")
 
         self.subscribe(cordelia, "Shakespeare")
@@ -1477,6 +1486,18 @@ class ScrubRealmTest(ZulipTestCase):
             self.send_stream_message(cordelia, "Shakespeare")
             self.send_stream_message(king, "Shakespeare")
 
+        internal_send_stream_message(
+            notification_bot, get_stream("Scotland", zulip), "test", "test"
+        )
+        internal_send_private_message(notification_bot, othello, "test")
+        internal_send_huddle_message(zulip, notification_bot, [othello.email, iago.email], "test")
+
+        internal_send_stream_message(
+            notification_bot, get_stream("Shakespeare", lear), "test", "test"
+        )
+        internal_send_private_message(notification_bot, king, "test")
+        internal_send_huddle_message(lear, notification_bot, [cordelia.email, king.email], "test")
+
         Attachment.objects.filter(realm=zulip).delete()
         Attachment.objects.filter(realm=lear).delete()
         assert settings.LOCAL_UPLOADS_DIR is not None
@@ -1497,8 +1518,9 @@ class ScrubRealmTest(ZulipTestCase):
 
         self.assertEqual(Message.objects.filter(sender__in=[iago, othello]).count(), 10)
         self.assertEqual(Message.objects.filter(sender__in=[cordelia, king]).count(), 10)
-        self.assertEqual(UserMessage.objects.filter(user_profile__in=[iago, othello]).count(), 20)
-        self.assertEqual(UserMessage.objects.filter(user_profile__in=[cordelia, king]).count(), 20)
+        self.assertEqual(Message.objects.filter(sender=notification_bot).count(), 6)
+        self.assertEqual(UserMessage.objects.filter(user_profile__in=[iago, othello]).count(), 25)
+        self.assertEqual(UserMessage.objects.filter(user_profile__in=[cordelia, king]).count(), 25)
 
         self.assertNotEqual(CustomProfileField.objects.filter(realm=zulip).count(), 0)
 
@@ -1507,8 +1529,9 @@ class ScrubRealmTest(ZulipTestCase):
 
         self.assertEqual(Message.objects.filter(sender__in=[iago, othello]).count(), 0)
         self.assertEqual(Message.objects.filter(sender__in=[cordelia, king]).count(), 10)
+        self.assertEqual(Message.objects.filter(sender=notification_bot).count(), 3)
         self.assertEqual(UserMessage.objects.filter(user_profile__in=[iago, othello]).count(), 0)
-        self.assertEqual(UserMessage.objects.filter(user_profile__in=[cordelia, king]).count(), 20)
+        self.assertEqual(UserMessage.objects.filter(user_profile__in=[cordelia, king]).count(), 25)
 
         self.assertEqual(Attachment.objects.filter(realm=zulip).count(), 0)
         self.assertEqual(Attachment.objects.filter(realm=lear).count(), 2)