mirror of https://github.com/zulip/zulip.git
retention-period: Add more core code for retention policy.
This is a very old commit for #106, which has been on hiatus for a few years. It was significantly modified by tabbott to: * Improve coding style and variable names * Update mypy annotations style * Clean up the testing logic * Update for API changes elsewhere in our system But the actual runtime code is essentially unmodified from the original work by Kirill. It contains basic support for archiving Messages, UserMessages, and Attachments with a nice test suite. It's still not usable in production (e.g. it will probably break Reactions, SubMessages, etc.), but upcoming commits will address that.
This commit is contained in:
parent
ec4221abb9
commit
e930851d16
|
@ -3,29 +3,143 @@ from datetime import timedelta
|
||||||
|
|
||||||
from django.db import connection, transaction
|
from django.db import connection, transaction
|
||||||
from django.utils.timezone import now as timezone_now
|
from django.utils.timezone import now as timezone_now
|
||||||
from zerver.models import Realm, Message, UserMessage, ArchivedMessage, ArchivedUserMessage, \
|
from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
|
||||||
Attachment, ArchivedAttachment
|
Attachment, ArchivedAttachment)
|
||||||
|
|
||||||
from typing import Any, Dict, Optional, Generator, List
|
from typing import Any, List
|
||||||
|
|
||||||
|
|
||||||
def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]:
|
@transaction.atomic
|
||||||
expired_date = timezone_now() - timedelta(days=realm.message_retention_days)
|
def move_expired_rows(src_model: Any, raw_query: str, **kwargs: Any) -> None:
|
||||||
expired_messages = Message.objects.order_by('id').filter(sender__realm=realm,
|
src_db_table = src_model._meta.db_table
|
||||||
pub_date__lt=expired_date)
|
src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
|
||||||
if not expired_messages.exists():
|
dst_fields = [field.column for field in src_model._meta.fields]
|
||||||
return None
|
sql_args = {
|
||||||
return {'realm_id': realm.id, 'expired_messages': expired_messages}
|
'src_fields': ','.join(src_fields),
|
||||||
|
'dst_fields': ','.join(dst_fields),
|
||||||
|
'archive_timestamp': timezone_now()
|
||||||
|
}
|
||||||
|
sql_args.update(kwargs)
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute(
|
||||||
|
raw_query.format(**sql_args)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_expired_messages() -> Generator[Any, None, None]:
|
def move_expired_messages_to_archive(realm: Realm) -> None:
|
||||||
# Get all expired messages by Realm.
|
query = """
|
||||||
realms = Realm.objects.order_by('string_id').filter(
|
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
|
||||||
deactivated=False, message_retention_days__isnull=False)
|
SELECT {src_fields}, '{archive_timestamp}'
|
||||||
for realm in realms:
|
FROM zerver_message
|
||||||
realm_expired_messages = get_realm_expired_messages(realm)
|
INNER JOIN zerver_userprofile ON zerver_message.sender_id = zerver_userprofile.id
|
||||||
if realm_expired_messages:
|
LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
|
||||||
yield realm_expired_messages
|
WHERE zerver_userprofile.realm_id = {realm_id}
|
||||||
|
AND zerver_message.pub_date < '{check_date}'
|
||||||
|
AND zerver_archivedmessage.id is NULL
|
||||||
|
"""
|
||||||
|
assert realm.message_retention_days is not None
|
||||||
|
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
|
||||||
|
move_expired_rows(Message, query, realm_id=realm.id, check_date=check_date.isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def move_expired_user_messages_to_archive(realm: Realm) -> None:
|
||||||
|
query = """
|
||||||
|
INSERT INTO zerver_archivedusermessage ({dst_fields}, archive_timestamp)
|
||||||
|
SELECT {src_fields}, '{archive_timestamp}'
|
||||||
|
FROM zerver_usermessage
|
||||||
|
INNER JOIN zerver_userprofile ON zerver_usermessage.user_profile_id = zerver_userprofile.id
|
||||||
|
INNER JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_usermessage.message_id
|
||||||
|
LEFT JOIN zerver_archivedusermessage ON zerver_archivedusermessage.id = zerver_usermessage.id
|
||||||
|
LEFT JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id
|
||||||
|
WHERE zerver_userprofile.realm_id = {realm_id}
|
||||||
|
AND zerver_message.pub_date < '{check_date}'
|
||||||
|
AND zerver_archivedusermessage.id is NULL
|
||||||
|
"""
|
||||||
|
assert realm.message_retention_days is not None
|
||||||
|
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
|
||||||
|
move_expired_rows(UserMessage, query, realm_id=realm.id, check_date=check_date.isoformat())
|
||||||
|
|
||||||
|
def move_expired_attachments_to_archive(realm: Realm) -> None:
|
||||||
|
query = """
|
||||||
|
INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
|
||||||
|
SELECT {src_fields}, '{archive_timestamp}'
|
||||||
|
FROM zerver_attachment
|
||||||
|
INNER JOIN zerver_attachment_messages
|
||||||
|
ON zerver_attachment_messages.attachment_id = zerver_attachment.id
|
||||||
|
INNER JOIN zerver_archivedmessage
|
||||||
|
ON zerver_archivedmessage.id = zerver_attachment_messages.message_id
|
||||||
|
LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
|
||||||
|
WHERE zerver_attachment.realm_id = {realm_id}
|
||||||
|
AND zerver_archivedattachment.id IS NULL
|
||||||
|
GROUP BY zerver_attachment.id
|
||||||
|
"""
|
||||||
|
assert realm.message_retention_days is not None
|
||||||
|
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
|
||||||
|
move_expired_rows(Attachment, query, realm_id=realm.id, check_date=check_date.isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def move_expired_attachments_message_rows_to_archive(realm: Realm) -> None:
|
||||||
|
query = """
|
||||||
|
INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
|
||||||
|
SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
|
||||||
|
zerver_attachment_messages.message_id
|
||||||
|
FROM zerver_attachment_messages
|
||||||
|
INNER JOIN zerver_attachment
|
||||||
|
ON zerver_attachment_messages.attachment_id = zerver_attachment.id
|
||||||
|
INNER JOIN zerver_message ON zerver_attachment_messages.message_id = zerver_message.id
|
||||||
|
LEFT JOIN zerver_archivedattachment_messages
|
||||||
|
ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
|
||||||
|
WHERE zerver_attachment.realm_id = {realm_id}
|
||||||
|
AND zerver_message.pub_date < '{check_date}'
|
||||||
|
AND zerver_archivedattachment_messages.id IS NULL
|
||||||
|
"""
|
||||||
|
assert realm.message_retention_days is not None
|
||||||
|
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute(query.format(realm_id=realm.id, check_date=check_date.isoformat()))
|
||||||
|
|
||||||
|
|
||||||
|
def delete_expired_messages(realm: Realm) -> None:
|
||||||
|
removing_messages = Message.objects.filter(
|
||||||
|
usermessage__isnull=True, id__in=ArchivedMessage.objects.all(),
|
||||||
|
sender__realm_id=realm.id
|
||||||
|
)
|
||||||
|
removing_messages.delete()
|
||||||
|
|
||||||
|
|
||||||
|
def delete_expired_user_messages(realm: Realm) -> None:
|
||||||
|
removing_user_messages = UserMessage.objects.filter(
|
||||||
|
id__in=ArchivedUserMessage.objects.all(),
|
||||||
|
user_profile__realm_id=realm.id
|
||||||
|
)
|
||||||
|
removing_user_messages.delete()
|
||||||
|
|
||||||
|
|
||||||
|
def delete_expired_attachments(realm: Realm) -> None:
|
||||||
|
attachments_to_remove = Attachment.objects.filter(
|
||||||
|
messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
|
||||||
|
realm_id=realm.id
|
||||||
|
)
|
||||||
|
attachments_to_remove.delete()
|
||||||
|
|
||||||
|
|
||||||
|
def clean_unused_messages() -> None:
|
||||||
|
unused_messages = Message.objects.filter(
|
||||||
|
usermessage__isnull=True, id__in=ArchivedMessage.objects.all()
|
||||||
|
)
|
||||||
|
unused_messages.delete()
|
||||||
|
|
||||||
|
|
||||||
|
def archive_messages() -> None:
|
||||||
|
for realm in Realm.objects.filter(message_retention_days__isnull=False):
|
||||||
|
move_expired_messages_to_archive(realm)
|
||||||
|
move_expired_user_messages_to_archive(realm)
|
||||||
|
move_expired_attachments_to_archive(realm)
|
||||||
|
move_expired_attachments_message_rows_to_archive(realm)
|
||||||
|
delete_expired_user_messages(realm)
|
||||||
|
delete_expired_messages(realm)
|
||||||
|
delete_expired_attachments(realm)
|
||||||
|
clean_unused_messages()
|
||||||
|
|
||||||
|
|
||||||
def move_attachment_message_to_archive_by_message(message_ids: List[int]) -> None:
|
def move_attachment_message_to_archive_by_message(message_ids: List[int]) -> None:
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from zerver.lib.retention import archive_messages
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
|
||||||
|
def handle(self, *args: Any, **options: str) -> None:
|
||||||
|
archive_messages()
|
|
@ -1,16 +1,27 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import types
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from django.utils.timezone import now as timezone_now
|
from django.utils.timezone import now as timezone_now
|
||||||
|
|
||||||
|
from zerver.lib.actions import internal_send_private_message
|
||||||
from zerver.lib.test_classes import ZulipTestCase
|
from zerver.lib.test_classes import ZulipTestCase
|
||||||
from zerver.lib.upload import create_attachment
|
from zerver.lib.upload import create_attachment
|
||||||
from zerver.models import Message, Realm, UserMessage, ArchivedUserMessage, \
|
from zerver.models import (Message, Realm, UserProfile, ArchivedUserMessage,
|
||||||
ArchivedMessage, Attachment, ArchivedAttachment
|
ArchivedMessage, Attachment, ArchivedAttachment, UserMessage,
|
||||||
from zerver.lib.retention import get_expired_messages, move_messages_to_archive
|
get_user_profile_by_email, get_system_bot)
|
||||||
|
from zerver.lib.retention import (
|
||||||
from typing import Any, List, Tuple
|
archive_messages,
|
||||||
|
clean_unused_messages,
|
||||||
|
delete_expired_messages,
|
||||||
|
delete_expired_user_messages,
|
||||||
|
move_expired_messages_to_archive,
|
||||||
|
move_expired_user_messages_to_archive,
|
||||||
|
move_messages_to_archive
|
||||||
|
)
|
||||||
|
|
||||||
|
ZULIP_REALM_DAYS = 30
|
||||||
|
MIT_REALM_DAYS = 100
|
||||||
|
|
||||||
class TestRetentionLib(ZulipTestCase):
|
class TestRetentionLib(ZulipTestCase):
|
||||||
"""
|
"""
|
||||||
|
@ -19,8 +30,8 @@ class TestRetentionLib(ZulipTestCase):
|
||||||
|
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
super().setUp()
|
super().setUp()
|
||||||
self.zulip_realm = self._set_realm_message_retention_value('zulip', 30)
|
self.zulip_realm = self._set_realm_message_retention_value('zulip', ZULIP_REALM_DAYS)
|
||||||
self.mit_realm = self._set_realm_message_retention_value('zephyr', 100)
|
self.mit_realm = self._set_realm_message_retention_value('zephyr', MIT_REALM_DAYS)
|
||||||
Message.objects.all().update(pub_date=timezone_now())
|
Message.objects.all().update(pub_date=timezone_now())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -46,67 +57,280 @@ class TestRetentionLib(ZulipTestCase):
|
||||||
mit_messages = self._change_messages_pub_date(msgs_ids, pub_date)
|
mit_messages = self._change_messages_pub_date(msgs_ids, pub_date)
|
||||||
return mit_messages
|
return mit_messages
|
||||||
|
|
||||||
def test_expired_messages_result_type(self) -> None:
|
def _send_cross_realm_message(self) -> int:
|
||||||
# Check return type of get_expired_message method.
|
# Send message from bot to users from different realm.
|
||||||
result = get_expired_messages()
|
bot_email = 'notification-bot@zulip.com'
|
||||||
self.assertIsInstance(result, types.GeneratorType)
|
get_user_profile_by_email(bot_email)
|
||||||
|
mit_user = UserProfile.objects.filter(realm=self.mit_realm).first()
|
||||||
|
result = internal_send_private_message(
|
||||||
|
realm=mit_user.realm,
|
||||||
|
sender=get_system_bot(bot_email),
|
||||||
|
recipient_user=mit_user,
|
||||||
|
content='test message',
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _check_archive_data_by_realm(self, expected_messages: Any, realm: Realm) -> None:
|
||||||
|
self._check_archived_messages_ids_by_realm(
|
||||||
|
[msg.id for msg in expected_messages.order_by('id')],
|
||||||
|
realm
|
||||||
|
)
|
||||||
|
user_messages = UserMessage.objects.filter(message__in=expected_messages).order_by('id')
|
||||||
|
archived_user_messages = ArchivedUserMessage.objects.filter(
|
||||||
|
user_profile__realm=realm).order_by('id')
|
||||||
|
self.assertEqual(
|
||||||
|
[user_msg.id for user_msg in user_messages],
|
||||||
|
[arc_user_msg.id for arc_user_msg in archived_user_messages]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _check_archived_messages_ids_by_realm(self, expected_message_ids: List[int],
|
||||||
|
realm: Realm) -> None:
|
||||||
|
arc_messages = ArchivedMessage.objects.filter(
|
||||||
|
archivedusermessage__user_profile__realm=realm).distinct('id').order_by('id')
|
||||||
|
self.assertEqual(
|
||||||
|
expected_message_ids,
|
||||||
|
[arc_msg.id for arc_msg in arc_messages]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _send_messages_with_attachments(self) -> Dict[str, int]:
|
||||||
|
user_profile = self.example_user("hamlet")
|
||||||
|
sender_email = user_profile.email
|
||||||
|
sample_size = 10
|
||||||
|
dummy_files = [
|
||||||
|
('zulip.txt', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/zulip.txt', sample_size),
|
||||||
|
('temp_file.py', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/temp_file.py', sample_size),
|
||||||
|
('abc.py', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/abc.py', sample_size)
|
||||||
|
]
|
||||||
|
|
||||||
|
for file_name, path_id, size in dummy_files:
|
||||||
|
create_attachment(file_name, path_id, user_profile, size)
|
||||||
|
|
||||||
|
self.subscribe(user_profile, "Denmark")
|
||||||
|
body = "Some files here ...[zulip.txt](http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/zulip.txt)" + \
|
||||||
|
"http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/temp_file.py.... Some more...." + \
|
||||||
|
"http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/abc.py"
|
||||||
|
|
||||||
|
expired_message_id = self.send_stream_message(sender_email, "Denmark", body)
|
||||||
|
actual_message_id = self.send_stream_message(sender_email, "Denmark", body)
|
||||||
|
other_message_id = self.send_stream_message("othello@zulip.com", "Denmark", body)
|
||||||
|
self._change_messages_pub_date([expired_message_id], timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
|
return {'expired_message_id': expired_message_id, 'actual_message_id': actual_message_id,
|
||||||
|
'other_user_message_id': other_message_id}
|
||||||
|
|
||||||
|
def _check_cross_realm_messages_archiving(self, arc_user_msg_qty: int, period: int,
|
||||||
|
realm: Optional[Realm]=None) -> int:
|
||||||
|
sent_message_id = self._send_cross_realm_message()
|
||||||
|
all_user_messages_qty = UserMessage.objects.count()
|
||||||
|
self._change_messages_pub_date([sent_message_id], timezone_now() - timedelta(days=period))
|
||||||
|
realms = Realm.objects.filter(message_retention_days__isnull=False)
|
||||||
|
for realm_instance in realms:
|
||||||
|
move_expired_messages_to_archive(realm_instance)
|
||||||
|
move_expired_user_messages_to_archive(realm_instance)
|
||||||
|
user_messages_sent = UserMessage.objects.order_by('id').filter(
|
||||||
|
message_id=sent_message_id)
|
||||||
|
archived_messages = ArchivedMessage.objects.all()
|
||||||
|
archived_user_messages = ArchivedUserMessage.objects.order_by('id')
|
||||||
|
self.assertEqual(user_messages_sent.count(), 2)
|
||||||
|
|
||||||
|
# Compare archived messages and user messages
|
||||||
|
# with expired sent messages.
|
||||||
|
self.assertEqual(archived_messages.count(), 1)
|
||||||
|
self.assertEqual(archived_user_messages.count(), arc_user_msg_qty)
|
||||||
|
if realm:
|
||||||
|
user_messages_sent = user_messages_sent.filter(user_profile__realm=self.zulip_realm)
|
||||||
|
self.assertEqual(
|
||||||
|
[arc_user_msg.id for arc_user_msg in archived_user_messages],
|
||||||
|
[user_msg.id for user_msg in user_messages_sent]
|
||||||
|
)
|
||||||
|
for realm_instance in realms:
|
||||||
|
delete_expired_user_messages(realm_instance)
|
||||||
|
delete_expired_messages(realm_instance)
|
||||||
|
clean_unused_messages()
|
||||||
|
|
||||||
|
# Check messages and user messages after deleting expired messages
|
||||||
|
# from the main tables.
|
||||||
|
self.assertEqual(
|
||||||
|
UserMessage.objects.filter(message_id=sent_message_id).count(),
|
||||||
|
2 - arc_user_msg_qty)
|
||||||
|
self.assertEqual(
|
||||||
|
UserMessage.objects.count(),
|
||||||
|
all_user_messages_qty - arc_user_msg_qty)
|
||||||
|
return sent_message_id
|
||||||
|
|
||||||
|
def _make_expired_messages(self) -> Dict[str, List[int]]:
|
||||||
|
# Create messages in Zephyr realm with already-expired date
|
||||||
|
expected_mit_msgs = self._make_mit_messages(3, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
|
expected_mit_msgs_ids = [msg.id for msg in expected_mit_msgs.order_by('id')]
|
||||||
|
self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
|
||||||
|
|
||||||
|
# Move existing messages in Zulip realm to be expired
|
||||||
|
expected_zulip_msgs_ids = list(Message.objects.order_by('id').filter(
|
||||||
|
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10])
|
||||||
|
self._change_messages_pub_date(expected_zulip_msgs_ids,
|
||||||
|
timezone_now() - timedelta(days=ZULIP_REALM_DAYS + 1))
|
||||||
|
return {
|
||||||
|
"mit_msgs_ids": expected_mit_msgs_ids,
|
||||||
|
"zulip_msgs_ids": expected_zulip_msgs_ids
|
||||||
|
}
|
||||||
|
|
||||||
def test_no_expired_messages(self) -> None:
|
def test_no_expired_messages(self) -> None:
|
||||||
result = list(get_expired_messages())
|
for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
|
||||||
self.assertFalse(result)
|
move_expired_messages_to_archive(realm_instance)
|
||||||
|
move_expired_user_messages_to_archive(realm_instance)
|
||||||
|
self.assertEqual(ArchivedUserMessage.objects.count(), 0)
|
||||||
|
self.assertEqual(ArchivedMessage.objects.count(), 0)
|
||||||
|
|
||||||
def test_expired_messages_in_each_realm(self) -> None:
|
def test_expired_messages_in_each_realm(self) -> None:
|
||||||
# Check result realm messages order and result content
|
"""General test for archiving expired messages properly with
|
||||||
# when all realm has expired messages.
|
multiple realms involved"""
|
||||||
expired_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=101))
|
expected_message_ids = []
|
||||||
self._make_mit_messages(4, timezone_now() - timedelta(days=50))
|
expected_mit_msgs = self._make_mit_messages(3, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
zulip_messages_ids = Message.objects.order_by('id').filter(
|
expected_message_ids.extend([msg.id for msg in expected_mit_msgs.order_by('id')])
|
||||||
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10]
|
self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
|
||||||
expired_zulip_messages = self._change_messages_pub_date(zulip_messages_ids,
|
zulip_msgs_ids = list(Message.objects.order_by('id').filter(
|
||||||
timezone_now() - timedelta(days=31))
|
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10])
|
||||||
# Iterate by result
|
expected_message_ids.extend(zulip_msgs_ids)
|
||||||
expired_messages_result = [messages_list for messages_list in get_expired_messages()]
|
expected_zulip_msgs = self._change_messages_pub_date(
|
||||||
self.assertEqual(len(expired_messages_result), 2)
|
zulip_msgs_ids,
|
||||||
# Check mit.edu realm expired messages.
|
timezone_now() - timedelta(days=ZULIP_REALM_DAYS + 1))
|
||||||
self.assertEqual(len(expired_messages_result[0]['expired_messages']), 3)
|
|
||||||
self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id)
|
for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
|
||||||
# Check zulip.com realm expired messages.
|
move_expired_messages_to_archive(realm_instance)
|
||||||
self.assertEqual(len(expired_messages_result[1]['expired_messages']), 7)
|
move_expired_user_messages_to_archive(realm_instance)
|
||||||
self.assertEqual(expired_messages_result[1]['realm_id'], self.zulip_realm.id)
|
self.assertEqual(ArchivedMessage.objects.count(), len(expected_message_ids))
|
||||||
# Compare expected messages ids with result messages ids.
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sorted([message.id for message in expired_mit_messages]),
|
ArchivedUserMessage.objects.count(),
|
||||||
[message.id for message in expired_messages_result[0]['expired_messages']]
|
UserMessage.objects.filter(message_id__in=expected_message_ids).count()
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
sorted([message.id for message in expired_zulip_messages]),
|
|
||||||
[message.id for message in expired_messages_result[1]['expired_messages']]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Compare expected messages ids with archived messages for both realms
|
||||||
|
self._check_archive_data_by_realm(expected_mit_msgs, self.mit_realm)
|
||||||
|
self._check_archive_data_by_realm(expected_zulip_msgs, self.zulip_realm)
|
||||||
|
|
||||||
def test_expired_messages_in_one_realm(self) -> None:
|
def test_expired_messages_in_one_realm(self) -> None:
|
||||||
# Check realm with expired messages and messages
|
"""Test with a retention policy set for only the MIT realm"""
|
||||||
# with one day to expiration data.
|
expected_mit_msgs = self._make_mit_messages(
|
||||||
expired_mit_messages = self._make_mit_messages(5, timezone_now() - timedelta(days=101))
|
5, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
actual_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=99))
|
|
||||||
expired_messages_result = list(get_expired_messages())
|
for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
|
||||||
expired_mit_messages_ids = [message.id for message in expired_mit_messages]
|
move_expired_messages_to_archive(realm_instance)
|
||||||
expired_mit_messages_result_ids = [message.id for message in
|
move_expired_user_messages_to_archive(realm_instance)
|
||||||
expired_messages_result[0]['expired_messages']]
|
|
||||||
actual_mit_messages_ids = [message.id for message in actual_mit_messages]
|
self.assertEqual(ArchivedMessage.objects.count(), 5)
|
||||||
self.assertEqual(len(expired_messages_result), 1)
|
self.assertEqual(ArchivedUserMessage.objects.count(), 10)
|
||||||
self.assertEqual(len(expired_messages_result[0]['expired_messages']), 5)
|
|
||||||
self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id)
|
# Compare expected messages ids with archived messages in mit realm
|
||||||
# Compare expected messages ids with result messages ids.
|
self._check_archive_data_by_realm(expected_mit_msgs, self.mit_realm)
|
||||||
|
# Check no archive messages for zulip realm.
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sorted(expired_mit_messages_ids),
|
ArchivedMessage.objects.filter(
|
||||||
expired_mit_messages_result_ids
|
archivedusermessage__user_profile__realm=self.zulip_realm).count(),
|
||||||
|
0
|
||||||
)
|
)
|
||||||
# Check actual mit.edu messages are not contained in expired messages list
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
set(actual_mit_messages_ids) - set(expired_mit_messages_ids),
|
ArchivedUserMessage.objects.filter(user_profile__realm=self.zulip_realm).count(),
|
||||||
set(actual_mit_messages_ids)
|
0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_cross_realm_messages_archiving_one_realm_expired(self) -> None:
|
||||||
|
"""Test that a cross-realm message that is expired in only
|
||||||
|
one of the realms only has the UserMessage for that realm archived"""
|
||||||
|
arc_msg_id = self._check_cross_realm_messages_archiving(
|
||||||
|
1, ZULIP_REALM_DAYS + 1, realm=self.zulip_realm)
|
||||||
|
self.assertTrue(Message.objects.filter(id=arc_msg_id).exists())
|
||||||
|
|
||||||
|
def test_cross_realm_messages_archiving_two_realm_expired(self) -> None:
|
||||||
|
"""Check that archiving a message that's expired in both
|
||||||
|
realms is archived both in Message and UserMessage."""
|
||||||
|
arc_msg_id = self._check_cross_realm_messages_archiving(2, MIT_REALM_DAYS + 1)
|
||||||
|
self.assertFalse(Message.objects.filter(id=arc_msg_id).exists())
|
||||||
|
|
||||||
|
def test_archive_message_tool(self) -> None:
|
||||||
|
"""End-to-end test of the archiving tool, directly calling
|
||||||
|
archive_messages."""
|
||||||
|
expected_message_ids_dict = self._make_expired_messages()
|
||||||
|
|
||||||
|
# We also include a cross-realm message in this test.
|
||||||
|
sent_cross_realm_message_id = self._send_cross_realm_message()
|
||||||
|
expected_message_ids_dict['mit_msgs_ids'].append(sent_cross_realm_message_id)
|
||||||
|
self._change_messages_pub_date(
|
||||||
|
[sent_cross_realm_message_id],
|
||||||
|
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1)
|
||||||
|
)
|
||||||
|
expected_message_ids = expected_message_ids_dict['mit_msgs_ids'] + expected_message_ids_dict['zulip_msgs_ids']
|
||||||
|
|
||||||
|
# Get expired user messages by message ids
|
||||||
|
expected_user_msgs_ids = list(UserMessage.objects.filter(
|
||||||
|
message_id__in=expected_message_ids).order_by('id').values_list('id', flat=True))
|
||||||
|
|
||||||
|
msgs_qty = Message.objects.count()
|
||||||
|
archive_messages()
|
||||||
|
|
||||||
|
# Compare archived messages and user messages with expired messages
|
||||||
|
self.assertEqual(ArchivedMessage.objects.count(), len(expected_message_ids))
|
||||||
|
self.assertEqual(ArchivedUserMessage.objects.count(), len(expected_user_msgs_ids))
|
||||||
|
|
||||||
|
# Check non-archived messages messages after removing expired
|
||||||
|
# messages from main tables without cross-realm messages.
|
||||||
|
self.assertEqual(Message.objects.count(), msgs_qty - ArchivedMessage.objects.count())
|
||||||
|
self.assertEqual(
|
||||||
|
Message.objects.filter(id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
|
||||||
|
self.assertEqual(
|
||||||
|
Message.objects.filter(id__in=expected_message_ids_dict['mit_msgs_ids']).count(), 0)
|
||||||
|
self.assertEqual(
|
||||||
|
Message.objects.filter(id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
|
||||||
|
|
||||||
|
# Check archived messages by realm using our standard checker
|
||||||
|
# function; we add the cross-realm message ID to the
|
||||||
|
# zulip_realm list for this test because its sender lives in
|
||||||
|
# that realm in the development environment.
|
||||||
|
expected_message_ids_dict['zulip_msgs_ids'].append(sent_cross_realm_message_id)
|
||||||
|
self._check_archived_messages_ids_by_realm(
|
||||||
|
expected_message_ids_dict['zulip_msgs_ids'], self.zulip_realm)
|
||||||
|
self._check_archived_messages_ids_by_realm(
|
||||||
|
expected_message_ids_dict['mit_msgs_ids'], self.mit_realm)
|
||||||
|
|
||||||
|
def test_archiving_attachments(self) -> None:
|
||||||
|
"""End-to-end test for the logic for archiving attachments. This test
|
||||||
|
is hard to read without first reading _send_messages_with_attachments"""
|
||||||
|
msgs_ids = self._send_messages_with_attachments()
|
||||||
|
|
||||||
|
# First, confirm deleting the oldest message
|
||||||
|
# (`expired_message_id`) creates ArchivedAttachment objects
|
||||||
|
# and associates that message ID with them, but does not
|
||||||
|
# delete the Attachment object.
|
||||||
|
archive_messages()
|
||||||
|
archived_attachment = ArchivedAttachment.objects.all()
|
||||||
|
attachment = Attachment.objects.all()
|
||||||
|
self.assertEqual(archived_attachment.count(), 3)
|
||||||
|
self.assertEqual(
|
||||||
|
list(archived_attachment.distinct('messages__id').values_list('messages__id',
|
||||||
|
flat=True)),
|
||||||
|
[msgs_ids['expired_message_id']])
|
||||||
|
self.assertEqual(attachment.count(), 3)
|
||||||
|
|
||||||
|
# Now make `actual_message_id` expired too. We still don't
|
||||||
|
# delete the Attachment objects.
|
||||||
|
self._change_messages_pub_date([msgs_ids['actual_message_id']],
|
||||||
|
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
|
archive_messages()
|
||||||
|
self.assertEqual(attachment.count(), 3)
|
||||||
|
|
||||||
|
# Finally, make the last message mentioning those attachments
|
||||||
|
# expired. We should now delete the Attachment objects and
|
||||||
|
# each ArchivedAttachment object should list all 3 messages.
|
||||||
|
self._change_messages_pub_date([msgs_ids['other_user_message_id']],
|
||||||
|
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
|
||||||
|
|
||||||
|
archive_messages()
|
||||||
|
self.assertEqual(attachment.count(), 0)
|
||||||
|
self.assertEqual(archived_attachment.count(), 3)
|
||||||
|
self.assertEqual(
|
||||||
|
list(archived_attachment.distinct('messages__id').order_by('messages__id').values_list(
|
||||||
|
'messages__id', flat=True)),
|
||||||
|
sorted(msgs_ids.values()))
|
||||||
|
|
||||||
|
|
||||||
class TestMoveMessageToArchive(ZulipTestCase):
|
class TestMoveMessageToArchive(ZulipTestCase):
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue