retention-period: Add more core code for retention policy.

This is a very old commit for #106, which has been on hiatus for a few
years.  It was significantly modified by tabbott to:
* Improve coding style and variable names
* Update mypy annotations style
* Clean up the testing logic
* Update for API changes elsewhere in our system

But the actual runtime code is essentially unmodified from the
original work by Kirill.

It contains basic support for archiving Messages, UserMessages, and
Attachments with a nice test suite.  It's still not usable in
production (e.g. it will probably break Reactions, SubMessages, etc.),
but upcoming commits will address that.
This commit is contained in:
K.Kanakhin 2016-11-01 16:26:38 +06:00 committed by Tim Abbott
parent ec4221abb9
commit e930851d16
3 changed files with 424 additions and 74 deletions

View File

@ -3,29 +3,143 @@ from datetime import timedelta
from django.db import connection, transaction from django.db import connection, transaction
from django.utils.timezone import now as timezone_now from django.utils.timezone import now as timezone_now
from zerver.models import Realm, Message, UserMessage, ArchivedMessage, ArchivedUserMessage, \ from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
Attachment, ArchivedAttachment Attachment, ArchivedAttachment)
from typing import Any, Dict, Optional, Generator, List from typing import Any, List
def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]: @transaction.atomic
expired_date = timezone_now() - timedelta(days=realm.message_retention_days) def move_expired_rows(src_model: Any, raw_query: str, **kwargs: Any) -> None:
expired_messages = Message.objects.order_by('id').filter(sender__realm=realm, src_db_table = src_model._meta.db_table
pub_date__lt=expired_date) src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
if not expired_messages.exists(): dst_fields = [field.column for field in src_model._meta.fields]
return None sql_args = {
return {'realm_id': realm.id, 'expired_messages': expired_messages} 'src_fields': ','.join(src_fields),
'dst_fields': ','.join(dst_fields),
'archive_timestamp': timezone_now()
}
sql_args.update(kwargs)
with connection.cursor() as cursor:
cursor.execute(
raw_query.format(**sql_args)
)
def get_expired_messages() -> Generator[Any, None, None]: def move_expired_messages_to_archive(realm: Realm) -> None:
# Get all expired messages by Realm. query = """
realms = Realm.objects.order_by('string_id').filter( INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
deactivated=False, message_retention_days__isnull=False) SELECT {src_fields}, '{archive_timestamp}'
for realm in realms: FROM zerver_message
realm_expired_messages = get_realm_expired_messages(realm) INNER JOIN zerver_userprofile ON zerver_message.sender_id = zerver_userprofile.id
if realm_expired_messages: LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
yield realm_expired_messages WHERE zerver_userprofile.realm_id = {realm_id}
AND zerver_message.pub_date < '{check_date}'
AND zerver_archivedmessage.id is NULL
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
move_expired_rows(Message, query, realm_id=realm.id, check_date=check_date.isoformat())
def move_expired_user_messages_to_archive(realm: Realm) -> None:
query = """
INSERT INTO zerver_archivedusermessage ({dst_fields}, archive_timestamp)
SELECT {src_fields}, '{archive_timestamp}'
FROM zerver_usermessage
INNER JOIN zerver_userprofile ON zerver_usermessage.user_profile_id = zerver_userprofile.id
INNER JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_usermessage.message_id
LEFT JOIN zerver_archivedusermessage ON zerver_archivedusermessage.id = zerver_usermessage.id
LEFT JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id
WHERE zerver_userprofile.realm_id = {realm_id}
AND zerver_message.pub_date < '{check_date}'
AND zerver_archivedusermessage.id is NULL
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
move_expired_rows(UserMessage, query, realm_id=realm.id, check_date=check_date.isoformat())
def move_expired_attachments_to_archive(realm: Realm) -> None:
query = """
INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
SELECT {src_fields}, '{archive_timestamp}'
FROM zerver_attachment
INNER JOIN zerver_attachment_messages
ON zerver_attachment_messages.attachment_id = zerver_attachment.id
INNER JOIN zerver_archivedmessage
ON zerver_archivedmessage.id = zerver_attachment_messages.message_id
LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
WHERE zerver_attachment.realm_id = {realm_id}
AND zerver_archivedattachment.id IS NULL
GROUP BY zerver_attachment.id
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
move_expired_rows(Attachment, query, realm_id=realm.id, check_date=check_date.isoformat())
def move_expired_attachments_message_rows_to_archive(realm: Realm) -> None:
query = """
INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
zerver_attachment_messages.message_id
FROM zerver_attachment_messages
INNER JOIN zerver_attachment
ON zerver_attachment_messages.attachment_id = zerver_attachment.id
INNER JOIN zerver_message ON zerver_attachment_messages.message_id = zerver_message.id
LEFT JOIN zerver_archivedattachment_messages
ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
WHERE zerver_attachment.realm_id = {realm_id}
AND zerver_message.pub_date < '{check_date}'
AND zerver_archivedattachment_messages.id IS NULL
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
with connection.cursor() as cursor:
cursor.execute(query.format(realm_id=realm.id, check_date=check_date.isoformat()))
def delete_expired_messages(realm: Realm) -> None:
removing_messages = Message.objects.filter(
usermessage__isnull=True, id__in=ArchivedMessage.objects.all(),
sender__realm_id=realm.id
)
removing_messages.delete()
def delete_expired_user_messages(realm: Realm) -> None:
removing_user_messages = UserMessage.objects.filter(
id__in=ArchivedUserMessage.objects.all(),
user_profile__realm_id=realm.id
)
removing_user_messages.delete()
def delete_expired_attachments(realm: Realm) -> None:
attachments_to_remove = Attachment.objects.filter(
messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
realm_id=realm.id
)
attachments_to_remove.delete()
def clean_unused_messages() -> None:
unused_messages = Message.objects.filter(
usermessage__isnull=True, id__in=ArchivedMessage.objects.all()
)
unused_messages.delete()
def archive_messages() -> None:
for realm in Realm.objects.filter(message_retention_days__isnull=False):
move_expired_messages_to_archive(realm)
move_expired_user_messages_to_archive(realm)
move_expired_attachments_to_archive(realm)
move_expired_attachments_message_rows_to_archive(realm)
delete_expired_user_messages(realm)
delete_expired_messages(realm)
delete_expired_attachments(realm)
clean_unused_messages()
def move_attachment_message_to_archive_by_message(message_ids: List[int]) -> None: def move_attachment_message_to_archive_by_message(message_ids: List[int]) -> None:

View File

@ -0,0 +1,12 @@
from __future__ import absolute_import
from typing import Any
from django.core.management.base import BaseCommand
from zerver.lib.retention import archive_messages
class Command(BaseCommand):
def handle(self, *args: Any, **options: str) -> None:
archive_messages()

View File

@ -1,16 +1,27 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import types
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional, Tuple
from django.utils.timezone import now as timezone_now from django.utils.timezone import now as timezone_now
from zerver.lib.actions import internal_send_private_message
from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.upload import create_attachment from zerver.lib.upload import create_attachment
from zerver.models import Message, Realm, UserMessage, ArchivedUserMessage, \ from zerver.models import (Message, Realm, UserProfile, ArchivedUserMessage,
ArchivedMessage, Attachment, ArchivedAttachment ArchivedMessage, Attachment, ArchivedAttachment, UserMessage,
from zerver.lib.retention import get_expired_messages, move_messages_to_archive get_user_profile_by_email, get_system_bot)
from zerver.lib.retention import (
from typing import Any, List, Tuple archive_messages,
clean_unused_messages,
delete_expired_messages,
delete_expired_user_messages,
move_expired_messages_to_archive,
move_expired_user_messages_to_archive,
move_messages_to_archive
)
ZULIP_REALM_DAYS = 30
MIT_REALM_DAYS = 100
class TestRetentionLib(ZulipTestCase): class TestRetentionLib(ZulipTestCase):
""" """
@ -19,8 +30,8 @@ class TestRetentionLib(ZulipTestCase):
def setUp(self) -> None: def setUp(self) -> None:
super().setUp() super().setUp()
self.zulip_realm = self._set_realm_message_retention_value('zulip', 30) self.zulip_realm = self._set_realm_message_retention_value('zulip', ZULIP_REALM_DAYS)
self.mit_realm = self._set_realm_message_retention_value('zephyr', 100) self.mit_realm = self._set_realm_message_retention_value('zephyr', MIT_REALM_DAYS)
Message.objects.all().update(pub_date=timezone_now()) Message.objects.all().update(pub_date=timezone_now())
@staticmethod @staticmethod
@ -46,67 +57,280 @@ class TestRetentionLib(ZulipTestCase):
mit_messages = self._change_messages_pub_date(msgs_ids, pub_date) mit_messages = self._change_messages_pub_date(msgs_ids, pub_date)
return mit_messages return mit_messages
def test_expired_messages_result_type(self) -> None: def _send_cross_realm_message(self) -> int:
# Check return type of get_expired_message method. # Send message from bot to users from different realm.
result = get_expired_messages() bot_email = 'notification-bot@zulip.com'
self.assertIsInstance(result, types.GeneratorType) get_user_profile_by_email(bot_email)
mit_user = UserProfile.objects.filter(realm=self.mit_realm).first()
result = internal_send_private_message(
realm=mit_user.realm,
sender=get_system_bot(bot_email),
recipient_user=mit_user,
content='test message',
)
assert result is not None
return result
def _check_archive_data_by_realm(self, expected_messages: Any, realm: Realm) -> None:
self._check_archived_messages_ids_by_realm(
[msg.id for msg in expected_messages.order_by('id')],
realm
)
user_messages = UserMessage.objects.filter(message__in=expected_messages).order_by('id')
archived_user_messages = ArchivedUserMessage.objects.filter(
user_profile__realm=realm).order_by('id')
self.assertEqual(
[user_msg.id for user_msg in user_messages],
[arc_user_msg.id for arc_user_msg in archived_user_messages]
)
def _check_archived_messages_ids_by_realm(self, expected_message_ids: List[int],
realm: Realm) -> None:
arc_messages = ArchivedMessage.objects.filter(
archivedusermessage__user_profile__realm=realm).distinct('id').order_by('id')
self.assertEqual(
expected_message_ids,
[arc_msg.id for arc_msg in arc_messages]
)
def _send_messages_with_attachments(self) -> Dict[str, int]:
user_profile = self.example_user("hamlet")
sender_email = user_profile.email
sample_size = 10
dummy_files = [
('zulip.txt', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/zulip.txt', sample_size),
('temp_file.py', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/temp_file.py', sample_size),
('abc.py', '1/31/4CBjtTLYZhk66pZrF8hnYGwc/abc.py', sample_size)
]
for file_name, path_id, size in dummy_files:
create_attachment(file_name, path_id, user_profile, size)
self.subscribe(user_profile, "Denmark")
body = "Some files here ...[zulip.txt](http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/zulip.txt)" + \
"http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/temp_file.py.... Some more...." + \
"http://localhost:9991/user_uploads/1/31/4CBjtTLYZhk66pZrF8hnYGwc/abc.py"
expired_message_id = self.send_stream_message(sender_email, "Denmark", body)
actual_message_id = self.send_stream_message(sender_email, "Denmark", body)
other_message_id = self.send_stream_message("othello@zulip.com", "Denmark", body)
self._change_messages_pub_date([expired_message_id], timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
return {'expired_message_id': expired_message_id, 'actual_message_id': actual_message_id,
'other_user_message_id': other_message_id}
def _check_cross_realm_messages_archiving(self, arc_user_msg_qty: int, period: int,
realm: Optional[Realm]=None) -> int:
sent_message_id = self._send_cross_realm_message()
all_user_messages_qty = UserMessage.objects.count()
self._change_messages_pub_date([sent_message_id], timezone_now() - timedelta(days=period))
realms = Realm.objects.filter(message_retention_days__isnull=False)
for realm_instance in realms:
move_expired_messages_to_archive(realm_instance)
move_expired_user_messages_to_archive(realm_instance)
user_messages_sent = UserMessage.objects.order_by('id').filter(
message_id=sent_message_id)
archived_messages = ArchivedMessage.objects.all()
archived_user_messages = ArchivedUserMessage.objects.order_by('id')
self.assertEqual(user_messages_sent.count(), 2)
# Compare archived messages and user messages
# with expired sent messages.
self.assertEqual(archived_messages.count(), 1)
self.assertEqual(archived_user_messages.count(), arc_user_msg_qty)
if realm:
user_messages_sent = user_messages_sent.filter(user_profile__realm=self.zulip_realm)
self.assertEqual(
[arc_user_msg.id for arc_user_msg in archived_user_messages],
[user_msg.id for user_msg in user_messages_sent]
)
for realm_instance in realms:
delete_expired_user_messages(realm_instance)
delete_expired_messages(realm_instance)
clean_unused_messages()
# Check messages and user messages after deleting expired messages
# from the main tables.
self.assertEqual(
UserMessage.objects.filter(message_id=sent_message_id).count(),
2 - arc_user_msg_qty)
self.assertEqual(
UserMessage.objects.count(),
all_user_messages_qty - arc_user_msg_qty)
return sent_message_id
def _make_expired_messages(self) -> Dict[str, List[int]]:
# Create messages in Zephyr realm with already-expired date
expected_mit_msgs = self._make_mit_messages(3, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
expected_mit_msgs_ids = [msg.id for msg in expected_mit_msgs.order_by('id')]
self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
# Move existing messages in Zulip realm to be expired
expected_zulip_msgs_ids = list(Message.objects.order_by('id').filter(
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10])
self._change_messages_pub_date(expected_zulip_msgs_ids,
timezone_now() - timedelta(days=ZULIP_REALM_DAYS + 1))
return {
"mit_msgs_ids": expected_mit_msgs_ids,
"zulip_msgs_ids": expected_zulip_msgs_ids
}
def test_no_expired_messages(self) -> None: def test_no_expired_messages(self) -> None:
result = list(get_expired_messages()) for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
self.assertFalse(result) move_expired_messages_to_archive(realm_instance)
move_expired_user_messages_to_archive(realm_instance)
self.assertEqual(ArchivedUserMessage.objects.count(), 0)
self.assertEqual(ArchivedMessage.objects.count(), 0)
def test_expired_messages_in_each_realm(self) -> None: def test_expired_messages_in_each_realm(self) -> None:
# Check result realm messages order and result content """General test for archiving expired messages properly with
# when all realm has expired messages. multiple realms involved"""
expired_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=101)) expected_message_ids = []
self._make_mit_messages(4, timezone_now() - timedelta(days=50)) expected_mit_msgs = self._make_mit_messages(3, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
zulip_messages_ids = Message.objects.order_by('id').filter( expected_message_ids.extend([msg.id for msg in expected_mit_msgs.order_by('id')])
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10] self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
expired_zulip_messages = self._change_messages_pub_date(zulip_messages_ids, zulip_msgs_ids = list(Message.objects.order_by('id').filter(
timezone_now() - timedelta(days=31)) sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10])
# Iterate by result expected_message_ids.extend(zulip_msgs_ids)
expired_messages_result = [messages_list for messages_list in get_expired_messages()] expected_zulip_msgs = self._change_messages_pub_date(
self.assertEqual(len(expired_messages_result), 2) zulip_msgs_ids,
# Check mit.edu realm expired messages. timezone_now() - timedelta(days=ZULIP_REALM_DAYS + 1))
self.assertEqual(len(expired_messages_result[0]['expired_messages']), 3)
self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id) for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
# Check zulip.com realm expired messages. move_expired_messages_to_archive(realm_instance)
self.assertEqual(len(expired_messages_result[1]['expired_messages']), 7) move_expired_user_messages_to_archive(realm_instance)
self.assertEqual(expired_messages_result[1]['realm_id'], self.zulip_realm.id) self.assertEqual(ArchivedMessage.objects.count(), len(expected_message_ids))
# Compare expected messages ids with result messages ids.
self.assertEqual( self.assertEqual(
sorted([message.id for message in expired_mit_messages]), ArchivedUserMessage.objects.count(),
[message.id for message in expired_messages_result[0]['expired_messages']] UserMessage.objects.filter(message_id__in=expected_message_ids).count()
)
self.assertEqual(
sorted([message.id for message in expired_zulip_messages]),
[message.id for message in expired_messages_result[1]['expired_messages']]
) )
# Compare expected messages ids with archived messages for both realms
self._check_archive_data_by_realm(expected_mit_msgs, self.mit_realm)
self._check_archive_data_by_realm(expected_zulip_msgs, self.zulip_realm)
def test_expired_messages_in_one_realm(self) -> None: def test_expired_messages_in_one_realm(self) -> None:
# Check realm with expired messages and messages """Test with a retention policy set for only the MIT realm"""
# with one day to expiration data. expected_mit_msgs = self._make_mit_messages(
expired_mit_messages = self._make_mit_messages(5, timezone_now() - timedelta(days=101)) 5, timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
actual_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=99))
expired_messages_result = list(get_expired_messages()) for realm_instance in Realm.objects.filter(message_retention_days__isnull=False):
expired_mit_messages_ids = [message.id for message in expired_mit_messages] move_expired_messages_to_archive(realm_instance)
expired_mit_messages_result_ids = [message.id for message in move_expired_user_messages_to_archive(realm_instance)
expired_messages_result[0]['expired_messages']]
actual_mit_messages_ids = [message.id for message in actual_mit_messages] self.assertEqual(ArchivedMessage.objects.count(), 5)
self.assertEqual(len(expired_messages_result), 1) self.assertEqual(ArchivedUserMessage.objects.count(), 10)
self.assertEqual(len(expired_messages_result[0]['expired_messages']), 5)
self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id) # Compare expected messages ids with archived messages in mit realm
# Compare expected messages ids with result messages ids. self._check_archive_data_by_realm(expected_mit_msgs, self.mit_realm)
# Check no archive messages for zulip realm.
self.assertEqual( self.assertEqual(
sorted(expired_mit_messages_ids), ArchivedMessage.objects.filter(
expired_mit_messages_result_ids archivedusermessage__user_profile__realm=self.zulip_realm).count(),
0
) )
# Check actual mit.edu messages are not contained in expired messages list
self.assertEqual( self.assertEqual(
set(actual_mit_messages_ids) - set(expired_mit_messages_ids), ArchivedUserMessage.objects.filter(user_profile__realm=self.zulip_realm).count(),
set(actual_mit_messages_ids) 0
) )
def test_cross_realm_messages_archiving_one_realm_expired(self) -> None:
"""Test that a cross-realm message that is expired in only
one of the realms only has the UserMessage for that realm archived"""
arc_msg_id = self._check_cross_realm_messages_archiving(
1, ZULIP_REALM_DAYS + 1, realm=self.zulip_realm)
self.assertTrue(Message.objects.filter(id=arc_msg_id).exists())
def test_cross_realm_messages_archiving_two_realm_expired(self) -> None:
"""Check that archiving a message that's expired in both
realms is archived both in Message and UserMessage."""
arc_msg_id = self._check_cross_realm_messages_archiving(2, MIT_REALM_DAYS + 1)
self.assertFalse(Message.objects.filter(id=arc_msg_id).exists())
def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
expected_message_ids_dict = self._make_expired_messages()
# We also include a cross-realm message in this test.
sent_cross_realm_message_id = self._send_cross_realm_message()
expected_message_ids_dict['mit_msgs_ids'].append(sent_cross_realm_message_id)
self._change_messages_pub_date(
[sent_cross_realm_message_id],
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1)
)
expected_message_ids = expected_message_ids_dict['mit_msgs_ids'] + expected_message_ids_dict['zulip_msgs_ids']
# Get expired user messages by message ids
expected_user_msgs_ids = list(UserMessage.objects.filter(
message_id__in=expected_message_ids).order_by('id').values_list('id', flat=True))
msgs_qty = Message.objects.count()
archive_messages()
# Compare archived messages and user messages with expired messages
self.assertEqual(ArchivedMessage.objects.count(), len(expected_message_ids))
self.assertEqual(ArchivedUserMessage.objects.count(), len(expected_user_msgs_ids))
# Check non-archived messages messages after removing expired
# messages from main tables without cross-realm messages.
self.assertEqual(Message.objects.count(), msgs_qty - ArchivedMessage.objects.count())
self.assertEqual(
Message.objects.filter(id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
self.assertEqual(
Message.objects.filter(id__in=expected_message_ids_dict['mit_msgs_ids']).count(), 0)
self.assertEqual(
Message.objects.filter(id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
# Check archived messages by realm using our standard checker
# function; we add the cross-realm message ID to the
# zulip_realm list for this test because its sender lives in
# that realm in the development environment.
expected_message_ids_dict['zulip_msgs_ids'].append(sent_cross_realm_message_id)
self._check_archived_messages_ids_by_realm(
expected_message_ids_dict['zulip_msgs_ids'], self.zulip_realm)
self._check_archived_messages_ids_by_realm(
expected_message_ids_dict['mit_msgs_ids'], self.mit_realm)
def test_archiving_attachments(self) -> None:
"""End-to-end test for the logic for archiving attachments. This test
is hard to read without first reading _send_messages_with_attachments"""
msgs_ids = self._send_messages_with_attachments()
# First, confirm deleting the oldest message
# (`expired_message_id`) creates ArchivedAttachment objects
# and associates that message ID with them, but does not
# delete the Attachment object.
archive_messages()
archived_attachment = ArchivedAttachment.objects.all()
attachment = Attachment.objects.all()
self.assertEqual(archived_attachment.count(), 3)
self.assertEqual(
list(archived_attachment.distinct('messages__id').values_list('messages__id',
flat=True)),
[msgs_ids['expired_message_id']])
self.assertEqual(attachment.count(), 3)
# Now make `actual_message_id` expired too. We still don't
# delete the Attachment objects.
self._change_messages_pub_date([msgs_ids['actual_message_id']],
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(attachment.count(), 3)
# Finally, make the last message mentioning those attachments
# expired. We should now delete the Attachment objects and
# each ArchivedAttachment object should list all 3 messages.
self._change_messages_pub_date([msgs_ids['other_user_message_id']],
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(attachment.count(), 0)
self.assertEqual(archived_attachment.count(), 3)
self.assertEqual(
list(archived_attachment.distinct('messages__id').order_by('messages__id').values_list(
'messages__id', flat=True)),
sorted(msgs_ids.values()))
class TestMoveMessageToArchive(ZulipTestCase): class TestMoveMessageToArchive(ZulipTestCase):