zulip/zerver/tests/test_zulip_update_announcements.py

import os
from datetime import timedelta
from unittest import mock
from unittest.mock import call, patch

import time_machine
from django.conf import settings
from django.utils.timezone import now as timezone_now
from typing_extensions import override

from zerver.actions.create_realm import do_create_realm
from zerver.data_import.mattermost import do_convert_data
from zerver.lib.import_realm import do_import_realm
from zerver.lib.message import remove_single_newlines
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.zulip_update_announcements import (
    ZulipUpdateAnnouncement,
    send_zulip_update_announcements,
)
from zerver.models.messages import Message
from zerver.models.realms import get_realm
from zerver.models.recipients import Recipient, get_direct_message_group_user_ids
from zerver.models.streams import get_stream
from zerver.models.users import get_system_bot


class ZulipUpdateAnnouncementsTest(ZulipTestCase):
    @override
    def setUp(self) -> None:
        super().setUp()

        self.zulip_update_announcements = [
            ZulipUpdateAnnouncement(
                level=1,
                message="Announcement message 1.",
            ),
            ZulipUpdateAnnouncement(
                level=2,
                message="Announcement message 2.",
            ),
        ]

    def test_send_zulip_update_announcements(self) -> None:
        with mock.patch(
            "zerver.lib.zulip_update_announcements.zulip_update_announcements",
            self.zulip_update_announcements,
        ):
            realm = get_realm("zulip")

            # realm predates the "zulip updates" feature with the
            # zulip_update_announcements_stream set to None.
            realm.zulip_update_announcements_level = None
            realm.zulip_update_announcements_stream = None
            realm.save(
                update_fields=[
                    "zulip_update_announcements_level",
                    "zulip_update_announcements_stream",
                ]
            )

            group_direct_messages = Message.objects.filter(
                realm=realm, recipient__type=Recipient.DIRECT_MESSAGE_GROUP
            )
            self.assertFalse(group_direct_messages.exists())

            admin_user_ids = set(realm.get_human_admin_users().values_list("id", flat=True))
            notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
            expected_group_direct_message_user_ids = admin_user_ids | {notification_bot.id}

            now = timezone_now()
            with time_machine.travel(now, tick=False):
                send_zulip_update_announcements(skip_delay=False)

            realm.refresh_from_db()
            group_direct_message = group_direct_messages.first()
            assert group_direct_message is not None
            self.assertEqual(group_direct_message.sender, notification_bot)
            self.assertEqual(group_direct_message.date_sent, now)
            self.assertEqual(
                set(get_direct_message_group_user_ids(group_direct_message.recipient)),
                expected_group_direct_message_user_ids,
            )
            self.assertEqual(realm.zulip_update_announcements_level, 0)
            self.assertIn(
                "These notifications are currently turned off in your organization.",
                group_direct_message.content,
            )

            # Wait for one week before starting to skip sending updates.
            with time_machine.travel(now + timedelta(days=2), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            self.assertEqual(realm.zulip_update_announcements_level, 0)

            with time_machine.travel(now + timedelta(days=8), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            self.assertEqual(realm.zulip_update_announcements_level, 2)

            # Configure a stream. Two new updates added.
            verona = get_stream("verona", realm)
            realm.zulip_update_announcements_stream = verona
            realm.save(update_fields=["zulip_update_announcements_stream"])
            new_updates = [
                ZulipUpdateAnnouncement(
                    level=3,
                    message="Announcement message 3.",
                ),
                ZulipUpdateAnnouncement(
                    level=4,
                    message="Announcement message 4.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            # verify zulip update announcements sent to configured stream.
            with time_machine.travel(now + timedelta(days=10), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            stream_messages = Message.objects.filter(
                realm=realm,
                sender=notification_bot,
                recipient__type_id=verona.id,
                date_sent__gte=now + timedelta(days=10),
            ).order_by("id")
            self.assert_length(stream_messages, 3)
            self.assertIn("To help you learn about new features", stream_messages[0].content)
            self.assertEqual(stream_messages[1].content, "Announcement message 3.")
            self.assertEqual(stream_messages[2].content, "Announcement message 4.")
            self.assertEqual(realm.zulip_update_announcements_level, 4)

    def test_send_zulip_update_announcements_with_stream_configured(self) -> None:
        with mock.patch(
            "zerver.lib.zulip_update_announcements.zulip_update_announcements",
            self.zulip_update_announcements,
        ):
            realm = get_realm("zulip")

            # realm predates the "zulip updates" feature with the
            # zulip_update_announcements_stream configured.
            realm.zulip_update_announcements_level = None
            realm.zulip_update_announcements_stream = get_stream("verona", realm)
            realm.save(
                update_fields=[
                    "zulip_update_announcements_level",
                    "zulip_update_announcements_stream",
                ]
            )

            group_direct_messages = Message.objects.filter(
                realm=realm, recipient__type=Recipient.DIRECT_MESSAGE_GROUP
            )
            self.assertFalse(group_direct_messages.exists())

            now = timezone_now()
            with time_machine.travel(now, tick=False):
                send_zulip_update_announcements(skip_delay=False)

            realm.refresh_from_db()
            self.assertTrue(group_direct_messages.exists())
            self.assertEqual(realm.zulip_update_announcements_level, 0)

            # Wait for 24 hours before starting to send updates.
            with time_machine.travel(now + timedelta(hours=10), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            self.assertEqual(realm.zulip_update_announcements_level, 0)

            with time_machine.travel(now + timedelta(days=1), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            self.assertEqual(realm.zulip_update_announcements_level, 2)

            # Two new updates added.
            new_updates = [
                ZulipUpdateAnnouncement(
                    level=3,
                    message="Announcement message 3.",
                ),
                ZulipUpdateAnnouncement(
                    level=4,
                    message="Announcement message 4.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            # verify zulip update announcements sent to configured stream.
            with time_machine.travel(now + timedelta(days=2), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            realm.refresh_from_db()
            notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
            stream_messages = Message.objects.filter(
                realm=realm,
                sender=notification_bot,
                recipient__type_id=realm.zulip_update_announcements_stream.id,
                date_sent__gte=now + timedelta(days=2),
            ).order_by("id")
            self.assert_length(stream_messages, 2)
            self.assertEqual(stream_messages[0].content, "Announcement message 3.")
            self.assertEqual(stream_messages[1].content, "Announcement message 4.")
            self.assertEqual(realm.zulip_update_announcements_level, 4)

    def test_send_zulip_update_announcements_skip_delay(self) -> None:
        with mock.patch(
            "zerver.lib.zulip_update_announcements.zulip_update_announcements",
            self.zulip_update_announcements,
        ):
            realm = get_realm("zulip")

            # realm predates the "zulip updates" feature with the
            # zulip_update_announcements_stream configured.
            realm.zulip_update_announcements_level = None
            realm.zulip_update_announcements_stream = get_stream("verona", realm)
            realm.save(
                update_fields=[
                    "zulip_update_announcements_level",
                    "zulip_update_announcements_stream",
                ]
            )

            group_direct_messages = Message.objects.filter(
                realm=realm, recipient__type=Recipient.DIRECT_MESSAGE_GROUP
            )
            self.assertFalse(group_direct_messages.exists())

            # post-upgrade hook sends a group DM.
            now = timezone_now()
            with time_machine.travel(now, tick=False):
                send_zulip_update_announcements(skip_delay=False)

            realm.refresh_from_db()
            self.assertTrue(group_direct_messages.exists())
            self.assertEqual(realm.zulip_update_announcements_level, 0)

            # For self-hosted servers, 9.0 upgrade notes suggests to run
            # 'send_zulip_update_announcements' management command with
            # '--skip-delay' argument to immediately send update messages.
            # 'zulip_update_announcements_stream' should be configured.
            with time_machine.travel(now, tick=False):
                send_zulip_update_announcements(skip_delay=True)

            realm.refresh_from_db()
            self.assertEqual(realm.zulip_update_announcements_level, 2)

    def test_group_direct_message_with_zulip_updates_stream_set(self) -> None:
        realm = get_realm("zulip")

        # realm predates the "zulip updates" feature.
        realm.zulip_update_announcements_level = None
        realm.save(update_fields=["zulip_update_announcements_level"])

        self.assertIsNotNone(realm.zulip_update_announcements_stream)

        group_direct_messages = Message.objects.filter(
            realm=realm, recipient__type=Recipient.DIRECT_MESSAGE_GROUP
        )
        self.assertFalse(group_direct_messages.exists())

        admin_user_ids = set(realm.get_human_admin_users().values_list("id", flat=True))
        notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
        expected_group_direct_message_user_ids = admin_user_ids | {notification_bot.id}

        now = timezone_now()
        with time_machine.travel(now, tick=False):
            send_zulip_update_announcements(skip_delay=False)

        realm.refresh_from_db()
        group_direct_message = group_direct_messages.first()
        assert group_direct_message is not None
        self.assertEqual(group_direct_message.sender, notification_bot)
        self.assertEqual(group_direct_message.date_sent, now)
        self.assertEqual(
            set(get_direct_message_group_user_ids(group_direct_message.recipient)),
            expected_group_direct_message_user_ids,
        )
        self.assertEqual(realm.zulip_update_announcements_level, 0)
        self.assertIn(
            "Starting tomorrow, users in your organization will receive "
            "[updates](/help/configure-automated-notices#zulip-update-announcements) about new Zulip features in "
            f"#**{realm.zulip_update_announcements_stream}>{realm.ZULIP_UPDATE_ANNOUNCEMENTS_TOPIC_NAME}**",
            group_direct_message.content,
        )

    def test_remove_single_newlines(self) -> None:
        # single newlines and double newlines
        input_text = "This is a sentence.\nThis is another sentence.\n\nThis is a third sentence."
        expected_output = (
            "This is a sentence. This is another sentence.\n\nThis is a third sentence."
        )
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # single newline at the beginning
        input_text = "\nThis is a sentence.\nThis is another sentence.\n\nThis is a third sentence."
        expected_output = (
            "This is a sentence. This is another sentence.\n\nThis is a third sentence."
        )
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # single newline at the end
        input_text = "This is a sentence.\nThis is another sentence.\n\nThis is a third sentence.\n"
        expected_output = (
            "This is a sentence. This is another sentence.\n\nThis is a third sentence."
        )
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # only single newlines in the middle
        input_text = "This is a sentence.\nThis is another sentence.\nThis is a third sentence.\nThis is a fourth sentence."
        expected_output = "This is a sentence. This is another sentence. This is a third sentence. This is a fourth sentence."
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # Bulleted lists on lines.
        input_text = "- This is a bullet.\n- This is another bullet.\n\n1. This is a list\n1. This is more list."
        expected_output = "- This is a bullet.\n- This is another bullet.\n\n1. This is a list\n1. This is more list."
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # Asterisks after newline for bold.
        input_text = "* This is a bullet.\n**word in bold** on the same line.\n* Another bullet."
        expected_output = (
            "* This is a bullet. **word in bold** on the same line.\n* Another bullet."
        )
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # Digit after newline.
        input_text = "1. This is a numbered list.\n2. Second list element.\n3.5 is a decimal.\n3. Third list element."
        expected_output = "1. This is a numbered list.\n2. Second list element. 3.5 is a decimal.\n3. Third list element."
        self.assertEqual(remove_single_newlines(input_text), expected_output)

        # Hyphen after newline.
        input_text = "- This is a list.\n-C-C- organic molecule structure.\n- Another list element."
        expected_output = (
            "- This is a list. -C-C- organic molecule structure.\n- Another list element."
        )
        self.assertEqual(remove_single_newlines(input_text), expected_output)

    def test_zulip_updates_for_realm_imported_from_other_product(self) -> None:
        with mock.patch(
            "zerver.lib.zulip_update_announcements.zulip_update_announcements",
            self.zulip_update_announcements,
        ):
            mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
            output_dir = self.make_import_output_dir("mattermost")

            with patch("builtins.print") as mock_print, self.assertLogs(level="WARNING"):
                do_convert_data(
                    mattermost_data_dir=mattermost_data_dir,
                    output_dir=output_dir,
                    masking_content=True,
                )
            self.assertEqual(
                mock_print.mock_calls,
                [
                    call("Generating data for", "gryffindor"),
                    call("Generating data for", "slytherin"),
                ],
            )

            gryffindor_output_dir = os.path.join(output_dir, "gryffindor")

            with self.assertLogs(level="INFO"):
                do_import_realm(
                    import_dir=gryffindor_output_dir,
                    subdomain="gryffindor",
                )

            imported_realm = get_realm("gryffindor")
            notification_bot = get_system_bot(settings.NOTIFICATION_BOT, imported_realm.id)

            # Verify for realm imported from other product:
            # * zulip_update_announcements_level = latest level
            # * zulip_update_announcements_stream = None
            # * group DM sent to admins suggesting to set the stream.
            self.assertEqual(imported_realm.zulip_update_announcements_level, 2)
            self.assertIsNone(imported_realm.zulip_update_announcements_stream)
            group_direct_message = Message.objects.filter(
                realm=imported_realm, sender=notification_bot
            ).first()
            assert group_direct_message is not None
            self.assertIn(
                "These notifications are currently turned off in your organization. "
                "If you configure a stream within one week, your organization will not miss any update messages.",
                group_direct_message.content,
            )

            # Two new updates added.
            new_updates = [
                ZulipUpdateAnnouncement(
                    level=3,
                    message="Announcement message 3.",
                ),
                ZulipUpdateAnnouncement(
                    level=4,
                    message="Announcement message 4.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            # Wait for one week before starting to skip sending updates.
            now = timezone_now()
            with time_machine.travel(now + timedelta(days=6), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            imported_realm.refresh_from_db()
            self.assertEqual(imported_realm.zulip_update_announcements_level, 2)

            # No stream configured. Skip updates.
            with time_machine.travel(now + timedelta(days=8), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            imported_realm.refresh_from_db()
            self.assertEqual(imported_realm.zulip_update_announcements_level, 4)
            zulip_updates_message_query = Message.objects.filter(
                realm=imported_realm,
                sender=notification_bot,
                recipient__type=Recipient.STREAM,
            )
            self.assertFalse(zulip_updates_message_query.exists())

            new_updates = [
                ZulipUpdateAnnouncement(
                    level=5,
                    message="Announcement message 5.",
                ),
                ZulipUpdateAnnouncement(
                    level=6,
                    message="Announcement message 6.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            # Stream configured, send update messages.
            imported_realm.zulip_update_announcements_stream = get_stream(
                "Gryffindor common room", imported_realm
            )
            imported_realm.save()

            with time_machine.travel(now + timedelta(days=10), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            imported_realm.refresh_from_db()
            self.assertEqual(imported_realm.zulip_update_announcements_level, 6)
            self.assertTrue(zulip_updates_message_query.exists())

    def test_send_zulip_update_announcements_new_realm(self) -> None:
        with mock.patch(
            "zerver.lib.zulip_update_announcements.zulip_update_announcements",
            self.zulip_update_announcements,
        ):
            new_realm = do_create_realm(string_id="new_realm", name="new_realm")
            # New realm created is on the latest level.
            self.assertEqual(new_realm.zulip_update_announcements_level, 2)
            assert new_realm.zulip_update_announcements_stream is not None

            # New update added.
            new_updates = [
                ZulipUpdateAnnouncement(
                    level=3,
                    message="Announcement message 3.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            now = timezone_now()
            with time_machine.travel(now, tick=False):
                send_zulip_update_announcements(skip_delay=False)
            new_realm.refresh_from_db()
            notification_bot = get_system_bot(settings.NOTIFICATION_BOT, new_realm.id)
            stream_messages = Message.objects.filter(
                realm=new_realm,
                sender=notification_bot,
                recipient__type_id=new_realm.zulip_update_announcements_stream.id,
                date_sent__gte=now,
            ).order_by("id")
            # Verify introductory message + update message sent.
            self.assert_length(stream_messages, 2)
            self.assertIn("To help you learn about new features", stream_messages[0].content)
            self.assertEqual(stream_messages[1].content, "Announcement message 3.")
            self.assertEqual(new_realm.zulip_update_announcements_level, 3)

            new_realm.zulip_update_announcements_stream = None
            new_realm.save(update_fields=["zulip_update_announcements_stream"])

            # Another new update added.
            new_updates = [
                ZulipUpdateAnnouncement(
                    level=4,
                    message="Announcement message 4.",
                ),
            ]
            self.zulip_update_announcements.extend(new_updates)

            # Verify update message missed as stream was manually set to None.
            with time_machine.travel(now + timedelta(days=1), tick=False):
                send_zulip_update_announcements(skip_delay=False)
            new_realm.refresh_from_db()
            stream_messages = Message.objects.filter(
                realm=new_realm,
                sender=notification_bot,
                date_sent__gte=now + timedelta(days=1),
            ).order_by("id")
            self.assert_length(stream_messages, 0)
            self.assertEqual(new_realm.zulip_update_announcements_level, 4)