zulip_updates: Send zulip updates based on zulip_update_*_level.

This commit adds a management command that will run regularly
as a cron job to send zulip updates to realms based on their
current and latest zulip_update_announcements_level.

For realms with:
* level = None: Send a group DM to admins notifying them about
this new feature & suggestion to set the stream accordingly.

* level = 0:
  * If stream is still not configured, wait for a week
    before setting their level to latest level. They will
    miss updates until their configure the stream.
  * If stream is configured, send updates.

* level > 0: Send one message/update per level & increase
  the level by 1 till the latest level.

Fixes #28604.

zerver/actions/create_realm.py

@@ -22,6 +22,7 @@ from zerver.lib.user_groups import (
     create_system_user_groups_for_realm,
     get_role_based_system_groups_dict,
 )
+from zerver.lib.zulip_update_announcements import get_latest_zulip_update_announcements_level
 from zerver.models import (
     DefaultStream,
     PreregistrationRealm,
@@ -294,11 +295,16 @@ def do_create_realm(
     )
     realm.signup_announcements_stream = signup_announcements_stream
 
+    # New realm is initialized with the latest zulip update announcements
+    # level as it shouldn't receive a bunch of old updates.
+    realm.zulip_update_announcements_level = get_latest_zulip_update_announcements_level()
+
     realm.save(
         update_fields=[
             "new_stream_announcements_stream",
             "signup_announcements_stream",
             "zulip_update_announcements_stream",
+            "zulip_update_announcements_level",
         ]
     )
 

zerver/actions/message_send.py

@@ -1978,9 +1978,15 @@ def internal_prep_huddle_message(
     realm: Realm,
     sender: UserProfile,
     content: str,
-    emails: List[str],
+    *,
+    emails: Optional[List[str]] = None,
+    recipient_users: Optional[List[UserProfile]] = None,
 ) -> Optional[SendMessageRequest]:
-    addressee = Addressee.for_private(emails, realm)
+    if recipient_users is not None:
+        addressee = Addressee.for_user_profiles(recipient_users)
+    else:
+        assert emails is not None
+        addressee = Addressee.for_private(emails, realm)
 
     return _internal_prep_message(
         realm=realm,
@@ -1993,7 +1999,7 @@ def internal_prep_huddle_message(
 def internal_send_huddle_message(
     realm: Realm, sender: UserProfile, emails: List[str], content: str
 ) -> Optional[int]:
-    message = internal_prep_huddle_message(realm, sender, content, emails)
+    message = internal_prep_huddle_message(realm, sender, content, emails=emails)
 
     if message is None:
         return None

zerver/lib/addressee.py

@@ -197,3 +197,11 @@ class Addressee:
             msg_type="private",
             user_profiles=user_profiles,
         )
+
+    @staticmethod
+    def for_user_profiles(user_profiles: Sequence[UserProfile]) -> "Addressee":
+        assert len(user_profiles) > 0
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )

zerver/lib/onboarding.py

@@ -314,6 +314,21 @@ def send_initial_realm_messages(realm: Realm) -> None:
             start_topic_help_url="/help/starting-a-new-topic",
         )
 
+        content_of_zulip_update_announcements_topic_name = (
+            _("""
+Welcome! To help you learn about new features and configuration options, \
+this topic will receive messages about important changes in Zulip.
+
+You can read these update messages whenever it's convenient, or \
+[mute]({mute_topic_help_url}) this topic if you are not interested. \
+If your organization does not want to receive these announcements, \
+they can be disabled. [Learn more]({zulip_update_announcements_help_url}).
+            """)
+        ).format(
+            zulip_update_announcements_help_url="/help/configure-automated-notices#zulip-update-announcements",
+            mute_topic_help_url="/help/mute-a-topic",
+        )
+
     welcome_messages: List[Dict[str, str]] = [
         {
             "stream": Realm.INITIAL_PRIVATE_STREAM_NAME,
@@ -335,6 +350,11 @@ def send_initial_realm_messages(realm: Realm) -> None:
             "topic_name": "swimming turtles",
             "content": content_of_swimming_turtles_topic_name,
         },
+        {
+            "stream": Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
+            "topic_name": str(Realm.ZULIP_UPDATE_ANNOUNCEMENTS_TOPIC_NAME),
+            "content": content_of_zulip_update_announcements_topic_name,
+        },
     ]
 
     messages = [

zerver/lib/zulip_update_announcements.py

@@ -0,0 +1,211 @@
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import List, Optional
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Q, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import override as override_language
+
+from zerver.actions.message_send import (
+    do_send_messages,
+    internal_prep_huddle_message,
+    internal_prep_stream_message,
+)
+from zerver.lib.message import SendMessageRequest
+from zerver.models.realm_audit_logs import RealmAuditLog
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile, get_system_bot
+
+
+@dataclass
+class ZulipUpdateAnnouncement:
+    level: int
+    message: str
+
+
+# We don't translate the announcement message because they are quite unlikely to be
+# translated during the time between when we draft them and when they are published.
+zulip_update_announcements: List[ZulipUpdateAnnouncement] = [
+    ZulipUpdateAnnouncement(
+        level=1,
+        message="""\
+Zulip is introducing **Zulip updates**! To help you learn about new features and \
+configuration options, this topic will receive messages about important changes in Zulip.
+
+You can read these update messages whenever it's convenient, or [mute]({mute_topic_help_url}) \
+this topic if you are not interested. If your organization does not want to receive these \
+announcements, they can be disabled. [Learn more]({zulip_update_announcements_help_url}).
+""".format(
+            zulip_update_announcements_help_url="/help/configure-automated-notices#zulip-update-announcements",
+            mute_topic_help_url="/help/mute-a-topic",
+        ),
+    ),
+]
+
+
+def get_latest_zulip_update_announcements_level() -> int:
+    latest_zulip_update_announcement = zulip_update_announcements[-1]
+    return latest_zulip_update_announcement.level
+
+
+def get_zulip_update_announcements_message_for_level(level: int) -> str:
+    zulip_update_announcement = zulip_update_announcements[level - 1]
+    return zulip_update_announcement.message
+
+
+def get_realms_behind_zulip_update_announcements_level(level: int) -> QuerySet[Realm]:
+    # Filter out deactivated realms. When a realm is later
+    # reactivated, send the notices it missed while it was deactivated.
+    realms = Realm.objects.filter(
+        Q(zulip_update_announcements_level__isnull=True)
+        | Q(zulip_update_announcements_level__lt=level),
+        deactivated=False,
+    ).exclude(string_id=settings.SYSTEM_BOT_REALM)
+    return realms
+
+
+def internal_prep_group_direct_message_for_old_realm(
+    realm: Realm, sender: UserProfile
+) -> Optional[SendMessageRequest]:
+    administrators = list(realm.get_human_admin_users())
+    with override_language(realm.default_language):
+        topic_name = str(realm.ZULIP_UPDATE_ANNOUNCEMENTS_TOPIC_NAME)
+    if realm.zulip_update_announcements_stream is None:
+        content = """\
+You can now [configure]({organization_settings_url}) a stream where Zulip \
+will send [updates]({zulip_update_announcements_help_url}) about new Zulip features. \
+These notifications are currently turned off in your organization.
+""".format(
+            zulip_update_announcements_help_url="/help/configure-automated-notices#zulip-update-announcements",
+            organization_settings_url="/#organization/organization-settings",
+        )
+    else:
+        content = """\
+Users in your organization will now receive [updates]({zulip_update_announcements_help_url}) \
+about new Zulip features in #**{zulip_update_announcements_stream}>{topic_name}**.
+
+If you like, you can [configure]({organization_settings_url}) a different stream for \
+these updates (and [move]({move_content_another_stream_help_url}) the initial updates there), \
+or [turn this feature off]({organization_settings_url}) altogether.
+""".format(
+            zulip_update_announcements_help_url="/help/configure-automated-notices#zulip-update-announcements",
+            zulip_update_announcements_stream=realm.zulip_update_announcements_stream.name,
+            topic_name=topic_name,
+            organization_settings_url="/#organization/organization-settings",
+            move_content_another_stream_help_url="/help/move-content-to-another-stream",
+        )
+    return internal_prep_huddle_message(realm, sender, content, recipient_users=administrators)
+
+
+def is_group_direct_message_sent_to_admins_atleast_one_week_ago(realm: Realm) -> bool:
+    level_none_to_zero_auditlog = RealmAuditLog.objects.filter(
+        realm=realm,
+        event_type=RealmAuditLog.REALM_PROPERTY_CHANGED,
+        extra_data__contains={
+            RealmAuditLog.OLD_VALUE: None,
+            RealmAuditLog.NEW_VALUE: 0,
+            "property": "zulip_update_announcements_level",
+        },
+    ).first()
+    assert level_none_to_zero_auditlog is not None
+    group_direct_message_sent_on = level_none_to_zero_auditlog.event_time
+    return timezone_now() - group_direct_message_sent_on > timedelta(days=7)
+
+
+def internal_prep_zulip_update_announcements_stream_messages(
+    current_level: int, latest_level: int, sender: UserProfile, realm: Realm
+) -> List[Optional[SendMessageRequest]]:
+    message_requests = []
+    stream = realm.zulip_update_announcements_stream
+    assert stream is not None
+    with override_language(realm.default_language):
+        topic_name = str(realm.ZULIP_UPDATE_ANNOUNCEMENTS_TOPIC_NAME)
+    while current_level < latest_level:
+        content = get_zulip_update_announcements_message_for_level(level=current_level + 1)
+        message_requests.append(
+            internal_prep_stream_message(
+                sender,
+                stream,
+                topic_name,
+                content,
+            )
+        )
+        current_level += 1
+    return message_requests
+
+
+def send_messages_and_update_level(
+    realm: Realm,
+    new_zulip_update_announcements_level: int,
+    send_message_requests: List[Optional[SendMessageRequest]],
+) -> None:
+    sent_message_ids = []
+    if send_message_requests:
+        sent_messages = do_send_messages(send_message_requests)
+        sent_message_ids = [sent_message.message_id for sent_message in sent_messages]
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=RealmAuditLog.REALM_PROPERTY_CHANGED,
+        event_time=timezone_now(),
+        extra_data={
+            RealmAuditLog.OLD_VALUE: realm.zulip_update_announcements_level,
+            RealmAuditLog.NEW_VALUE: new_zulip_update_announcements_level,
+            "property": "zulip_update_announcements_level",
+            "zulip_update_announcements_message_ids": sent_message_ids,
+        },
+    )
+
+    realm.zulip_update_announcements_level = new_zulip_update_announcements_level
+    realm.save(update_fields=["zulip_update_announcements_level"])
+
+
+def send_zulip_update_announcements() -> None:
+    latest_zulip_update_announcements_level = get_latest_zulip_update_announcements_level()
+
+    realms = get_realms_behind_zulip_update_announcements_level(
+        level=latest_zulip_update_announcements_level
+    )
+
+    for realm in realms:
+        realm_zulip_update_announcements_level = realm.zulip_update_announcements_level
+        sender = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+
+        messages = []
+        new_zulip_update_announcements_level = None
+        with transaction.atomic(savepoint=False):
+            if realm_zulip_update_announcements_level is None:
+                # realm predates the zulip update announcements feature.
+                # Group DM the administrators to set or verify the stream for
+                # zulip update announcements.
+                group_direct_message = internal_prep_group_direct_message_for_old_realm(
+                    realm, sender
+                )
+                messages = [group_direct_message]
+                new_zulip_update_announcements_level = 0
+            elif (
+                realm_zulip_update_announcements_level == 0
+                and realm.zulip_update_announcements_stream is None
+            ):
+                # We wait for a week after sending group DMs to let admins configure
+                # stream for zulip update announcements. After that, they miss updates
+                # until they don't configure.
+                if is_group_direct_message_sent_to_admins_atleast_one_week_ago(realm):
+                    new_zulip_update_announcements_level = latest_zulip_update_announcements_level
+            else:
+                if realm.zulip_update_announcements_stream is not None:
+                    messages = internal_prep_zulip_update_announcements_stream_messages(
+                        current_level=realm_zulip_update_announcements_level,
+                        latest_level=latest_zulip_update_announcements_level,
+                        sender=sender,
+                        realm=realm,
+                    )
+
+                new_zulip_update_announcements_level = latest_zulip_update_announcements_level
+
+            if new_zulip_update_announcements_level is not None:
+                send_messages_and_update_level(
+                    realm, new_zulip_update_announcements_level, messages
+                )

zerver/management/commands/send_zulip_update_announcements.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.zulip_update_announcements import send_zulip_update_announcements
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to send zulip update announcements to realms."""
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        send_zulip_update_announcements()

zerver/migrations/0503_realm_zulip_update_announcements_level.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.9 on 2024-02-13 06:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0502_merge_20240319_2236"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="realm",
+            name="zulip_update_announcements_level",
+            field=models.PositiveIntegerField(null=True),
+        ),
+    ]

zerver/models/realms.py

@@ -359,6 +359,8 @@ class Realm(models.Model):  # type: ignore[django-manager-missing] # django-stub
         on_delete=models.SET_NULL,
     )
 
+    zulip_update_announcements_level = models.PositiveIntegerField(null=True)
+
     MESSAGE_RETENTION_SPECIAL_VALUES_MAP = {
         "unlimited": -1,
     }

zerver/tests/test_signup.py

@@ -1302,7 +1302,7 @@ class RealmCreationTest(ZulipTestCase):
 
         # Check welcome messages
         for stream_name, text, message_count in [
-            (Realm.DEFAULT_NOTIFICATION_STREAM_NAME, "with the topic", 3),
+            (Realm.DEFAULT_NOTIFICATION_STREAM_NAME, "with the topic", 4),
             (Realm.INITIAL_PRIVATE_STREAM_NAME, "private stream", 1),
         ]:
             stream = get_stream(stream_name, realm)
@@ -1741,7 +1741,7 @@ class RealmCreationTest(ZulipTestCase):
         private_stream_in_italian = "canale privato"
 
         for stream_name, text, message_count in [
-            (Realm.DEFAULT_NOTIFICATION_STREAM_NAME, with_the_topic_in_italian, 3),
+            (Realm.DEFAULT_NOTIFICATION_STREAM_NAME, with_the_topic_in_italian, 4),
             (Realm.INITIAL_PRIVATE_STREAM_NAME, private_stream_in_italian, 1),
         ]:
             stream = get_stream(stream_name, realm)

zerver/tests/test_zulip_update_announcements.py

@@ -0,0 +1,154 @@
+from datetime import timedelta
+from unittest import mock
+
+import time_machine
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.zulip_update_announcements import (
+    ZulipUpdateAnnouncement,
+    send_zulip_update_announcements,
+)
+from zerver.models.messages import Message
+from zerver.models.realms import get_realm
+from zerver.models.recipients import Recipient, get_huddle_user_ids
+from zerver.models.streams import get_stream
+from zerver.models.users import get_system_bot
+
+test_zulip_update_announcements = [
+    ZulipUpdateAnnouncement(
+        level=1,
+        message="Announcement message 1.",
+    ),
+    ZulipUpdateAnnouncement(
+        level=2,
+        message="Announcement message 2.",
+    ),
+]
+
+
+class ZulipUpdateAnnouncementsTest(ZulipTestCase):
+    @mock.patch(
+        "zerver.lib.zulip_update_announcements.zulip_update_announcements",
+        test_zulip_update_announcements,
+    )
+    def test_send_zulip_update_announcements(self) -> None:
+        realm = get_realm("zulip")
+
+        # realm predates the "zulip updates" feature with the
+        # zulip_update_announcements_stream set to None.
+        realm.zulip_update_announcements_level = None
+        realm.zulip_update_announcements_stream = None
+        realm.save(
+            update_fields=["zulip_update_announcements_level", "zulip_update_announcements_stream"]
+        )
+
+        group_direct_messages = Message.objects.filter(
+            realm=realm, recipient__type=Recipient.HUDDLE
+        )
+        self.assertFalse(group_direct_messages.exists())
+
+        admin_user_ids = set(realm.get_human_admin_users().values_list("id", flat=True))
+        notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+        expected_group_direct_message_user_ids = admin_user_ids | {notification_bot.id}
+
+        now = timezone_now()
+        with time_machine.travel(now, tick=False):
+            send_zulip_update_announcements()
+
+        realm.refresh_from_db()
+        group_direct_message = group_direct_messages.first()
+        assert group_direct_message is not None
+        self.assertEqual(group_direct_message.sender, notification_bot)
+        self.assertEqual(group_direct_message.date_sent, now)
+        self.assertEqual(
+            set(get_huddle_user_ids(group_direct_message.recipient)),
+            expected_group_direct_message_user_ids,
+        )
+        self.assertEqual(realm.zulip_update_announcements_level, 0)
+        self.assertIn(
+            "These notifications are currently turned off in your organization.",
+            group_direct_message.content,
+        )
+
+        # Wait for one week before starting to skip sending updates.
+        with time_machine.travel(now + timedelta(days=2), tick=False):
+            send_zulip_update_announcements()
+        realm.refresh_from_db()
+        self.assertEqual(realm.zulip_update_announcements_level, 0)
+
+        with time_machine.travel(now + timedelta(days=8), tick=False):
+            send_zulip_update_announcements()
+        realm.refresh_from_db()
+        self.assertEqual(realm.zulip_update_announcements_level, 2)
+
+        # Configure a stream. Two new updates added.
+        verona = get_stream("verona", realm)
+        realm.zulip_update_announcements_stream = verona
+        realm.save(update_fields=["zulip_update_announcements_stream"])
+        new_updates = [
+            ZulipUpdateAnnouncement(
+                level=3,
+                message="Announcement message 3.",
+            ),
+            ZulipUpdateAnnouncement(
+                level=4,
+                message="Announcement message 4.",
+            ),
+        ]
+        test_zulip_update_announcements.extend(new_updates)
+
+        # verify zulip update announcements sent to configured stream.
+        with time_machine.travel(now + timedelta(days=10), tick=False):
+            send_zulip_update_announcements()
+        realm.refresh_from_db()
+        stream_messages = Message.objects.filter(
+            realm=realm,
+            sender=notification_bot,
+            recipient__type_id=verona.id,
+            date_sent__gte=now + timedelta(days=10),
+        ).order_by("id")
+        self.assert_length(stream_messages, 2)
+        self.assertEqual(stream_messages[0].content, "Announcement message 3.")
+        self.assertEqual(stream_messages[1].content, "Announcement message 4.")
+        self.assertEqual(realm.zulip_update_announcements_level, 4)
+
+    def test_group_direct_message_with_zulip_updates_stream_set(self) -> None:
+        realm = get_realm("zulip")
+
+        # realm predates the "zulip updates" feature.
+        realm.zulip_update_announcements_level = None
+        realm.save(update_fields=["zulip_update_announcements_level"])
+
+        self.assertIsNotNone(realm.zulip_update_announcements_stream)
+
+        group_direct_messages = Message.objects.filter(
+            realm=realm, recipient__type=Recipient.HUDDLE
+        )
+        self.assertFalse(group_direct_messages.exists())
+
+        admin_user_ids = set(realm.get_human_admin_users().values_list("id", flat=True))
+        notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+        expected_group_direct_message_user_ids = admin_user_ids | {notification_bot.id}
+
+        now = timezone_now()
+        with time_machine.travel(now, tick=False):
+            send_zulip_update_announcements()
+
+        realm.refresh_from_db()
+        group_direct_message = group_direct_messages.first()
+        assert group_direct_message is not None
+        self.assertEqual(group_direct_message.sender, notification_bot)
+        self.assertEqual(group_direct_message.date_sent, now)
+        self.assertEqual(
+            set(get_huddle_user_ids(group_direct_message.recipient)),
+            expected_group_direct_message_user_ids,
+        )
+        self.assertEqual(realm.zulip_update_announcements_level, 0)
+        self.assertIn(
+            "Users in your organization will now receive "
+            "[updates](/help/configure-automated-notices#zulip-update-announcements) about new Zulip features in "
+            f"#**{realm.zulip_update_announcements_stream}>{realm.ZULIP_UPDATE_ANNOUNCEMENTS_TOPIC_NAME}**",
+            group_direct_message.content,
+        )