models: New PresenceSequence model and UserPresence.last_update_id col.

Migration plan:
1. Add NULLable .last_update_id column to UserPresence with default 0
   for new objects.
2. Backfill the value to 0 for old UserPresences, can be done in the
   background while server is running.
3. Make the column non-NULL.
4. Add new model PresenceSequence and create its rows for old realms.

zerver/actions/create_realm.py

@@ -35,6 +35,7 @@ from zerver.models import (
     Stream,
     UserProfile,
 )
+from zerver.models.presence import PresenceSequence
 from zerver.models.realms import (
     CommonPolicyEnum,
     InviteToRealmPolicyEnum,
@@ -287,6 +288,8 @@ def do_create_realm(
             ]
         )
 
+        PresenceSequence.objects.create(realm=realm, last_update_id=0)
+
         maybe_enqueue_audit_log_upload(realm)
 
     # Create channels once Realm object has been saved

zerver/lib/server_initialization.py

@@ -13,6 +13,7 @@ from zerver.models import (
     UserProfile,
 )
 from zerver.models.clients import get_client
+from zerver.models.presence import PresenceSequence
 from zerver.models.users import get_system_bot
 from zproject.backends import all_default_backend_names
 
@@ -48,6 +49,8 @@ def create_internal_realm() -> None:
         ]
     )
 
+    PresenceSequence.objects.create(realm=realm, last_update_id=0)
+
     # Create some client objects for common requests.  Not required;
     # just ensures these get low IDs in production, and in development
     # avoids an extra database write for the first HTTP request in

zerver/migrations/0525_userpresence_last_update_id.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.0.5 on 2024-05-02 02:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zerver", "0524_remove_userprofile_onboarding_steps"),
+    ]
+
+    operations = [
+        # Create a new column, making it NULLable to avoid locking the table
+        # rewriting rows with a non-NULL default value.
+        migrations.AddField(
+            model_name="userpresence",
+            name="last_update_id",
+            field=models.PositiveBigIntegerField(db_index=True, null=True),
+        ),
+        # This is an SQL noop, since Django doesn't add defaults at database level.
+        # The default guarantees new rows will have a value. Old rows can get backfilled
+        # in the next migration.
+        migrations.AlterField(
+            model_name="userpresence",
+            name="last_update_id",
+            field=models.PositiveBigIntegerField(db_index=True, default=0, null=True),
+        ),
+    ]

zerver/migrations/0526_user_presence_backfill_last_update_id_to_0.py

@@ -0,0 +1,57 @@
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def backfill_user_presence_last_update_id(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    UserPresence = apps.get_model("zerver", "UserPresence")
+
+    max_id = UserPresence.objects.aggregate(models.Max("id"))["id__max"]
+    if max_id is None:
+        # Nothing to do if there are no rows yet.
+        return
+
+    BATCH_SIZE = 10000
+    lower_bound = 0
+
+    # Add a slop factor to make it likely we run past the end in case
+    # of new rows created while we run. The next step will fail to
+    # remove the null possibility if we race, so this is safe.
+    max_id += BATCH_SIZE / 2
+
+    while lower_bound < max_id:
+        UserPresence.objects.filter(
+            id__gt=lower_bound, id__lte=lower_bound + BATCH_SIZE, last_update_id=None
+        ).update(last_update_id=0)
+        lower_bound += BATCH_SIZE
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zerver", "0525_userpresence_last_update_id"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            backfill_user_presence_last_update_id,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+        migrations.AlterField(
+            model_name="userpresence",
+            name="last_update_id",
+            field=models.PositiveBigIntegerField(db_index=True, default=0),
+        ),
+        AddIndexConcurrently(
+            model_name="userpresence",
+            index=models.Index(
+                fields=["realm", "last_update_id"],
+                name="zerver_userpresence_realm_last_update_id_idx",
+            ),
+        ),
+    ]

zerver/migrations/0527_presencesequence.py

@@ -0,0 +1,76 @@
+# Generated by Django 5.0.5 on 2024-05-02 22:36
+
+import django.db.models.deletion
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def create_presence_sequence_for_old_realms(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    Realm = apps.get_model("zerver", "Realm")
+    PresenceSequence = apps.get_model("zerver", "PresenceSequence")
+
+    max_id = Realm.objects.aggregate(models.Max("id"))["id__max"]
+    if max_id is None:
+        # Nothing to do if there are no rows yet.
+        return
+
+    BATCH_SIZE = 2000
+    lower_bound = 0
+
+    # Add a slop factor to make it likely we run past the end in case
+    # of new rows created while we run. Races with realm creation are
+    # pretty unlikely, and should throw an exception, so we should
+    # catch them.
+    max_id += BATCH_SIZE / 2
+
+    while lower_bound < max_id:
+        realm_ids = Realm.objects.filter(
+            id__gt=lower_bound,
+            id__lte=lower_bound + BATCH_SIZE,
+            # Filter to realm whose PresenceSequence does not exist, to avoid
+            # running into IntegrityError by trying to create duplicate PresenceSequence.
+            presencesequence=None,
+        ).values_list("id", flat=True)
+
+        PresenceSequence.objects.bulk_create(
+            PresenceSequence(realm_id=realm_id, last_update_id=0) for realm_id in realm_ids
+        )
+
+        lower_bound += BATCH_SIZE
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zerver", "0526_user_presence_backfill_last_update_id_to_0"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PresenceSequence",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("last_update_id", models.PositiveBigIntegerField()),
+                (
+                    "realm",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+            ],
+        ),
+        migrations.RunPython(
+            create_presence_sequence_for_old_realms,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+    ]

zerver/models/presence.py

@@ -30,6 +30,8 @@ class UserPresence(models.Model):
     # queries to fetch all presence data for a given realm.
     realm = models.ForeignKey(Realm, on_delete=CASCADE)
 
+    last_update_id = models.PositiveBigIntegerField(db_index=True, default=0)
+
     # The last time the user had a client connected to Zulip,
     # including idle clients where the user hasn't interacted with the
     # system recently (and thus might be AFK).
@@ -58,6 +60,10 @@ class UserPresence(models.Model):
                 fields=["realm", "last_connected_time"],
                 name="zerver_userpresence_realm_id_last_connected_time_98d2fc9f_idx",
             ),
+            models.Index(
+                fields=["realm", "last_update_id"],
+                name="zerver_userpresence_realm_last_update_id_idx",
+            ),
         ]
 
     @staticmethod
@@ -70,6 +76,11 @@ class UserPresence(models.Model):
         return None
 
 
+class PresenceSequence(models.Model):
+    realm = models.OneToOneField(Realm, on_delete=CASCADE)
+    last_update_id = models.PositiveBigIntegerField()
+
+
 class UserStatus(AbstractEmoji):
     user_profile = models.OneToOneField(UserProfile, on_delete=CASCADE)