From 0dca8f2a38fee056338812d2eb60d014247c4022 Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Fri, 3 May 2024 00:52:41 +0200 Subject: [PATCH] models: New PresenceSequence model and UserPresence.last_update_id col. Migration plan: 1. Add NULLable .last_update_id column to UserPresence with default 0 for new objects. 2. Backfill the value to 0 for old UserPresences, can be done in the background while server is running. 3. Make the column non-NULL. 4. Add new model PresenceSequence and create its rows for old realms. --- zerver/actions/create_realm.py | 3 + zerver/lib/server_initialization.py | 3 + .../0525_userpresence_last_update_id.py | 29 +++++++ ...r_presence_backfill_last_update_id_to_0.py | 57 ++++++++++++++ zerver/migrations/0527_presencesequence.py | 76 +++++++++++++++++++ zerver/models/presence.py | 11 +++ 6 files changed, 179 insertions(+) create mode 100644 zerver/migrations/0525_userpresence_last_update_id.py create mode 100644 zerver/migrations/0526_user_presence_backfill_last_update_id_to_0.py create mode 100644 zerver/migrations/0527_presencesequence.py diff --git a/zerver/actions/create_realm.py b/zerver/actions/create_realm.py index 64afafd883..7de5e1000a 100644 --- a/zerver/actions/create_realm.py +++ b/zerver/actions/create_realm.py @@ -35,6 +35,7 @@ from zerver.models import ( Stream, UserProfile, ) +from zerver.models.presence import PresenceSequence from zerver.models.realms import ( CommonPolicyEnum, InviteToRealmPolicyEnum, @@ -287,6 +288,8 @@ def do_create_realm( ] ) + PresenceSequence.objects.create(realm=realm, last_update_id=0) + maybe_enqueue_audit_log_upload(realm) # Create channels once Realm object has been saved diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py index d75e3139e8..7172c3490d 100644 --- a/zerver/lib/server_initialization.py +++ b/zerver/lib/server_initialization.py @@ -13,6 +13,7 @@ from zerver.models import ( UserProfile, ) from zerver.models.clients import get_client +from zerver.models.presence import PresenceSequence from zerver.models.users import get_system_bot from zproject.backends import all_default_backend_names @@ -48,6 +49,8 @@ def create_internal_realm() -> None: ] ) + PresenceSequence.objects.create(realm=realm, last_update_id=0) + # Create some client objects for common requests. Not required; # just ensures these get low IDs in production, and in development # avoids an extra database write for the first HTTP request in diff --git a/zerver/migrations/0525_userpresence_last_update_id.py b/zerver/migrations/0525_userpresence_last_update_id.py new file mode 100644 index 0000000000..348bdf1bf6 --- /dev/null +++ b/zerver/migrations/0525_userpresence_last_update_id.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.5 on 2024-05-02 02:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("zerver", "0524_remove_userprofile_onboarding_steps"), + ] + + operations = [ + # Create a new column, making it NULLable to avoid locking the table + # rewriting rows with a non-NULL default value. + migrations.AddField( + model_name="userpresence", + name="last_update_id", + field=models.PositiveBigIntegerField(db_index=True, null=True), + ), + # This is an SQL noop, since Django doesn't add defaults at database level. + # The default guarantees new rows will have a value. Old rows can get backfilled + # in the next migration. + migrations.AlterField( + model_name="userpresence", + name="last_update_id", + field=models.PositiveBigIntegerField(db_index=True, default=0, null=True), + ), + ] diff --git a/zerver/migrations/0526_user_presence_backfill_last_update_id_to_0.py b/zerver/migrations/0526_user_presence_backfill_last_update_id_to_0.py new file mode 100644 index 0000000000..babf436730 --- /dev/null +++ b/zerver/migrations/0526_user_presence_backfill_last_update_id_to_0.py @@ -0,0 +1,57 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + + +def backfill_user_presence_last_update_id( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + UserPresence = apps.get_model("zerver", "UserPresence") + + max_id = UserPresence.objects.aggregate(models.Max("id"))["id__max"] + if max_id is None: + # Nothing to do if there are no rows yet. + return + + BATCH_SIZE = 10000 + lower_bound = 0 + + # Add a slop factor to make it likely we run past the end in case + # of new rows created while we run. The next step will fail to + # remove the null possibility if we race, so this is safe. + max_id += BATCH_SIZE / 2 + + while lower_bound < max_id: + UserPresence.objects.filter( + id__gt=lower_bound, id__lte=lower_bound + BATCH_SIZE, last_update_id=None + ).update(last_update_id=0) + lower_bound += BATCH_SIZE + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("zerver", "0525_userpresence_last_update_id"), + ] + + operations = [ + migrations.RunPython( + backfill_user_presence_last_update_id, + reverse_code=migrations.RunPython.noop, + elidable=True, + ), + migrations.AlterField( + model_name="userpresence", + name="last_update_id", + field=models.PositiveBigIntegerField(db_index=True, default=0), + ), + AddIndexConcurrently( + model_name="userpresence", + index=models.Index( + fields=["realm", "last_update_id"], + name="zerver_userpresence_realm_last_update_id_idx", + ), + ), + ] diff --git a/zerver/migrations/0527_presencesequence.py b/zerver/migrations/0527_presencesequence.py new file mode 100644 index 0000000000..76a172c06c --- /dev/null +++ b/zerver/migrations/0527_presencesequence.py @@ -0,0 +1,76 @@ +# Generated by Django 5.0.5 on 2024-05-02 22:36 + +import django.db.models.deletion +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + + +def create_presence_sequence_for_old_realms( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + Realm = apps.get_model("zerver", "Realm") + PresenceSequence = apps.get_model("zerver", "PresenceSequence") + + max_id = Realm.objects.aggregate(models.Max("id"))["id__max"] + if max_id is None: + # Nothing to do if there are no rows yet. + return + + BATCH_SIZE = 2000 + lower_bound = 0 + + # Add a slop factor to make it likely we run past the end in case + # of new rows created while we run. Races with realm creation are + # pretty unlikely, and should throw an exception, so we should + # catch them. + max_id += BATCH_SIZE / 2 + + while lower_bound < max_id: + realm_ids = Realm.objects.filter( + id__gt=lower_bound, + id__lte=lower_bound + BATCH_SIZE, + # Filter to realm whose PresenceSequence does not exist, to avoid + # running into IntegrityError by trying to create duplicate PresenceSequence. + presencesequence=None, + ).values_list("id", flat=True) + + PresenceSequence.objects.bulk_create( + PresenceSequence(realm_id=realm_id, last_update_id=0) for realm_id in realm_ids + ) + + lower_bound += BATCH_SIZE + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("zerver", "0526_user_presence_backfill_last_update_id_to_0"), + ] + + operations = [ + migrations.CreateModel( + name="PresenceSequence", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("last_update_id", models.PositiveBigIntegerField()), + ( + "realm", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, to="zerver.realm" + ), + ), + ], + ), + migrations.RunPython( + create_presence_sequence_for_old_realms, + reverse_code=migrations.RunPython.noop, + elidable=True, + ), + ] diff --git a/zerver/models/presence.py b/zerver/models/presence.py index 3a1a47c616..e4adbc5222 100644 --- a/zerver/models/presence.py +++ b/zerver/models/presence.py @@ -30,6 +30,8 @@ class UserPresence(models.Model): # queries to fetch all presence data for a given realm. realm = models.ForeignKey(Realm, on_delete=CASCADE) + last_update_id = models.PositiveBigIntegerField(db_index=True, default=0) + # The last time the user had a client connected to Zulip, # including idle clients where the user hasn't interacted with the # system recently (and thus might be AFK). @@ -58,6 +60,10 @@ class UserPresence(models.Model): fields=["realm", "last_connected_time"], name="zerver_userpresence_realm_id_last_connected_time_98d2fc9f_idx", ), + models.Index( + fields=["realm", "last_update_id"], + name="zerver_userpresence_realm_last_update_id_idx", + ), ] @staticmethod @@ -70,6 +76,11 @@ class UserPresence(models.Model): return None +class PresenceSequence(models.Model): + realm = models.OneToOneField(Realm, on_delete=CASCADE) + last_update_id = models.PositiveBigIntegerField() + + class UserStatus(AbstractEmoji): user_profile = models.OneToOneField(UserProfile, on_delete=CASCADE)