From 3afe585922cc9ed185dfc912c43933b266ab5fad Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Thu, 12 Oct 2023 20:10:07 +0200 Subject: [PATCH] realm: Add uuid and associated secret columns. Thisi and the following commit follow the approach used in 3e2ad84bbe7cbc8099a42b6860070cad55cb9b52. First migration requires a server restart - after that any new realms will be created with the columns set. The following migrations are in the next commit: Second migration does a backfill for older realms and can run in the background while the server is operating normally. Third migration enforces null=False now that all realms have the columns set. --- ...0479_realm_uuid_realm_uuid_owner_secret.py | 22 +++++++++++ zerver/models.py | 13 +++++++ zerver/tests/test_import_export.py | 38 +++++++++++++------ 3 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 zerver/migrations/0479_realm_uuid_realm_uuid_owner_secret.py diff --git a/zerver/migrations/0479_realm_uuid_realm_uuid_owner_secret.py b/zerver/migrations/0479_realm_uuid_realm_uuid_owner_secret.py new file mode 100644 index 0000000000..68d27b4dfc --- /dev/null +++ b/zerver/migrations/0479_realm_uuid_realm_uuid_owner_secret.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.5 on 2023-10-06 09:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("zerver", "0478_realm_enable_guest_user_indicator"), + ] + + operations = [ + migrations.AddField( + model_name="realm", + name="uuid", + field=models.UUIDField(null=True, unique=True), + ), + migrations.AddField( + model_name="realm", + name="uuid_owner_secret", + field=models.TextField(null=True), + ), + ] diff --git a/zerver/models.py b/zerver/models.py index 2e3231fb02..0874a9b923 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -275,6 +275,14 @@ class RealmAuthenticationMethod(models.Model): unique_together = ("realm", "name") +def generate_realm_uuid_owner_secret() -> str: + token = generate_api_key() + + # We include a prefix to facilitate scanning for accidental + # disclosure of secrets e.g. in Github commit pushes. + return f"zuliprealm_{token}" + + class Realm(models.Model): # type: ignore[django-manager-missing] # django-stubs cannot resolve the custom CTEManager yet https://github.com/typeddjango/django-stubs/issues/1023 MAX_REALM_NAME_LENGTH = 40 MAX_REALM_DESCRIPTION_LENGTH = 1000 @@ -295,6 +303,11 @@ class Realm(models.Model): # type: ignore[django-manager-missing] # django-stub # at `foo.example.com`. string_id = models.CharField(max_length=MAX_REALM_SUBDOMAIN_LENGTH, unique=True) + # uuid and a secret for the sake of per-realm authentication with the push notification + # bouncer. + uuid = models.UUIDField(default=uuid4, unique=True) + uuid_owner_secret = models.TextField(default=generate_realm_uuid_owner_secret) + date_created = models.DateTimeField(default=timezone_now) demo_organization_scheduled_deletion_date = models.DateTimeField(default=None, null=True) deactivated = models.BooleanField(default=False) diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index af6f24697b..3ad0caef4d 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -1,6 +1,7 @@ import datetime import os import shutil +import uuid from collections import defaultdict from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple from unittest.mock import patch @@ -329,6 +330,13 @@ class RealmImportExportTest(ExportFile): consent_message_id=consent_message_id, public_only=public_only, ) + + # This is a unique field and thus the cycle of export->import + # within the same server (which is what happens in our tests) + # will cause a conflict - so rotate it. + realm.uuid = uuid.uuid4() + realm.save() + export_usermessages_batch( input_path=os.path.join(output_dir, "messages-000001.json.partial"), output_path=os.path.join(output_dir, "messages-000001.json"), @@ -1538,25 +1546,31 @@ class RealmImportExportTest(ExportFile): self.export_realm(realm) with self.settings(BILLING_ENABLED=True), self.assertLogs(level="INFO"): - realm = do_import_realm(get_output_dir(), "test-zulip-1") - self.assertEqual(realm.plan_type, Realm.PLAN_TYPE_LIMITED) - self.assertEqual(realm.max_invites, 100) - self.assertEqual(realm.upload_quota_gb, 5) - self.assertEqual(realm.message_visibility_limit, 10000) + imported_realm = do_import_realm(get_output_dir(), "test-zulip-1") + self.assertEqual(imported_realm.plan_type, Realm.PLAN_TYPE_LIMITED) + self.assertEqual(imported_realm.max_invites, 100) + self.assertEqual(imported_realm.upload_quota_gb, 5) + self.assertEqual(imported_realm.message_visibility_limit, 10000) self.assertTrue( RealmAuditLog.objects.filter( - realm=realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED + realm=imported_realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED ).exists() ) + + # Importing the same export data twice would cause conflict on unique fields, + # so instead re-export the original realm via self.export_realm, which handles + # this issue. + self.export_realm(realm) + with self.settings(BILLING_ENABLED=False), self.assertLogs(level="INFO"): - realm = do_import_realm(get_output_dir(), "test-zulip-2") - self.assertEqual(realm.plan_type, Realm.PLAN_TYPE_SELF_HOSTED) - self.assertEqual(realm.max_invites, 100) - self.assertEqual(realm.upload_quota_gb, None) - self.assertEqual(realm.message_visibility_limit, None) + imported_realm = do_import_realm(get_output_dir(), "test-zulip-2") + self.assertEqual(imported_realm.plan_type, Realm.PLAN_TYPE_SELF_HOSTED) + self.assertEqual(imported_realm.max_invites, 100) + self.assertEqual(imported_realm.upload_quota_gb, None) + self.assertEqual(imported_realm.message_visibility_limit, None) self.assertTrue( RealmAuditLog.objects.filter( - realm=realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED + realm=imported_realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED ).exists() )