diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index db5cfbbbbc..5aad3419b3 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -13,7 +13,7 @@ from version import ZULIP_VERSION from zerver.lib.exceptions import JsonableError from zerver.lib.export import floatify_datetime_fields from zerver.lib.outgoing_http import OutgoingSession -from zerver.models import RealmAuditLog +from zerver.models import Realm, RealmAuditLog class PushBouncerSession(OutgoingSession): @@ -172,6 +172,24 @@ def build_analytics_data( ) +def get_realms_info_for_push_bouncer() -> List[Dict[str, Any]]: + realms = Realm.objects.order_by("id") + realm_info_dicts = [ + dict( + id=realm.id, + uuid=str(realm.uuid), + uuid_owner_secret=realm.uuid_owner_secret, + host=realm.host, + url=realm.uri, + deactivated=realm.deactivated, + date_created=realm.date_created.timestamp(), + ) + for realm in realms + ] + + return realm_info_dicts + + def send_analytics_to_push_bouncer() -> None: # first, check what's latest try: @@ -203,6 +221,7 @@ def send_analytics_to_push_bouncer() -> None: "realm_counts": orjson.dumps(realm_count_data).decode(), "installation_counts": orjson.dumps(installation_count_data).decode(), "realmauditlog_rows": orjson.dumps(realmauditlog_data).decode(), + "realms": orjson.dumps(get_realms_info_for_push_bouncer()).decode(), "version": orjson.dumps(ZULIP_VERSION).decode(), } diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index af37be1b0c..0be27d9588 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -70,6 +70,7 @@ from zerver.models import ( Message, NotificationTriggers, PushDeviceToken, + Realm, RealmAuditLog, Recipient, Stream, @@ -87,6 +88,7 @@ if settings.ZILENCER_ENABLED: from zilencer.models import ( RemoteInstallationCount, RemotePushDeviceToken, + RemoteRealm, RemoteRealmAuditLog, RemoteRealmCount, RemoteZulipServer, @@ -1014,6 +1016,34 @@ class AnalyticsBouncerTest(BouncerTestCase): send_analytics_to_push_bouncer() check_counts(2, 2, 1, 1, 1) + self.assertEqual( + list( + RemoteRealm.objects.order_by("id").values( + "server_id", + "uuid", + "uuid_owner_secret", + "host", + "realm_date_created", + "registration_deactivated", + "realm_deactivated", + "plan_type", + ) + ), + [ + { + "server_id": self.server.id, + "uuid": realm.uuid, + "uuid_owner_secret": realm.uuid_owner_secret, + "host": realm.host, + "realm_date_created": realm.date_created, + "registration_deactivated": False, + "realm_deactivated": False, + "plan_type": RemoteRealm.PLAN_TYPE_SELF_HOSTED, + } + for realm in Realm.objects.order_by("id") + ], + ) + # Test having no new rows send_analytics_to_push_bouncer() check_counts(3, 2, 1, 1, 1) @@ -1148,6 +1178,44 @@ class AnalyticsBouncerTest(BouncerTestCase): self.assertEqual(m.output, ["WARNING:root:Invalid property invalid count stat"]) self.assertEqual(RemoteRealmCount.objects.count(), 0) + @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @responses.activate + def test_remote_realm_duplicate_uuid(self) -> None: + """ + Tests for a case where a RemoteRealm with a certain uuid is already registered for one server, + and then another server tries to register the same uuid. This generally shouldn't happen, + because export->import of a realm should re-generate the uuid, but we should have error + handling for this edge case nonetheless. + """ + + second_server = RemoteZulipServer.objects.create( + uuid=uuid.uuid4(), + api_key="magic_secret_api_key2", + hostname="demo2.example.com", + last_updated=now(), + ) + + self.add_mock_response() + user = self.example_user("hamlet") + realm = user.realm + + RemoteRealm.objects.create( + server=second_server, + uuid=realm.uuid, + uuid_owner_secret=realm.uuid_owner_secret, + host=realm.host, + realm_date_created=realm.date_created, + registration_deactivated=False, + realm_deactivated=False, + plan_type=RemoteRealm.PLAN_TYPE_SELF_HOSTED, + ) + + with transaction.atomic(), self.assertLogs(level="WARNING") as m: + # The usual atomic() wrapper to avoid IntegrityError breaking the test's + # transaction. + send_analytics_to_push_bouncer() + self.assertEqual(m.output, ["WARNING:root:Duplicate registration detected."]) + # Servers on Zulip 2.0.6 and earlier only send realm_counts and installation_counts data, # and don't send realmauditlog_rows. Make sure that continues to work. @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") diff --git a/zilencer/migrations/0033_remoterealm.py b/zilencer/migrations/0033_remoterealm.py new file mode 100644 index 0000000000..6cff187d31 --- /dev/null +++ b/zilencer/migrations/0033_remoterealm.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.6 on 2023-11-02 23:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("zilencer", "0032_remotepushdevicetoken_backfill_ios_app_id"), + ] + + operations = [ + migrations.CreateModel( + name="RemoteRealm", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("uuid", models.UUIDField(unique=True)), + ("uuid_owner_secret", models.TextField()), + ("host", models.TextField()), + ("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")), + ("registration_deactivated", models.BooleanField(default=False)), + ("realm_deactivated", models.BooleanField(default=False)), + ("realm_date_created", models.DateTimeField()), + ("plan_type", models.PositiveSmallIntegerField(db_index=True, default=1)), + ( + "server", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver" + ), + ), + ], + ), + ] diff --git a/zilencer/models.py b/zilencer/models.py index cd97caf5b8..2142adb6d4 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -84,6 +84,41 @@ class RemotePushDeviceToken(AbstractPushDeviceToken): return f"{self.server!r} {self.user_id}" +class RemoteRealm(models.Model): + """ + Each object corresponds to a single remote Realm that is using the + Mobile Push Notifications Service via `manage.py register_server`. + """ + + server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE) + + # The unique UUID and secret for this realm. + uuid = models.UUIDField(unique=True) + uuid_owner_secret = models.TextField() + + # Value obtained's from the remote server's realm.host. + host = models.TextField() + + # The fields below are analogical to RemoteZulipServer fields. + + last_updated = models.DateTimeField("last updated", auto_now=True) + + # Whether the realm registration has been deactivated. + registration_deactivated = models.BooleanField(default=False) + # Whether the realm has been deactivated on the remote server. + realm_deactivated = models.BooleanField(default=False) + + # When the realm was created on the remote server. + realm_date_created = models.DateTimeField() + + # Plan types for self-hosted customers + PLAN_TYPE_SELF_HOSTED = 1 + PLAN_TYPE_STANDARD = 102 + + # The current billing plan for the remote server, similar to Realm.plan_type. + plan_type = models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_HOSTED, db_index=True) + + class RemoteZulipServerAuditLog(AbstractRealmAuditLog): """Audit data associated with a remote Zulip server (not specific to a realm). Used primarily for tracking registration and billing diff --git a/zilencer/views.py b/zilencer/views.py index 514c6ef25a..bcf32bcfb1 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -34,6 +34,7 @@ from zerver.lib.push_notifications import ( ) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success +from zerver.lib.timestamp import timestamp_to_datetime from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint from zerver.lib.validator import ( check_bool, @@ -53,6 +54,7 @@ from zilencer.auth import InvalidZulipServerKeyError from zilencer.models import ( RemoteInstallationCount, RemotePushDeviceToken, + RemoteRealm, RemoteRealmAuditLog, RemoteRealmCount, RemoteZulipServer, @@ -501,6 +503,36 @@ def batch_create_table_data( row_objects = row_objects[BATCH_SIZE:] +def update_remote_realm_data_for_server( + server: RemoteZulipServer, server_realms_info: List[Dict[str, Any]] +) -> None: + uuids = [realm["uuid"] for realm in server_realms_info] + already_registered_uuids = set( + str(uuid) + for uuid in RemoteRealm.objects.filter(uuid__in=uuids, server=server).values_list( + "uuid", flat=True + ) + ) + + new_remote_realms = [ + RemoteRealm( + server=server, + uuid=realm["uuid"], + uuid_owner_secret=realm["uuid_owner_secret"], + host=realm["host"], + realm_deactivated=realm["deactivated"], + realm_date_created=timestamp_to_datetime(realm["date_created"]), + ) + for realm in server_realms_info + if realm["uuid"] not in already_registered_uuids + ] + + try: + RemoteRealm.objects.bulk_create(new_remote_realms) + except IntegrityError: + raise JsonableError(_("Duplicate registration detected.")) + + @has_request_variables def remote_server_post_analytics( request: HttpRequest, @@ -547,12 +579,32 @@ def remote_server_post_analytics( ), default=None, ), + realms: Optional[List[Dict[str, Any]]] = REQ( + # Pre-8.0 servers don't send this data. + default=None, + json_validator=check_list( + check_dict_only( + [ + ("id", check_int), + ("uuid", check_string), + ("uuid_owner_secret", check_string), + ("host", check_string), + ("url", check_string), + ("deactivated", check_bool), + ("date_created", check_float), + ] + ) + ), + ), ) -> HttpResponse: validate_incoming_table_data(server, RemoteRealmCount, realm_counts, True) validate_incoming_table_data(server, RemoteInstallationCount, installation_counts, True) if realmauditlog_rows is not None: validate_incoming_table_data(server, RemoteRealmAuditLog, realmauditlog_rows) + if realms is not None: + update_remote_realm_data_for_server(server, realms) + remote_realm_counts = [ RemoteRealmCount( property=row["property"],