From 76e0511481193ace9e4c1aad4982562a7bb9eb83 Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Mon, 30 Oct 2023 23:50:53 +0100 Subject: [PATCH] zilencer: Add new model RemoteRealm and send the data to the bouncer. Add the new model for recording basic information about Realms on remote server, to go with the other analytics data. Also adds necessary changes to the bouncer endpoint and the send_analytics_to_push_bouncer() function to submit such Realm information. --- zerver/lib/remote_server.py | 21 +++++++- zerver/tests/test_push_notifications.py | 68 +++++++++++++++++++++++++ zilencer/migrations/0033_remoterealm.py | 38 ++++++++++++++ zilencer/models.py | 35 +++++++++++++ zilencer/views.py | 52 +++++++++++++++++++ 5 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 zilencer/migrations/0033_remoterealm.py diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index db5cfbbbbc..5aad3419b3 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -13,7 +13,7 @@ from version import ZULIP_VERSION from zerver.lib.exceptions import JsonableError from zerver.lib.export import floatify_datetime_fields from zerver.lib.outgoing_http import OutgoingSession -from zerver.models import RealmAuditLog +from zerver.models import Realm, RealmAuditLog class PushBouncerSession(OutgoingSession): @@ -172,6 +172,24 @@ def build_analytics_data( ) +def get_realms_info_for_push_bouncer() -> List[Dict[str, Any]]: + realms = Realm.objects.order_by("id") + realm_info_dicts = [ + dict( + id=realm.id, + uuid=str(realm.uuid), + uuid_owner_secret=realm.uuid_owner_secret, + host=realm.host, + url=realm.uri, + deactivated=realm.deactivated, + date_created=realm.date_created.timestamp(), + ) + for realm in realms + ] + + return realm_info_dicts + + def send_analytics_to_push_bouncer() -> None: # first, check what's latest try: @@ -203,6 +221,7 @@ def send_analytics_to_push_bouncer() -> None: "realm_counts": orjson.dumps(realm_count_data).decode(), "installation_counts": orjson.dumps(installation_count_data).decode(), "realmauditlog_rows": orjson.dumps(realmauditlog_data).decode(), + "realms": orjson.dumps(get_realms_info_for_push_bouncer()).decode(), "version": orjson.dumps(ZULIP_VERSION).decode(), } diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index af37be1b0c..0be27d9588 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -70,6 +70,7 @@ from zerver.models import ( Message, NotificationTriggers, PushDeviceToken, + Realm, RealmAuditLog, Recipient, Stream, @@ -87,6 +88,7 @@ if settings.ZILENCER_ENABLED: from zilencer.models import ( RemoteInstallationCount, RemotePushDeviceToken, + RemoteRealm, RemoteRealmAuditLog, RemoteRealmCount, RemoteZulipServer, @@ -1014,6 +1016,34 @@ class AnalyticsBouncerTest(BouncerTestCase): send_analytics_to_push_bouncer() check_counts(2, 2, 1, 1, 1) + self.assertEqual( + list( + RemoteRealm.objects.order_by("id").values( + "server_id", + "uuid", + "uuid_owner_secret", + "host", + "realm_date_created", + "registration_deactivated", + "realm_deactivated", + "plan_type", + ) + ), + [ + { + "server_id": self.server.id, + "uuid": realm.uuid, + "uuid_owner_secret": realm.uuid_owner_secret, + "host": realm.host, + "realm_date_created": realm.date_created, + "registration_deactivated": False, + "realm_deactivated": False, + "plan_type": RemoteRealm.PLAN_TYPE_SELF_HOSTED, + } + for realm in Realm.objects.order_by("id") + ], + ) + # Test having no new rows send_analytics_to_push_bouncer() check_counts(3, 2, 1, 1, 1) @@ -1148,6 +1178,44 @@ class AnalyticsBouncerTest(BouncerTestCase): self.assertEqual(m.output, ["WARNING:root:Invalid property invalid count stat"]) self.assertEqual(RemoteRealmCount.objects.count(), 0) + @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @responses.activate + def test_remote_realm_duplicate_uuid(self) -> None: + """ + Tests for a case where a RemoteRealm with a certain uuid is already registered for one server, + and then another server tries to register the same uuid. This generally shouldn't happen, + because export->import of a realm should re-generate the uuid, but we should have error + handling for this edge case nonetheless. + """ + + second_server = RemoteZulipServer.objects.create( + uuid=uuid.uuid4(), + api_key="magic_secret_api_key2", + hostname="demo2.example.com", + last_updated=now(), + ) + + self.add_mock_response() + user = self.example_user("hamlet") + realm = user.realm + + RemoteRealm.objects.create( + server=second_server, + uuid=realm.uuid, + uuid_owner_secret=realm.uuid_owner_secret, + host=realm.host, + realm_date_created=realm.date_created, + registration_deactivated=False, + realm_deactivated=False, + plan_type=RemoteRealm.PLAN_TYPE_SELF_HOSTED, + ) + + with transaction.atomic(), self.assertLogs(level="WARNING") as m: + # The usual atomic() wrapper to avoid IntegrityError breaking the test's + # transaction. + send_analytics_to_push_bouncer() + self.assertEqual(m.output, ["WARNING:root:Duplicate registration detected."]) + # Servers on Zulip 2.0.6 and earlier only send realm_counts and installation_counts data, # and don't send realmauditlog_rows. Make sure that continues to work. @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") diff --git a/zilencer/migrations/0033_remoterealm.py b/zilencer/migrations/0033_remoterealm.py new file mode 100644 index 0000000000..6cff187d31 --- /dev/null +++ b/zilencer/migrations/0033_remoterealm.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.6 on 2023-11-02 23:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("zilencer", "0032_remotepushdevicetoken_backfill_ios_app_id"), + ] + + operations = [ + migrations.CreateModel( + name="RemoteRealm", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("uuid", models.UUIDField(unique=True)), + ("uuid_owner_secret", models.TextField()), + ("host", models.TextField()), + ("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")), + ("registration_deactivated", models.BooleanField(default=False)), + ("realm_deactivated", models.BooleanField(default=False)), + ("realm_date_created", models.DateTimeField()), + ("plan_type", models.PositiveSmallIntegerField(db_index=True, default=1)), + ( + "server", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver" + ), + ), + ], + ), + ] diff --git a/zilencer/models.py b/zilencer/models.py index cd97caf5b8..2142adb6d4 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -84,6 +84,41 @@ class RemotePushDeviceToken(AbstractPushDeviceToken): return f"{self.server!r} {self.user_id}" +class RemoteRealm(models.Model): + """ + Each object corresponds to a single remote Realm that is using the + Mobile Push Notifications Service via `manage.py register_server`. + """ + + server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE) + + # The unique UUID and secret for this realm. + uuid = models.UUIDField(unique=True) + uuid_owner_secret = models.TextField() + + # Value obtained's from the remote server's realm.host. + host = models.TextField() + + # The fields below are analogical to RemoteZulipServer fields. + + last_updated = models.DateTimeField("last updated", auto_now=True) + + # Whether the realm registration has been deactivated. + registration_deactivated = models.BooleanField(default=False) + # Whether the realm has been deactivated on the remote server. + realm_deactivated = models.BooleanField(default=False) + + # When the realm was created on the remote server. + realm_date_created = models.DateTimeField() + + # Plan types for self-hosted customers + PLAN_TYPE_SELF_HOSTED = 1 + PLAN_TYPE_STANDARD = 102 + + # The current billing plan for the remote server, similar to Realm.plan_type. + plan_type = models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_HOSTED, db_index=True) + + class RemoteZulipServerAuditLog(AbstractRealmAuditLog): """Audit data associated with a remote Zulip server (not specific to a realm). Used primarily for tracking registration and billing diff --git a/zilencer/views.py b/zilencer/views.py index 514c6ef25a..bcf32bcfb1 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -34,6 +34,7 @@ from zerver.lib.push_notifications import ( ) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success +from zerver.lib.timestamp import timestamp_to_datetime from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint from zerver.lib.validator import ( check_bool, @@ -53,6 +54,7 @@ from zilencer.auth import InvalidZulipServerKeyError from zilencer.models import ( RemoteInstallationCount, RemotePushDeviceToken, + RemoteRealm, RemoteRealmAuditLog, RemoteRealmCount, RemoteZulipServer, @@ -501,6 +503,36 @@ def batch_create_table_data( row_objects = row_objects[BATCH_SIZE:] +def update_remote_realm_data_for_server( + server: RemoteZulipServer, server_realms_info: List[Dict[str, Any]] +) -> None: + uuids = [realm["uuid"] for realm in server_realms_info] + already_registered_uuids = set( + str(uuid) + for uuid in RemoteRealm.objects.filter(uuid__in=uuids, server=server).values_list( + "uuid", flat=True + ) + ) + + new_remote_realms = [ + RemoteRealm( + server=server, + uuid=realm["uuid"], + uuid_owner_secret=realm["uuid_owner_secret"], + host=realm["host"], + realm_deactivated=realm["deactivated"], + realm_date_created=timestamp_to_datetime(realm["date_created"]), + ) + for realm in server_realms_info + if realm["uuid"] not in already_registered_uuids + ] + + try: + RemoteRealm.objects.bulk_create(new_remote_realms) + except IntegrityError: + raise JsonableError(_("Duplicate registration detected.")) + + @has_request_variables def remote_server_post_analytics( request: HttpRequest, @@ -547,12 +579,32 @@ def remote_server_post_analytics( ), default=None, ), + realms: Optional[List[Dict[str, Any]]] = REQ( + # Pre-8.0 servers don't send this data. + default=None, + json_validator=check_list( + check_dict_only( + [ + ("id", check_int), + ("uuid", check_string), + ("uuid_owner_secret", check_string), + ("host", check_string), + ("url", check_string), + ("deactivated", check_bool), + ("date_created", check_float), + ] + ) + ), + ), ) -> HttpResponse: validate_incoming_table_data(server, RemoteRealmCount, realm_counts, True) validate_incoming_table_data(server, RemoteInstallationCount, installation_counts, True) if realmauditlog_rows is not None: validate_incoming_table_data(server, RemoteRealmAuditLog, realmauditlog_rows) + if realms is not None: + update_remote_realm_data_for_server(server, realms) + remote_realm_counts = [ RemoteRealmCount( property=row["property"],