Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

zilencer: Add new model RemoteRealm and send the data to the bouncer. 

Add the new model for recording basic information about Realms on remote
server, to go with the other analytics data. Also adds necessary changes
to the bouncer endpoint and the send_analytics_to_push_bouncer()
function to submit such Realm information.
This commit is contained in:
Mateusz Mandera 2023-10-30 23:50:53 +01:00 committed by Tim Abbott
parent e72a9fb814
commit 76e0511481
5 changed files with 213 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
zerver/lib/remote_server.py
View File

@ -13,7 +13,7 @@ from version import ZULIP_VERSION
from zerver.lib.exceptions import JsonableError
from zerver.lib.export import floatify_datetime_fields
from zerver.lib.outgoing_http import OutgoingSession
from zerver.models import RealmAuditLog
from zerver.models import Realm, RealmAuditLog
class PushBouncerSession(OutgoingSession):
@ -172,6 +172,24 @@ def build_analytics_data(
)
def get_realms_info_for_push_bouncer() -> List[Dict[str, Any]]:
realms = Realm.objects.order_by("id")
realm_info_dicts = [
dict(
id=realm.id,
uuid=str(realm.uuid),
uuid_owner_secret=realm.uuid_owner_secret,
host=realm.host,
url=realm.uri,
deactivated=realm.deactivated,
date_created=realm.date_created.timestamp(),
)
for realm in realms
]
return realm_info_dicts
def send_analytics_to_push_bouncer() -> None:
# first, check what's latest
try:
@ -203,6 +221,7 @@ def send_analytics_to_push_bouncer() -> None:
"realm_counts": orjson.dumps(realm_count_data).decode(),
"installation_counts": orjson.dumps(installation_count_data).decode(),
"realmauditlog_rows": orjson.dumps(realmauditlog_data).decode(),
"realms": orjson.dumps(get_realms_info_for_push_bouncer()).decode(),
"version": orjson.dumps(ZULIP_VERSION).decode(),
}

68
zerver/tests/test_push_notifications.py
View File

@ -70,6 +70,7 @@ from zerver.models import (
Message,
NotificationTriggers,
PushDeviceToken,
Realm,
RealmAuditLog,
Recipient,
Stream,
@ -87,6 +88,7 @@ if settings.ZILENCER_ENABLED:
from zilencer.models import (
RemoteInstallationCount,
RemotePushDeviceToken,
RemoteRealm,
RemoteRealmAuditLog,
RemoteRealmCount,
RemoteZulipServer,
@ -1014,6 +1016,34 @@ class AnalyticsBouncerTest(BouncerTestCase):
send_analytics_to_push_bouncer()
check_counts(2, 2, 1, 1, 1)
self.assertEqual(
list(
RemoteRealm.objects.order_by("id").values(
"server_id",
"uuid",
"uuid_owner_secret",
"host",
"realm_date_created",
"registration_deactivated",
"realm_deactivated",
"plan_type",
)
),
[
{
"server_id": self.server.id,
"uuid": realm.uuid,
"uuid_owner_secret": realm.uuid_owner_secret,
"host": realm.host,
"realm_date_created": realm.date_created,
"registration_deactivated": False,
"realm_deactivated": False,
"plan_type": RemoteRealm.PLAN_TYPE_SELF_HOSTED,
}
for realm in Realm.objects.order_by("id")
],
)
# Test having no new rows
send_analytics_to_push_bouncer()
check_counts(3, 2, 1, 1, 1)
@ -1148,6 +1178,44 @@ class AnalyticsBouncerTest(BouncerTestCase):
self.assertEqual(m.output, ["WARNING:root:Invalid property invalid count stat"])
self.assertEqual(RemoteRealmCount.objects.count(), 0)
@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
@responses.activate
def test_remote_realm_duplicate_uuid(self) -> None:
"""
Tests for a case where a RemoteRealm with a certain uuid is already registered for one server,
and then another server tries to register the same uuid. This generally shouldn't happen,
because export->import of a realm should re-generate the uuid, but we should have error
handling for this edge case nonetheless.
"""
second_server = RemoteZulipServer.objects.create(
uuid=uuid.uuid4(),
api_key="magic_secret_api_key2",
hostname="demo2.example.com",
last_updated=now(),
)
self.add_mock_response()
user = self.example_user("hamlet")
realm = user.realm
RemoteRealm.objects.create(
server=second_server,
uuid=realm.uuid,
uuid_owner_secret=realm.uuid_owner_secret,
host=realm.host,
realm_date_created=realm.date_created,
registration_deactivated=False,
realm_deactivated=False,
plan_type=RemoteRealm.PLAN_TYPE_SELF_HOSTED,
)
with transaction.atomic(), self.assertLogs(level="WARNING") as m:
# The usual atomic() wrapper to avoid IntegrityError breaking the test's
# transaction.
send_analytics_to_push_bouncer()
self.assertEqual(m.output, ["WARNING:root:Duplicate registration detected."])
# Servers on Zulip 2.0.6 and earlier only send realm_counts and installation_counts data,
# and don't send realmauditlog_rows. Make sure that continues to work.
@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")

38
zilencer/migrations/0033_remoterealm.py Normal file
View File

@ -0,0 +1,38 @@
# Generated by Django 4.2.6 on 2023-11-02 23:22
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("zilencer", "0032_remotepushdevicetoken_backfill_ios_app_id"),
]
operations = [
migrations.CreateModel(
name="RemoteRealm",
fields=[
(
"id",
models.AutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("uuid", models.UUIDField(unique=True)),
("uuid_owner_secret", models.TextField()),
("host", models.TextField()),
("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")),
("registration_deactivated", models.BooleanField(default=False)),
("realm_deactivated", models.BooleanField(default=False)),
("realm_date_created", models.DateTimeField()),
("plan_type", models.PositiveSmallIntegerField(db_index=True, default=1)),
(
"server",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
),
),
],
),
]

35
zilencer/models.py
View File

@ -84,6 +84,41 @@ class RemotePushDeviceToken(AbstractPushDeviceToken):
return f"{self.server!r} {self.user_id}"
class RemoteRealm(models.Model):
"""
Each object corresponds to a single remote Realm that is using the
Mobile Push Notifications Service via `manage.py register_server`.
"""
server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
# The unique UUID and secret for this realm.
uuid = models.UUIDField(unique=True)
uuid_owner_secret = models.TextField()
# Value obtained's from the remote server's realm.host.
host = models.TextField()
# The fields below are analogical to RemoteZulipServer fields.
last_updated = models.DateTimeField("last updated", auto_now=True)
# Whether the realm registration has been deactivated.
registration_deactivated = models.BooleanField(default=False)
# Whether the realm has been deactivated on the remote server.
realm_deactivated = models.BooleanField(default=False)
# When the realm was created on the remote server.
realm_date_created = models.DateTimeField()
# Plan types for self-hosted customers
PLAN_TYPE_SELF_HOSTED = 1
PLAN_TYPE_STANDARD = 102
# The current billing plan for the remote server, similar to Realm.plan_type.
plan_type = models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_HOSTED, db_index=True)
class RemoteZulipServerAuditLog(AbstractRealmAuditLog):
"""Audit data associated with a remote Zulip server (not specific to a
realm). Used primarily for tracking registration and billing

52
zilencer/views.py
View File

@ -34,6 +34,7 @@ from zerver.lib.push_notifications import (
)
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_success
from zerver.lib.timestamp import timestamp_to_datetime
from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
from zerver.lib.validator import (
check_bool,
@ -53,6 +54,7 @@ from zilencer.auth import InvalidZulipServerKeyError
from zilencer.models import (
RemoteInstallationCount,
RemotePushDeviceToken,
RemoteRealm,
RemoteRealmAuditLog,
RemoteRealmCount,
RemoteZulipServer,
@ -501,6 +503,36 @@ def batch_create_table_data(
row_objects = row_objects[BATCH_SIZE:]
def update_remote_realm_data_for_server(
server: RemoteZulipServer, server_realms_info: List[Dict[str, Any]]
) -> None:
uuids = [realm["uuid"] for realm in server_realms_info]
already_registered_uuids = set(
str(uuid)
for uuid in RemoteRealm.objects.filter(uuid__in=uuids, server=server).values_list(
"uuid", flat=True
)
)
new_remote_realms = [
RemoteRealm(
server=server,
uuid=realm["uuid"],
uuid_owner_secret=realm["uuid_owner_secret"],
host=realm["host"],
realm_deactivated=realm["deactivated"],
realm_date_created=timestamp_to_datetime(realm["date_created"]),
)
for realm in server_realms_info
if realm["uuid"] not in already_registered_uuids
]
try:
RemoteRealm.objects.bulk_create(new_remote_realms)
except IntegrityError:
raise JsonableError(_("Duplicate registration detected."))
@has_request_variables
def remote_server_post_analytics(
request: HttpRequest,
@ -547,12 +579,32 @@ def remote_server_post_analytics(
),
default=None,
),
realms: Optional[List[Dict[str, Any]]] = REQ(
# Pre-8.0 servers don't send this data.
default=None,
json_validator=check_list(
check_dict_only(
[
("id", check_int),
("uuid", check_string),
("uuid_owner_secret", check_string),
("host", check_string),
("url", check_string),
("deactivated", check_bool),
("date_created", check_float),
]
)
),
),
) -> HttpResponse:
validate_incoming_table_data(server, RemoteRealmCount, realm_counts, True)
validate_incoming_table_data(server, RemoteInstallationCount, installation_counts, True)
if realmauditlog_rows is not None:
validate_incoming_table_data(server, RemoteRealmAuditLog, realmauditlog_rows)
if realms is not None:
update_remote_realm_data_for_server(server, realms)
remote_realm_counts = [
RemoteRealmCount(
property=row["property"],