From 02d5740f0fea174b69276bddb257c83baf105936 Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Mon, 27 Nov 2023 02:06:23 +0100 Subject: [PATCH] remote_realm: Add syncing of org_type. --- zerver/lib/remote_server.py | 14 +++++- zerver/lib/typed_endpoint.py | 3 ++ zerver/models.py | 43 +++++++++++++------ zerver/tests/test_push_notifications.py | 22 ++++++++++ .../migrations/0039_remoterealm_org_type.py | 34 +++++++++++++++ zilencer/models.py | 7 ++- zilencer/views.py | 1 + 7 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 zilencer/migrations/0039_remoterealm_org_type.py diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 0c2592a8ce..2b40cda2b6 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -7,14 +7,14 @@ import requests from django.conf import settings from django.forms.models import model_to_dict from django.utils.translation import gettext as _ -from pydantic import UUID4, BaseModel, ConfigDict +from pydantic import UUID4, BaseModel, ConfigDict, field_validator from analytics.models import InstallationCount, RealmCount from version import ZULIP_VERSION from zerver.lib.exceptions import JsonableError, MissingRemoteRealmError from zerver.lib.export import floatify_datetime_fields from zerver.lib.outgoing_http import OutgoingSession -from zerver.models import Realm, RealmAuditLog +from zerver.models import OrgTypeEnum, Realm, RealmAuditLog class PushBouncerSession(OutgoingSession): @@ -36,12 +36,21 @@ class RealmDataForAnalytics(BaseModel): id: int host: str url: str + org_type: int = 0 date_created: float deactivated: bool uuid: UUID4 uuid_owner_secret: str + @field_validator("org_type") + @classmethod + def check_is_allowed_value(cls, value: int) -> int: + if value not in [org_type.value for org_type in OrgTypeEnum]: + raise ValueError("Not a valid org_type value") + + return value + class UserDataForRemoteBilling(BaseModel): uuid: UUID4 @@ -214,6 +223,7 @@ def get_realms_info_for_push_bouncer(realm_id: Optional[int] = None) -> List[Rea url=realm.uri, deactivated=realm.deactivated, date_created=realm.date_created.timestamp(), + org_type=realm.org_type, ) for realm in realms ] diff --git a/zerver/lib/typed_endpoint.py b/zerver/lib/typed_endpoint.py index 1b79451625..ab511c38a0 100644 --- a/zerver/lib/typed_endpoint.py +++ b/zerver/lib/typed_endpoint.py @@ -364,6 +364,9 @@ def parse_value_for_parameter(parameter: FuncParam[T], value: object) -> T: # This condition matches our StringRequiredConstraint elif error["type"] == "string_too_short" and error["ctx"].get("min_length") == 1: error_template = _("{var_name} cannot be blank") + elif error["type"] == "value_error": + context["msg"] = error["msg"] + error_template = _("Invalid {var_name}: {msg}") assert error_template is not None, MISSING_ERROR_TEMPLATE.format( error_type=error["type"], diff --git a/zerver/models.py b/zerver/models.py index 4392246ba0..1e1661be3d 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -8,6 +8,7 @@ import time from collections import defaultdict from datetime import timedelta from email.headerregistry import Address +from enum import Enum from typing import ( TYPE_CHECKING, Any, @@ -295,6 +296,22 @@ def generate_realm_uuid_owner_secret() -> str: return f"zuliprealm_{token}" +class OrgTypeEnum(Enum): + Unspecified = 0 + Business = 10 + OpenSource = 20 + EducationNonProfit = 30 + Education = 35 + Research = 40 + Event = 50 + NonProfit = 60 + Government = 70 + PoliticalGroup = 80 + Community = 90 + Personal = 100 + Other = 1000 + + class OrgTypeDict(TypedDict): name: str id: int @@ -565,91 +582,91 @@ class Realm(models.Model): # type: ignore[django-manager-missing] # django-stub ORG_TYPES: Dict[str, OrgTypeDict] = { "unspecified": { "name": "Unspecified", - "id": 0, + "id": OrgTypeEnum.Unspecified.value, "hidden": True, "display_order": 0, "onboarding_zulip_guide_url": None, }, "business": { "name": "Business", - "id": 10, + "id": OrgTypeEnum.Business.value, "hidden": False, "display_order": 1, "onboarding_zulip_guide_url": "https://zulip.com/for/business/", }, "opensource": { "name": "Open-source project", - "id": 20, + "id": OrgTypeEnum.OpenSource.value, "hidden": False, "display_order": 2, "onboarding_zulip_guide_url": "https://zulip.com/for/open-source/", }, "education_nonprofit": { "name": "Education (non-profit)", - "id": 30, + "id": OrgTypeEnum.EducationNonProfit.value, "hidden": False, "display_order": 3, "onboarding_zulip_guide_url": "https://zulip.com/for/education/", }, "education": { "name": "Education (for-profit)", - "id": 35, + "id": OrgTypeEnum.Education.value, "hidden": False, "display_order": 4, "onboarding_zulip_guide_url": "https://zulip.com/for/education/", }, "research": { "name": "Research", - "id": 40, + "id": OrgTypeEnum.Research.value, "hidden": False, "display_order": 5, "onboarding_zulip_guide_url": "https://zulip.com/for/research/", }, "event": { "name": "Event or conference", - "id": 50, + "id": OrgTypeEnum.Event.value, "hidden": False, "display_order": 6, "onboarding_zulip_guide_url": "https://zulip.com/for/events/", }, "nonprofit": { "name": "Non-profit (registered)", - "id": 60, + "id": OrgTypeEnum.NonProfit.value, "hidden": False, "display_order": 7, "onboarding_zulip_guide_url": "https://zulip.com/for/communities/", }, "government": { "name": "Government", - "id": 70, + "id": OrgTypeEnum.Government.value, "hidden": False, "display_order": 8, "onboarding_zulip_guide_url": None, }, "political_group": { "name": "Political group", - "id": 80, + "id": OrgTypeEnum.PoliticalGroup.value, "hidden": False, "display_order": 9, "onboarding_zulip_guide_url": None, }, "community": { "name": "Community", - "id": 90, + "id": OrgTypeEnum.Community.value, "hidden": False, "display_order": 10, "onboarding_zulip_guide_url": "https://zulip.com/for/communities/", }, "personal": { "name": "Personal", - "id": 100, + "id": OrgTypeEnum.Personal.value, "hidden": False, "display_order": 100, "onboarding_zulip_guide_url": None, }, "other": { "name": "Other", - "id": 1000, + "id": OrgTypeEnum.Other.value, "hidden": False, "display_order": 1000, "onboarding_zulip_guide_url": None, diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 00676fd5a8..1e6066a9ec 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -979,6 +979,7 @@ class AnalyticsBouncerTest(BouncerTestCase): "uuid", "uuid_owner_secret", "host", + "org_type", "realm_date_created", "registration_deactivated", "realm_deactivated", @@ -991,6 +992,7 @@ class AnalyticsBouncerTest(BouncerTestCase): "uuid": realm.uuid, "uuid_owner_secret": realm.uuid_owner_secret, "host": realm.host, + "org_type": realm.org_type, "realm_date_created": realm.date_created, "registration_deactivated": False, "realm_deactivated": False, @@ -1160,6 +1162,26 @@ class AnalyticsBouncerTest(BouncerTestCase): # Only the request counts go up -- all of the other rows' duplicates are dropped check_counts(10, 8, 3, 2, 5) + # Test that only valid org_type values are accepted - integers defined in OrgTypeEnum. + realms_data = [dict(realm) for realm in get_realms_info_for_push_bouncer()] + # Not a valid org_type value: + realms_data[0]["org_type"] = 11 + + result = self.uuid_post( + self.server_uuid, + "/api/v1/remotes/server/analytics", + { + "realm_counts": orjson.dumps([]).decode(), + "installation_counts": orjson.dumps([]).decode(), + "realmauditlog_rows": orjson.dumps([]).decode(), + "realms": orjson.dumps(realms_data).decode(), + }, + subdomain="", + ) + self.assert_json_error( + result, 'Invalid realms[0]["org_type"]: Value error, Not a valid org_type value' + ) + @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") @responses.activate def test_analytics_api_invalid(self) -> None: diff --git a/zilencer/migrations/0039_remoterealm_org_type.py b/zilencer/migrations/0039_remoterealm_org_type.py new file mode 100644 index 0000000000..887bffb9f9 --- /dev/null +++ b/zilencer/migrations/0039_remoterealm_org_type.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.7 on 2023-11-27 00:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("zilencer", "0038_unique_server_remote_id"), + ] + + operations = [ + migrations.AddField( + model_name="remoterealm", + name="org_type", + field=models.PositiveSmallIntegerField( + choices=[ + (0, "Unspecified"), + (10, "Business"), + (20, "Open-source project"), + (30, "Education (non-profit)"), + (35, "Education (for-profit)"), + (40, "Research"), + (50, "Event or conference"), + (60, "Non-profit (registered)"), + (70, "Government"), + (80, "Political group"), + (90, "Community"), + (100, "Personal"), + (1000, "Other"), + ], + default=0, + ), + ), + ] diff --git a/zilencer/models.py b/zilencer/models.py index bddefe4a7e..ece1bc1c98 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -12,7 +12,7 @@ from typing_extensions import override from analytics.models import BaseCount from zerver.lib.rate_limiter import RateLimitedObject from zerver.lib.rate_limiter import rules as rate_limiter_rules -from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog +from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog, Realm def get_remote_server_by_uuid(uuid: str) -> "RemoteZulipServer": @@ -104,6 +104,11 @@ class RemoteRealm(models.Model): # Value obtained's from the remote server's realm.host. host = models.TextField() + org_type = models.PositiveSmallIntegerField( + default=Realm.ORG_TYPES["unspecified"]["id"], + choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()], + ) + # The fields below are analogical to RemoteZulipServer fields. last_updated = models.DateTimeField("last updated", auto_now=True) diff --git a/zilencer/views.py b/zilencer/views.py index ebb17af28e..324ec7b085 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -542,6 +542,7 @@ def update_remote_realm_data_for_server( host=realm.host, realm_deactivated=realm.deactivated, realm_date_created=timestamp_to_datetime(realm.date_created), + org_type=realm.org_type, ) for realm in server_realms_info if realm.uuid not in already_registered_uuids