remote_realm: Add syncing of org_type.

zerver/lib/remote_server.py

@@ -7,14 +7,14 @@ import requests
 from django.conf import settings
 from django.forms.models import model_to_dict
 from django.utils.translation import gettext as _
-from pydantic import UUID4, BaseModel, ConfigDict
+from pydantic import UUID4, BaseModel, ConfigDict, field_validator
 
 from analytics.models import InstallationCount, RealmCount
 from version import ZULIP_VERSION
 from zerver.lib.exceptions import JsonableError, MissingRemoteRealmError
 from zerver.lib.export import floatify_datetime_fields
 from zerver.lib.outgoing_http import OutgoingSession
-from zerver.models import Realm, RealmAuditLog
+from zerver.models import OrgTypeEnum, Realm, RealmAuditLog
 
 
 class PushBouncerSession(OutgoingSession):
@@ -36,12 +36,21 @@ class RealmDataForAnalytics(BaseModel):
     id: int
     host: str
     url: str
+    org_type: int = 0
     date_created: float
     deactivated: bool
 
     uuid: UUID4
     uuid_owner_secret: str
 
+    @field_validator("org_type")
+    @classmethod
+    def check_is_allowed_value(cls, value: int) -> int:
+        if value not in [org_type.value for org_type in OrgTypeEnum]:
+            raise ValueError("Not a valid org_type value")
+
+        return value
+
 
 class UserDataForRemoteBilling(BaseModel):
     uuid: UUID4
@@ -214,6 +223,7 @@ def get_realms_info_for_push_bouncer(realm_id: Optional[int] = None) -> List[Rea
             url=realm.uri,
             deactivated=realm.deactivated,
             date_created=realm.date_created.timestamp(),
+            org_type=realm.org_type,
         )
         for realm in realms
     ]

zerver/lib/typed_endpoint.py

@@ -364,6 +364,9 @@ def parse_value_for_parameter(parameter: FuncParam[T], value: object) -> T:
         # This condition matches our StringRequiredConstraint
         elif error["type"] == "string_too_short" and error["ctx"].get("min_length") == 1:
             error_template = _("{var_name} cannot be blank")
+        elif error["type"] == "value_error":
+            context["msg"] = error["msg"]
+            error_template = _("Invalid {var_name}: {msg}")
 
         assert error_template is not None, MISSING_ERROR_TEMPLATE.format(
             error_type=error["type"],

zerver/models.py

@@ -8,6 +8,7 @@ import time
 from collections import defaultdict
 from datetime import timedelta
 from email.headerregistry import Address
+from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -295,6 +296,22 @@ def generate_realm_uuid_owner_secret() -> str:
     return f"zuliprealm_{token}"
 
 
+class OrgTypeEnum(Enum):
+    Unspecified = 0
+    Business = 10
+    OpenSource = 20
+    EducationNonProfit = 30
+    Education = 35
+    Research = 40
+    Event = 50
+    NonProfit = 60
+    Government = 70
+    PoliticalGroup = 80
+    Community = 90
+    Personal = 100
+    Other = 1000
+
+
 class OrgTypeDict(TypedDict):
     name: str
     id: int
@@ -565,91 +582,91 @@ class Realm(models.Model):  # type: ignore[django-manager-missing] # django-stub
     ORG_TYPES: Dict[str, OrgTypeDict] = {
         "unspecified": {
             "name": "Unspecified",
-            "id": 0,
+            "id": OrgTypeEnum.Unspecified.value,
             "hidden": True,
             "display_order": 0,
             "onboarding_zulip_guide_url": None,
         },
         "business": {
             "name": "Business",
-            "id": 10,
+            "id": OrgTypeEnum.Business.value,
             "hidden": False,
             "display_order": 1,
             "onboarding_zulip_guide_url": "https://zulip.com/for/business/",
         },
         "opensource": {
             "name": "Open-source project",
-            "id": 20,
+            "id": OrgTypeEnum.OpenSource.value,
             "hidden": False,
             "display_order": 2,
             "onboarding_zulip_guide_url": "https://zulip.com/for/open-source/",
         },
         "education_nonprofit": {
             "name": "Education (non-profit)",
-            "id": 30,
+            "id": OrgTypeEnum.EducationNonProfit.value,
             "hidden": False,
             "display_order": 3,
             "onboarding_zulip_guide_url": "https://zulip.com/for/education/",
         },
         "education": {
             "name": "Education (for-profit)",
-            "id": 35,
+            "id": OrgTypeEnum.Education.value,
             "hidden": False,
             "display_order": 4,
             "onboarding_zulip_guide_url": "https://zulip.com/for/education/",
         },
         "research": {
             "name": "Research",
-            "id": 40,
+            "id": OrgTypeEnum.Research.value,
             "hidden": False,
             "display_order": 5,
             "onboarding_zulip_guide_url": "https://zulip.com/for/research/",
         },
         "event": {
             "name": "Event or conference",
-            "id": 50,
+            "id": OrgTypeEnum.Event.value,
             "hidden": False,
             "display_order": 6,
             "onboarding_zulip_guide_url": "https://zulip.com/for/events/",
         },
         "nonprofit": {
             "name": "Non-profit (registered)",
-            "id": 60,
+            "id": OrgTypeEnum.NonProfit.value,
             "hidden": False,
             "display_order": 7,
             "onboarding_zulip_guide_url": "https://zulip.com/for/communities/",
         },
         "government": {
             "name": "Government",
-            "id": 70,
+            "id": OrgTypeEnum.Government.value,
             "hidden": False,
             "display_order": 8,
             "onboarding_zulip_guide_url": None,
         },
         "political_group": {
             "name": "Political group",
-            "id": 80,
+            "id": OrgTypeEnum.PoliticalGroup.value,
             "hidden": False,
             "display_order": 9,
             "onboarding_zulip_guide_url": None,
         },
         "community": {
             "name": "Community",
-            "id": 90,
+            "id": OrgTypeEnum.Community.value,
             "hidden": False,
             "display_order": 10,
             "onboarding_zulip_guide_url": "https://zulip.com/for/communities/",
         },
         "personal": {
             "name": "Personal",
-            "id": 100,
+            "id": OrgTypeEnum.Personal.value,
             "hidden": False,
             "display_order": 100,
             "onboarding_zulip_guide_url": None,
         },
         "other": {
             "name": "Other",
-            "id": 1000,
+            "id": OrgTypeEnum.Other.value,
             "hidden": False,
             "display_order": 1000,
             "onboarding_zulip_guide_url": None,

zerver/tests/test_push_notifications.py

@@ -979,6 +979,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
                     "uuid",
                     "uuid_owner_secret",
                     "host",
+                    "org_type",
                     "realm_date_created",
                     "registration_deactivated",
                     "realm_deactivated",
@@ -991,6 +992,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
                     "uuid": realm.uuid,
                     "uuid_owner_secret": realm.uuid_owner_secret,
                     "host": realm.host,
+                    "org_type": realm.org_type,
                     "realm_date_created": realm.date_created,
                     "registration_deactivated": False,
                     "realm_deactivated": False,
@@ -1160,6 +1162,26 @@ class AnalyticsBouncerTest(BouncerTestCase):
         # Only the request counts go up -- all of the other rows' duplicates are dropped
         check_counts(10, 8, 3, 2, 5)
 
+        # Test that only valid org_type values are accepted - integers defined in OrgTypeEnum.
+        realms_data = [dict(realm) for realm in get_realms_info_for_push_bouncer()]
+        # Not a valid org_type value:
+        realms_data[0]["org_type"] = 11
+
+        result = self.uuid_post(
+            self.server_uuid,
+            "/api/v1/remotes/server/analytics",
+            {
+                "realm_counts": orjson.dumps([]).decode(),
+                "installation_counts": orjson.dumps([]).decode(),
+                "realmauditlog_rows": orjson.dumps([]).decode(),
+                "realms": orjson.dumps(realms_data).decode(),
+            },
+            subdomain="",
+        )
+        self.assert_json_error(
+            result, 'Invalid realms[0]["org_type"]: Value error, Not a valid org_type value'
+        )
+
     @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
     @responses.activate
     def test_analytics_api_invalid(self) -> None:

zilencer/migrations/0039_remoterealm_org_type.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.7 on 2023-11-27 00:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0038_unique_server_remote_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="org_type",
+            field=models.PositiveSmallIntegerField(
+                choices=[
+                    (0, "Unspecified"),
+                    (10, "Business"),
+                    (20, "Open-source project"),
+                    (30, "Education (non-profit)"),
+                    (35, "Education (for-profit)"),
+                    (40, "Research"),
+                    (50, "Event or conference"),
+                    (60, "Non-profit (registered)"),
+                    (70, "Government"),
+                    (80, "Political group"),
+                    (90, "Community"),
+                    (100, "Personal"),
+                    (1000, "Other"),
+                ],
+                default=0,
+            ),
+        ),
+    ]

zilencer/models.py

@@ -12,7 +12,7 @@ from typing_extensions import override
 from analytics.models import BaseCount
 from zerver.lib.rate_limiter import RateLimitedObject
 from zerver.lib.rate_limiter import rules as rate_limiter_rules
-from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog
+from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog, Realm
 
 
 def get_remote_server_by_uuid(uuid: str) -> "RemoteZulipServer":
@@ -104,6 +104,11 @@ class RemoteRealm(models.Model):
     # Value obtained's from the remote server's realm.host.
     host = models.TextField()
 
+    org_type = models.PositiveSmallIntegerField(
+        default=Realm.ORG_TYPES["unspecified"]["id"],
+        choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()],
+    )
+
     # The fields below are analogical to RemoteZulipServer fields.
 
     last_updated = models.DateTimeField("last updated", auto_now=True)

zilencer/views.py

@@ -542,6 +542,7 @@ def update_remote_realm_data_for_server(
             host=realm.host,
             realm_deactivated=realm.deactivated,
             realm_date_created=timestamp_to_datetime(realm.date_created),
+            org_type=realm.org_type,
         )
         for realm in server_realms_info
         if realm.uuid not in already_registered_uuids