settings: Rework how push notifications service is configured.

Instead of the PUSH_NOTIFICATIONS_BOUNCER_URL and
SUBMIT_USAGE_STATISTICS settings, we want servers to configure
individual ZULIP_SERVICE_* settings, while maintaining backward
compatibility with the old settings. Thus, if all the new
ZULIP_SERVICE_* are at their default False value, but the legacy
settings are activated, they need to be translated in computed_settings
to the modern way.

analytics/management/commands/update_analytics_counts.py

@@ -11,7 +11,7 @@ from typing_extensions import override
 
 from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
 from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
-from zerver.lib.remote_server import send_server_data_to_push_bouncer
+from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 
@@ -83,7 +83,10 @@ class Command(ZulipBaseCommand):
             )
         logger.info("Finished updating analytics counts through %s", fill_to_time)
 
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL:
+        if should_send_analytics_data():
+            # Based on the specific value of the setting, the exact details to send
+            # will be decided. However, we proceed just based on this not being falsey.
+
             # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
             # that each server will report in at a somewhat consistent time.
             assert settings.ZULIP_ORG_ID

analytics/tests/test_counts.py

@@ -8,7 +8,6 @@ import time_machine
 from django.apps import apps
 from django.db import models
 from django.db.models import Sum
-from django.test import override_settings
 from django.utils.timezone import now as timezone_now
 from psycopg2.sql import SQL, Literal
 from typing_extensions import override
@@ -58,6 +57,7 @@ from zerver.lib.push_notifications import (
     hex_to_b64,
 )
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import activate_push_notification_service
 from zerver.lib.timestamp import TimeZoneNotUTCError, ceiling_to_day, floor_to_day
 from zerver.lib.topic import DB_TOPIC_NAME
 from zerver.lib.user_counts import realm_user_count_by_role
@@ -1373,7 +1373,7 @@ class TestLoggingCountStats(AnalyticsTestCase):
         self.assertTableState(UserCount, ["property", "value"], [["user test", 1]])
         self.assertTableState(StreamCount, ["property", "value"], [["stream test", 1]])
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     def test_mobile_pushes_received_count(self) -> None:
         self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe"
         self.server = RemoteZulipServer.objects.create(

corporate/tests/test_remote_billing.py

@@ -5,7 +5,6 @@ from unittest import mock
 import responses
 import time_machine
 from django.conf import settings
-from django.test import override_settings
 from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
@@ -28,7 +27,7 @@ from zerver.lib.rate_limiter import RateLimitedIPAddr
 from zerver.lib.remote_server import send_server_data_to_push_bouncer
 from zerver.lib.send_email import FromAddress
 from zerver.lib.test_classes import BouncerTestCase
-from zerver.lib.test_helpers import ratelimit_rule
+from zerver.lib.test_helpers import activate_push_notification_service, ratelimit_rule
 from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.models import Realm, UserProfile
 from zerver.models.realms import get_realm
@@ -187,7 +186,7 @@ class RemoteRealmBillingTestCase(BouncerTestCase):
         return result
 
 
-@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+@activate_push_notification_service()
 class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase):
     @responses.activate
     def test_self_hosted_billing_endpoints(self) -> None:
@@ -209,7 +208,7 @@ class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase):
         self_hosted_billing_url = "/self-hosted-billing/"
         self_hosted_billing_json_url = "/json/self-hosted-billing"
 
-        with self.settings(PUSH_NOTIFICATION_BOUNCER_URL=None):
+        with self.settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False):
             with self.settings(CORPORATE_ENABLED=True):
                 result = self.client_get(self_hosted_billing_url)
                 self.assertEqual(result.status_code, 404)
@@ -278,13 +277,13 @@ class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase):
         )
 
 
-@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+@activate_push_notification_service()
 class RemoteBillingAuthenticationTest(RemoteRealmBillingTestCase):
     def test_self_hosted_config_error_page(self) -> None:
         self.login("desdemona")
 
         with (
-            self.settings(CORPORATE_ENABLED=False, PUSH_NOTIFICATION_BOUNCER_URL=None),
+            self.settings(CORPORATE_ENABLED=False, ZULIP_SERVICE_PUSH_NOTIFICATIONS=False),
             self.assertLogs("django.request"),
         ):
             result = self.client_get("/self-hosted-billing/not-configured/")
@@ -299,7 +298,7 @@ class RemoteBillingAuthenticationTest(RemoteRealmBillingTestCase):
             self.assertEqual(result.status_code, 404)
 
         # Also doesn't make sense on zulipchat.com (where CORPORATE_ENABLED is True).
-        with self.settings(CORPORATE_ENABLED=True, PUSH_NOTIFICATION_BOUNCER_URL=None):
+        with self.settings(CORPORATE_ENABLED=True, ZULIP_SERVICE_PUSH_NOTIFICATIONS=False):
             result = self.client_get("/self-hosted-billing/not-configured/")
             self.assertEqual(result.status_code, 404)
 

corporate/tests/test_stripe.py

@@ -21,7 +21,6 @@ import stripe
 import time_machine
 from django.conf import settings
 from django.core import signing
-from django.test import override_settings
 from django.urls.resolvers import get_resolver
 from django.utils.crypto import get_random_string
 from django.utils.timezone import now as timezone_now
@@ -90,6 +89,7 @@ from zerver.actions.realm_settings import do_deactivate_realm, do_reactivate_rea
 from zerver.actions.users import do_deactivate_user
 from zerver.lib.remote_server import send_server_data_to_push_bouncer
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import activate_push_notification_service
 from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
 from zerver.lib.utils import assert_is_not_none
 from zerver.models import Message, Realm, RealmAuditLog, Recipient, UserProfile
@@ -6579,7 +6579,7 @@ class TestRemoteBillingWriteAuditLog(StripeTestCase):
             )
 
 
-@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+@activate_push_notification_service()
 class TestRemoteRealmBillingFlow(StripeTestCase, RemoteRealmBillingTestCase):
     @override
     def setUp(self) -> None:
@@ -8308,7 +8308,7 @@ class TestRemoteRealmBillingFlow(StripeTestCase, RemoteRealmBillingTestCase):
             self.assertEqual(invoice_item1[key], value)
 
 
-@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+@activate_push_notification_service()
 class TestRemoteServerBillingFlow(StripeTestCase, RemoteServerTestCase):
     @override
     def setUp(self) -> None:

docs/overview/changelog.md

@@ -191,6 +191,29 @@ log][commit-log] for an up-to-date list of all changes.
   to give time to potentially reconfigure which channel to use. You can
   override the delay by running `./manage.py send_zulip_update_announcements --skip-delay`
   once you've done any necessary configuration updates.
+- We've reworked how Zulip's mobile push notifications service is
+  configured to be easier to understand, more extensible, and avoid
+  hardcoding URLs unnecessarily. The old settings names are fully
+  supported with identical behavior, so no action is required before
+  upgrading.
+
+  Once you've upgraded, while you're [updating your settings.py
+  documentation][update-settings-docs], we recommend updating
+  `/etc/zulip/settings.py` to use the modern settings names: Replacing
+  `PUSH_NOTIFICATIONS_BOUNCER_URL = "https://push.zulipchat.com"` with
+  `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` and renaming
+  `SUBMIT_USAGE_STATISTICS` to
+  `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS`, if you have either of those
+  settings enabled. It's important not to set both the old and new
+  settings: The modern settings will be ignored if the legacy ones are
+  present.
+
+  The one minor functional change in this restructuring is that it is
+  now possible to configure sharing usage statistics with the Zulip
+  developers without attempting to send mobile push notifications via
+  the service, by setting `ZULIP_SERVICE_PUSH_NOTIFICATIONS = False`
+  and `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=True`.
+
 - The Zulip server now contains a KaTeX server worker, designed to
   make bulk-rendering LaTeX efficient. It has minimal memory
   footprint, but can be disabled using the `katex_server` [deployment

docs/production/mobile-push-notifications.md

@@ -1,41 +1,57 @@
 # Mobile push notification service
 
-Zulip's iOS and Android [mobile apps](https://zulip.com/apps/) support receiving
+Zulip's iOS and Android [mobile apps](https://zulip.com/apps/) support
-push notifications from Zulip servers to let users know when new messages have
+receiving push notifications from Zulip servers to notify users when
-arrived. This is an important feature for having a great mobile app experience.
+new messages have arrived. This is an important feature for having a
+great mobile app experience.
 
-To set up mobile push notifications, you will need to register your Zulip server
+The security model for mobile push notifications does not allow
-with the Zulip mobile push notification service. This service will forward push
+self-hosted Zulip servers to directly send mobile notifications to the
-notifications generated by your server to users' mobile apps.
+Zulip mobile apps. The Zulip mobile push notification service solves
+this problem by forwarding mobile push notifications generated by your
+server to the Zulip mobile apps.
 
-## How to sign up
+## Signing up
 
 You can enable the mobile push notification service for your Zulip server as
 follows:
 
+1. Check that your [server
+   version](https://zulip.com/help/view-zulip-version) is has Zulip
+   Server 9.0 or greater. For older versions, see the [Zulip 8.x
+   documentation](https://zulip.readthedocs.io/en/8.4/production/mobile-push-notifications.html).
+
 1. Make sure your server has outgoing HTTPS access to the public Internet. If
    that is restricted by a proxy, you will need to [configure Zulip to use your
    outgoing HTTP proxy](deployment.md#customizing-the-outgoing-http-proxy)
    first.
 
+1. Set `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` in your
+   `/etc/zulip/settings.py` file. The [comments in
+   settings.py][update-settings-docs] should contain this line,
+   commented out with a `# `. Delete the `# ` at the start of the line
+   to enable the setting.
+
 1. Decide whether to share usage statistics with the Zulip team.
 
    By default, Zulip installations using the Mobile Push Notification
    Service submit additional usage statistics that help Zulip's
    maintainers allocate resources towards supporting self-hosted
-   installations ([details](#uploading-usage-statistics)). You can
+   installations ([details](#uploading-usage-statistics)).
-   disable submitting usage statistics now or at any time by setting
+
-   `SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`.
+   You can disable submitting usage statistics now or at any time by
+   setting `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=False` in
+   `/etc/zulip/settings.py` (the template contains a convenient
+   commented line that you can uncomment).
 
    Note that all systems using the service upload [basic
    metadata](#uploading-basic-metadata) about the organizations hosted
    by the installation.
 
-1. Uncomment the
+   [update-settings-docs]: ../production/upgrade.md#updating-settingspy-inline-documentation
-   `PUSH_NOTIFICATION_BOUNCER_URL = 'https://push.zulipchat.com'` line
+
-   in your `/etc/zulip/settings.py` file (i.e., remove the `#` at the
+1. [Restart your Zulip server](settings.md#making-changes) so that
-   start of the line), and [restart your Zulip
+   your configuration changes take effect.
-   server](settings.md#making-changes).
 
 1. Run the registration command. If you installed Zulip directly on the server
    (without Docker), run as root:
@@ -248,7 +264,7 @@ Push Notifications Service itself.
 By default, Zulip installations that register for the Mobile Push
 Notifications Service upload the following usage statistics. You can
 disable these uploads any time by setting
-`SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`.
+`ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`.
 
 - Totals for messages sent and read with subtotals for various
   combinations of clients and integrations.
@@ -261,8 +277,8 @@ statistics.
 
 When enabled, usage statistics are submitted via an hourly cron
 job. If you'd like to access plan management immediately after
-enabling `SUBMIT_USAGE_STATISTICS=True` on a pre-8.0 Zulip server, you
+enabling `SUBMIT_USAGE_STATISTICS=True` (the legacy form of this setting)
-can run the analytics job manually via:
+on a pre-8.0 Zulip server, you can run the analytics job manually via:
 
 ```
 /home/zulip/deployments/current/manage.py update_analytics_counts
@@ -325,7 +341,7 @@ registration.
    ```
 
 1. Comment out the
-   `PUSH_NOTIFICATION_BOUNCER_URL = 'https://push.zulipchat.com'` line
+   `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` line
    in your `/etc/zulip/settings.py` file (i.e., add `# ` at the
    start of the line), and [restart your Zulip
    server](settings.md#making-changes).

zerver/lib/push_notifications.py

@@ -248,7 +248,7 @@ def send_apple_push_notification(
     if apns_context is None:
         logger.debug(
             "APNs: Dropping a notification because nothing configured.  "
-            "Set PUSH_NOTIFICATION_BOUNCER_URL (or APNS_CERT_FILE)."
+            "Set ZULIP_SERVICES_URL (or APNS_CERT_FILE)."
         )
         return 0
 
@@ -455,7 +455,7 @@ def send_android_push_notification(
     if not fcm_app:
         logger.debug(
             "Skipping sending a FCM push notification since "
-            "PUSH_NOTIFICATION_BOUNCER_URL and ANDROID_FCM_CREDENTIALS_PATH are both unset"
+            "ZULIP_SERVICE_PUSH_NOTIFICATIONS and ANDROID_FCM_CREDENTIALS_PATH are both unset"
         )
         return 0
 
@@ -529,7 +529,7 @@ def send_android_push_notification(
 
 
 def uses_notification_bouncer() -> bool:
-    return settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+    return settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS is True
 
 
 def sends_notifications_directly() -> bool:

zerver/lib/remote_server.py

@@ -27,6 +27,7 @@ from zerver.lib.exceptions import (
 from zerver.lib.outgoing_http import OutgoingSession
 from zerver.lib.queue import queue_event_on_commit
 from zerver.lib.redis_utils import get_redis_client
+from zerver.lib.types import AnalyticsDataUploadLevel
 from zerver.models import Realm, RealmAuditLog
 from zerver.models.realms import OrgTypeEnum
 
@@ -140,10 +141,10 @@ def send_to_push_bouncer(
       vs. client-side errors like an invalid token.
 
     """
-    assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+    assert settings.ZULIP_SERVICES_URL is not None
     assert settings.ZULIP_ORG_ID is not None
     assert settings.ZULIP_ORG_KEY is not None
-    url = urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL, "/api/v1/remotes/" + endpoint)
+    url = urljoin(settings.ZULIP_SERVICES_URL, "/api/v1/remotes/" + endpoint)
     api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID, settings.ZULIP_ORG_KEY)
 
     headers = {"User-agent": f"ZulipServer/{ZULIP_VERSION}"}
@@ -286,7 +287,7 @@ def maybe_mark_pushes_disabled(
     if isinstance(e, JsonableError):
         logger.warning(e.msg)
     else:
-        logger.exception("Exception communicating with %s", settings.PUSH_NOTIFICATION_BOUNCER_URL)
+        logger.exception("Exception communicating with %s", settings.ZULIP_SERVICES_URL)
 
     # An exception was thrown talking to the push bouncer. There may
     # be certain transient failures that we could ignore here -
@@ -381,6 +382,10 @@ def get_realms_info_for_push_bouncer(realm_id: int | None = None) -> list[RealmD
     return realm_info_list
 
 
+def should_send_analytics_data() -> bool:  # nocoverage
+    return settings.ANALYTICS_DATA_UPLOAD_LEVEL > AnalyticsDataUploadLevel.NONE
+
+
 def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) -> None:
     logger = logging.getLogger("zulip.analytics")
     # first, check what's latest
@@ -396,7 +401,10 @@ def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) ->
     last_acked_installation_count_id = result["last_installation_count_id"]
     last_acked_realmauditlog_id = result["last_realmauditlog_id"]
 
-    if settings.SUBMIT_USAGE_STATISTICS and consider_usage_statistics:
+    if (
+        settings.ANALYTICS_DATA_UPLOAD_LEVEL == AnalyticsDataUploadLevel.ALL
+        and consider_usage_statistics
+    ):
         # Only upload usage statistics, which is relatively expensive,
         # if called from the analytics cron job and the server has
         # uploading such statistics enabled.
@@ -410,12 +418,20 @@ def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) ->
         installation_count_query = InstallationCount.objects.none()
         realm_count_query = RealmCount.objects.none()
 
+    if settings.ANALYTICS_DATA_UPLOAD_LEVEL >= AnalyticsDataUploadLevel.BILLING:
+        realmauditlog_query = RealmAuditLog.objects.filter(
+            event_type__in=RealmAuditLog.SYNCED_BILLING_EVENTS, id__gt=last_acked_realmauditlog_id
+        )
+    else:
+        realmauditlog_query = RealmAuditLog.objects.none()
+
+    # This code shouldn't be called at all if we're not configured to send any data.
+    assert settings.ANALYTICS_DATA_UPLOAD_LEVEL > AnalyticsDataUploadLevel.NONE
+
     (realm_count_data, installation_count_data, realmauditlog_data) = build_analytics_data(
         realm_count_query=realm_count_query,
         installation_count_query=installation_count_query,
-        realmauditlog_query=RealmAuditLog.objects.filter(
+        realmauditlog_query=realmauditlog_query,
-            event_type__in=RealmAuditLog.SYNCED_BILLING_EVENTS, id__gt=last_acked_realmauditlog_id
-        ),
     )
 
     record_count = len(realm_count_data) + len(installation_count_data) + len(realmauditlog_data)

zerver/lib/test_classes.py

@@ -2566,8 +2566,8 @@ class BouncerTestCase(ZulipTestCase):
             # we can safely pick the first value.
             data = {k: v[0] for k, v in params.items()}
         assert request.url is not None  # allow mypy to infer url is present.
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        local_url = request.url.replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "")
+        local_url = request.url.replace(settings.ZULIP_SERVICES_URL, "")
         if request.method == "POST":
             result = self.uuid_post(self.server_uuid, local_url, data, subdomain="", **kwargs)
         elif request.method == "GET":
@@ -2575,9 +2575,9 @@ class BouncerTestCase(ZulipTestCase):
         return (result.status_code, result.headers, result.content)
 
     def add_mock_response(self) -> None:
-        # Match any endpoint with the PUSH_NOTIFICATION_BOUNCER_URL.
+        # Match any endpoint with the ZULIP_SERVICES_URL.
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        COMPILED_URL = re.compile(settings.PUSH_NOTIFICATION_BOUNCER_URL + r".*")
+        COMPILED_URL = re.compile(settings.ZULIP_SERVICES_URL + r".*")
         responses.add_callback(responses.POST, COMPILED_URL, callback=self.request_callback)
         responses.add_callback(responses.GET, COMPILED_URL, callback=self.request_callback)
 

zerver/lib/test_helpers.py

@@ -37,6 +37,7 @@ from zerver.lib.integrations import WEBHOOK_INTEGRATIONS
 from zerver.lib.per_request_cache import flush_per_request_caches
 from zerver.lib.rate_limiter import RateLimitedIPAddr, rules
 from zerver.lib.request import RequestNotes
+from zerver.lib.types import AnalyticsDataUploadLevel
 from zerver.lib.upload.s3 import S3UploadBackend
 from zerver.models import Client, Message, RealmUserDefault, Subscription, UserMessage, UserProfile
 from zerver.models.clients import clear_client_cache, get_client
@@ -78,6 +79,53 @@ def stub_event_queue_user_events(
         yield
 
 
+class activate_push_notification_service(override_settings):  # noqa: N801
+    """
+    Activating the push notification service involves a few different settings
+    that are logically related, and ordinarily set correctly in computed_settings.py
+    based on the admin-configured settings.
+    Having tests deal with overriding all the necessary settings every time they
+    want to simulate using the push notification service would be too
+    cumbersome, so we provide a convenient helper.
+    Can be used as either a context manager or a decorator applied to a test method
+    or class, just like original override_settings.
+    """
+
+    def __init__(
+        self, zulip_services_url: str | None = None, submit_usage_statistics: bool = False
+    ) -> None:
+        if zulip_services_url is None:
+            zulip_services_url = settings.ZULIP_SERVICES_URL
+        assert zulip_services_url is not None
+
+        # Ordinarily the ANALYTICS_DATA_UPLOAD_LEVEL setting is computed based on these
+        # ZULIP_SERVICE_* configured settings; but because these settings here won't get
+        # processed through computed_settings, we need to set ANALYTICS_DATA_UPLOAD_LEVEL
+        # manually.
+        # The logic here is:
+        # (1) If the currently active ANALYTICS_DATA_UPLOAD_LEVEL is lower than what's
+        #     demanded to enable push notifications, then we need to override it to
+        #     this minimum level (i.e. BILLING).
+        # (2) Otherwise, the test must have already somehow set up a higher level
+        #     of data upload, so we should leave it alone.
+        if settings.ANALYTICS_DATA_UPLOAD_LEVEL < AnalyticsDataUploadLevel.BILLING:
+            analytics_data_upload_level = AnalyticsDataUploadLevel.BILLING
+        else:  # nocoverage
+            analytics_data_upload_level = settings.ANALYTICS_DATA_UPLOAD_LEVEL
+
+        # Finally, the data upload level can be elevated by the submit_usage_statistics
+        # argument.
+        if submit_usage_statistics:
+            analytics_data_upload_level = AnalyticsDataUploadLevel.ALL
+
+        super().__init__(
+            ZULIP_SERVICES_URL=zulip_services_url,
+            ANALYTICS_DATA_UPLOAD_LEVEL=analytics_data_upload_level,
+            ZULIP_SERVICE_PUSH_NOTIFICATIONS=True,
+            ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=submit_usage_statistics,
+        )
+
+
 @contextmanager
 def cache_tries_captured() -> Iterator[list[tuple[str, str | list[str], str | None]]]:
     cache_queries: list[tuple[str, str | list[str], str | None]] = []

zerver/lib/types.py

@@ -1,6 +1,7 @@
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from datetime import datetime
+from enum import IntEnum
 from typing import Any, TypeAlias, TypeVar
 
 from django_stubs_ext import StrPromise
@@ -326,3 +327,10 @@ class RawUserDict(TypedDict):
 class RemoteRealmDictValue(TypedDict):
     can_push: bool
     expected_end_timestamp: int | None
+
+
+class AnalyticsDataUploadLevel(IntEnum):
+    NONE = 0
+    BASIC = 1
+    BILLING = 2
+    ALL = 3

zerver/management/commands/register_server.py

@@ -58,16 +58,14 @@ class Command(ZulipBaseCommand):
             raise CommandError(
                 "Missing zulip_org_key; run scripts/setup/generate_secrets.py to generate."
             )
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL is None:
+        if not settings.ZULIP_SERVICES_URL:
-            if settings.DEVELOPMENT:
+            raise CommandError(
-                settings.PUSH_NOTIFICATION_BOUNCER_URL = (
+                "ZULIP_SERVICES_URL is not set; was the default incorrectly overridden in /etc/zulip/settings.py?"
-                    settings.EXTERNAL_URI_SCHEME + settings.EXTERNAL_HOST
+            )
-                )
+        if not settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS:
-            else:
+            raise CommandError(
-                raise CommandError(
+                "Please set ZULIP_SERVICE_PUSH_NOTIFICATIONS to True in /etc/zulip/settings.py"
-                    "Please uncomment PUSH_NOTIFICATION_BOUNCER_URL "
+            )
-                    "in /etc/zulip/settings.py (remove the '#')"
-                )
 
         if options["deactivate"]:
             send_json_to_push_bouncer("POST", "server/deactivate", {})
@@ -139,15 +137,15 @@ class Command(ZulipBaseCommand):
             print("Mobile Push Notification Service registration successfully updated!")
 
     def _request_push_notification_bouncer_url(self, url: str, params: dict[str, Any]) -> Response:
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        registration_url = settings.PUSH_NOTIFICATION_BOUNCER_URL + url
+        registration_url = settings.ZULIP_SERVICES_URL + url
         session = PushBouncerSession()
         try:
             response = session.post(registration_url, data=params)
         except requests.RequestException:
             raise CommandError(
                 "Network error connecting to push notifications service "
-                f"({settings.PUSH_NOTIFICATION_BOUNCER_URL})",
+                f"({settings.ZULIP_SERVICES_URL})",
             )
         try:
             response.raise_for_status()

zerver/tests/test_home.py

@@ -23,7 +23,11 @@ from zerver.lib.home import (
 )
 from zerver.lib.soft_deactivation import do_soft_deactivate_users
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import get_user_messages, queries_captured
+from zerver.lib.test_helpers import (
+    activate_push_notification_service,
+    get_user_messages,
+    queries_captured,
+)
 from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.models import DefaultStream, Draft, Realm, UserActivity, UserProfile
 from zerver.models.realms import get_realm
@@ -973,7 +977,7 @@ class HomeTest(ZulipTestCase):
         self.assertEqual(page_params["narrow"], [dict(operator="stream", operand=stream_name)])
         self.assertEqual(page_params["state_data"]["max_message_id"], -1)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     def test_get_billing_info(self) -> None:
         user = self.example_user("desdemona")
         user.role = UserProfile.ROLE_REALM_OWNER
@@ -1129,7 +1133,7 @@ class HomeTest(ZulipTestCase):
         # If the server doesn't have the push bouncer configured,
         # remote billing should be shown anyway, as the billing endpoint
         # is supposed show a useful error page.
-        with self.settings(PUSH_NOTIFICATION_BOUNCER_URL=None, CORPORATE_ENABLED=False):
+        with self.settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False, CORPORATE_ENABLED=False):
             billing_info = get_billing_info(user)
         self.assertTrue(billing_info.show_remote_billing)
 

zerver/tests/test_import_export.py

@@ -12,7 +12,6 @@ import orjson
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db.models import Q, QuerySet
-from django.test import override_settings
 from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
@@ -47,6 +46,7 @@ from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids
 from zerver.lib.streams import create_stream_if_needed
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import (
+    activate_push_notification_service,
     create_s3_buckets,
     get_test_image_file,
     most_recent_message,
@@ -1542,7 +1542,7 @@ class RealmImportExportTest(ExportFile):
         self.assertEqual(realm_user_default.default_language, "en")
         self.assertEqual(realm_user_default.twenty_four_hour_time, False)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     def test_import_realm_notify_bouncer(self) -> None:
         original_realm = Realm.objects.get(string_id="zulip")
 

zerver/tests/test_push_notifications.py

@@ -83,10 +83,12 @@ from zerver.lib.remote_server import (
 from zerver.lib.response import json_response_from_error
 from zerver.lib.test_classes import BouncerTestCase, ZulipTestCase
 from zerver.lib.test_helpers import (
+    activate_push_notification_service,
     mock_queue_publish,
     reset_email_visibility_to_everyone_in_zulip_realm,
 )
 from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.types import AnalyticsDataUploadLevel
 from zerver.lib.user_counts import realm_user_count_by_role
 from zerver.models import (
     Message,
@@ -121,7 +123,7 @@ if settings.ZILENCER_ENABLED:
 
 
 class SendTestPushNotificationEndpointTest(BouncerTestCase):
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_send_test_push_notification_api_invalid_token(self) -> None:
         # What happens when the mobile device isn't registered with its server,
@@ -167,7 +169,7 @@ class SendTestPushNotificationEndpointTest(BouncerTestCase):
         error_response = json_response_from_error(InvalidRemotePushDeviceTokenError())
         responses.add(
             responses.POST,
-            f"{settings.PUSH_NOTIFICATION_BOUNCER_URL}/api/v1/remotes/push/test_notification",
+            f"{settings.ZULIP_SERVICES_URL}/api/v1/remotes/push/test_notification",
             body=error_response.content,
             status=error_response.status_code,
         )
@@ -293,7 +295,7 @@ class SendTestPushNotificationEndpointTest(BouncerTestCase):
         )
         self.assert_json_success(result)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_send_test_push_notification_api_with_bouncer_config(self) -> None:
         """
@@ -1104,9 +1106,7 @@ class PushBouncerNotificationTest(BouncerTestCase):
             )
 
         with (
-            mock.patch(
+            activate_push_notification_service(),
-                "zerver.lib.push_notifications.uses_notification_bouncer", return_value=True
-            ),
             mock.patch("zerver.lib.remote_server.send_to_push_bouncer") as m,
         ):
             post_response = {
@@ -1131,7 +1131,7 @@ class PushBouncerNotificationTest(BouncerTestCase):
             self.assertTrue(realm.push_notifications_enabled)
             self.assertEqual(realm.push_notifications_enabled_end_timestamp, None)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_register_token_realm_uuid_belongs_to_different_server(self) -> None:
         self.add_mock_response()
@@ -1176,7 +1176,7 @@ class PushBouncerNotificationTest(BouncerTestCase):
 
         self.assert_length(RemotePushDeviceToken.objects.filter(token=token), 0)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_push_bouncer_api(self) -> None:
         """This is a variant of the below test_push_api, but using the full
@@ -1221,8 +1221,8 @@ class PushBouncerNotificationTest(BouncerTestCase):
             result = self.client_delete(endpoint, {"token": "abcd1234"}, subdomain="zulip")
             self.assert_json_error(result, "Token does not exist")
 
-            assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+            assert settings.ZULIP_SERVICES_URL is not None
-            URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/register"
+            URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/register"
             with responses.RequestsMock() as resp, self.assertLogs(level="ERROR") as error_log:
                 resp.add(responses.POST, URL, body=ConnectionError(), status=502)
                 with self.assertRaisesRegex(
@@ -1382,11 +1382,11 @@ class AnalyticsBouncerTest(BouncerTestCase):
 
         return super().setUp()
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_analytics_failure_api(self) -> None:
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics"
+        ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics"
         ANALYTICS_STATUS_URL = ANALYTICS_URL + "/status"
 
         with (
@@ -1447,7 +1447,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
             send_server_data_to_push_bouncer()
             self.assertTrue(
                 mock_warning.output[0].startswith(
-                    f"ERROR:zulip.analytics:Exception communicating with {settings.PUSH_NOTIFICATION_BOUNCER_URL}\nTraceback",
+                    f"ERROR:zulip.analytics:Exception communicating with {settings.ZULIP_SERVICES_URL}\nTraceback",
                 )
             )
             self.assertTrue(resp.assert_call_count(ANALYTICS_STATUS_URL, 1))
@@ -1511,14 +1511,14 @@ class AnalyticsBouncerTest(BouncerTestCase):
             self.assertTrue(resp.assert_call_count(ANALYTICS_URL, 1))
             self.assertPushNotificationsAre(False)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service(submit_usage_statistics=True)
     @responses.activate
     def test_analytics_api(self) -> None:
         """This is a variant of the below test_push_api, but using the full
         push notification bouncer flow
         """
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics"
+        ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics"
         ANALYTICS_STATUS_URL = ANALYTICS_URL + "/status"
         user = self.example_user("hamlet")
         end_time = self.TIME_ZERO
@@ -1620,13 +1620,13 @@ class AnalyticsBouncerTest(BouncerTestCase):
         self.assertEqual(InstallationCount.objects.count(), 2)
         self.assertEqual(RealmAuditLog.objects.filter(id__gt=audit_log_max_id).count(), 2)
 
-        with self.settings(SUBMIT_USAGE_STATISTICS=False):
+        with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.BILLING):
-            # With this setting off, we don't send RealmCounts and InstallationCounts.
+            # With this setting, we don't send RealmCounts and InstallationCounts.
             send_server_data_to_push_bouncer()
         check_counts(2, 2, 0, 0, 1)
 
-        with self.settings(SUBMIT_USAGE_STATISTICS=True):
+        with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.ALL):
-            # With 'SUBMIT_USAGE_STATISTICS=True' but 'consider_usage_statistics=False',
+            # With ALL data upload enabled, but 'consider_usage_statistics=False',
             # we don't send RealmCount and InstallationCounts.
             send_server_data_to_push_bouncer(consider_usage_statistics=False)
         check_counts(3, 3, 0, 0, 1)
@@ -1837,8 +1837,15 @@ class AnalyticsBouncerTest(BouncerTestCase):
                 RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user.realm),
             },
         )
+        with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.BASIC):
+            # With the BASIC level, RealmAuditLog rows are not sent.
+            send_server_data_to_push_bouncer()
+        check_counts(10, 10, 3, 2, 7)
+
+        # Now, with ANALYTICS_DATA_UPLOAD_LEVEL back to the baseline for this test,
+        # the new RealmAuditLog event will be sent.
         send_server_data_to_push_bouncer()
-        check_counts(10, 10, 3, 2, 8)
+        check_counts(11, 11, 3, 2, 8)
 
         # Now create an InstallationCount with a property that's not supposed
         # to be tracked by the remote server - since the bouncer itself tracks
@@ -1858,7 +1865,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
         )
         # The analytics endpoint call counts increase by 1, but the actual RemoteCounts remain unchanged,
         # since syncing the data failed.
-        check_counts(11, 11, 3, 2, 8)
+        check_counts(12, 12, 3, 2, 8)
         forbidden_installation_count.delete()
 
         (realm_count_data, installation_count_data, realmauditlog_data) = build_analytics_data(
@@ -1899,7 +1906,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
             ],
         )
         # Only the request counts go up -- all of the other rows' duplicates are dropped
-        check_counts(12, 12, 3, 2, 8)
+        check_counts(13, 13, 3, 2, 8)
 
         # Test that only valid org_type values are accepted - integers defined in OrgTypeEnum.
         realms_data = get_realms_info_for_push_bouncer()
@@ -1927,7 +1934,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
             result, 'Invalid realms[0]["org_type"]: Value error, Not a valid org_type value'
         )
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service(submit_usage_statistics=True)
     @responses.activate
     def test_analytics_api_foreign_keys_to_remote_realm(self) -> None:
         self.add_mock_response()
@@ -2075,7 +2082,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
         for remote_realm_audit_log in RemoteRealmAuditLog.objects.filter(realm_id=user.realm.id):
             self.assertEqual(remote_realm_audit_log.remote_realm, remote_realm)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service(submit_usage_statistics=True)
     @responses.activate
     def test_analytics_api_invalid(self) -> None:
         """This is a variant of the below test_push_api, but using the full
@@ -2098,7 +2105,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
         self.assertEqual(m.output, ["WARNING:zulip.analytics:Invalid property invalid count stat"])
         self.assertEqual(RemoteRealmCount.objects.count(), 0)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_remote_realm_duplicate_uuid(self) -> None:
         """
@@ -2141,7 +2148,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
 
     # Servers on Zulip 2.0.6 and earlier only send realm_counts and installation_counts data,
     # and don't send realmauditlog_rows. Make sure that continues to work.
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_old_two_table_format(self) -> None:
         self.add_mock_response()
@@ -2155,15 +2162,15 @@ class AnalyticsBouncerTest(BouncerTestCase):
                 "version": '"2.0.6+git"',
             },
         )
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics"
+        ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics"
         self.assertTrue(responses.assert_call_count(ANALYTICS_URL, 1))
         self.assertEqual(RemoteRealmCount.objects.count(), 1)
         self.assertEqual(RemoteInstallationCount.objects.count(), 0)
         self.assertEqual(RemoteRealmAuditLog.objects.count(), 0)
 
     # Make sure we aren't sending data we don't mean to, even if we don't store it.
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_only_sending_intended_realmauditlog_data(self) -> None:
         self.add_mock_response()
@@ -2209,7 +2216,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
         ):
             send_server_data_to_push_bouncer()
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_realmauditlog_data_mapping(self) -> None:
         self.add_mock_response()
@@ -2236,7 +2243,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
 
     # This verifies that the bouncer is backwards-compatible with remote servers using
     # TextField to store extra_data.
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_realmauditlog_string_extra_data(self) -> None:
         self.add_mock_response()
@@ -2337,7 +2344,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
             )
         self.assertIn("Malformed audit log data", m.output[0])
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_realm_properties_after_send_analytics(self) -> None:
         self.add_mock_response()
@@ -2647,7 +2654,7 @@ class AnalyticsBouncerTest(BouncerTestCase):
             ],
         )
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_deleted_realm(self) -> None:
         self.add_mock_response()
@@ -2886,15 +2893,15 @@ class HandlePushNotificationTest(PushNotificationTest):
     @override
     def request_callback(self, request: PreparedRequest) -> tuple[int, ResponseHeaders, bytes]:
         assert request.url is not None  # allow mypy to infer url is present.
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        local_url = request.url.replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "")
+        local_url = request.url.replace(settings.ZULIP_SERVICES_URL, "")
         assert isinstance(request.body, bytes)
         result = self.uuid_post(
             self.server_uuid, local_url, request.body, content_type="application/json"
         )
         return (result.status_code, result.headers, result.content)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_end_to_end(self) -> None:
         self.add_mock_response()
@@ -2988,7 +2995,7 @@ class HandlePushNotificationTest(PushNotificationTest):
                 ),
             )
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_end_to_end_failure_due_to_no_plan(self) -> None:
         self.add_mock_response()
@@ -3062,7 +3069,7 @@ class HandlePushNotificationTest(PushNotificationTest):
             self.assertEqual(realm.push_notifications_enabled, True)
             self.assertEqual(realm.push_notifications_enabled_end_timestamp, None)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_unregistered_client(self) -> None:
         self.add_mock_response()
@@ -3171,7 +3178,7 @@ class HandlePushNotificationTest(PushNotificationTest):
             # Local registrations have also been deleted:
             self.assertEqual(PushDeviceToken.objects.filter(kind=PushDeviceToken.APNS).count(), 0)
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     @responses.activate
     def test_connection_error(self) -> None:
         self.setup_apns_tokens()
@@ -3192,13 +3199,14 @@ class HandlePushNotificationTest(PushNotificationTest):
             "message_id": message.id,
             "trigger": NotificationTriggers.DIRECT_MESSAGE,
         }
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/notify"
+        URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/notify"
         responses.add(responses.POST, URL, body=ConnectionError())
         with self.assertRaises(PushNotificationBouncerRetryLaterError):
             handle_push_notification(self.user_profile.id, missed_message)
 
     @mock.patch("zerver.lib.push_notifications.push_notifications_configured", return_value=True)
+    @override_settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False, ZULIP_SERVICES=set())
     def test_read_message(self, mock_push_notifications: mock.MagicMock) -> None:
         user_profile = self.example_user("hamlet")
         message = self.get_message(
@@ -3339,7 +3347,7 @@ class HandlePushNotificationTest(PushNotificationTest):
             "trigger": NotificationTriggers.DIRECT_MESSAGE,
         }
         with (
-            self.settings(PUSH_NOTIFICATION_BOUNCER_URL=True),
+            activate_push_notification_service(),
             mock.patch(
                 "zerver.lib.push_notifications.get_message_payload_apns",
                 return_value={"apns": True},
@@ -3472,7 +3480,7 @@ class HandlePushNotificationTest(PushNotificationTest):
         )
 
         with (
-            self.settings(PUSH_NOTIFICATION_BOUNCER_URL=True),
+            activate_push_notification_service(),
             mock.patch("zerver.lib.push_notifications.send_notifications_to_bouncer") as mock_send,
         ):
             handle_remove_push_notification(user_profile.id, [message.id])
@@ -3951,7 +3959,7 @@ class TestAPNs(PushNotificationTest):
             notification_drop_log = (
                 "DEBUG:zerver.lib.push_notifications:"
                 "APNs: Dropping a notification because nothing configured.  "
-                "Set PUSH_NOTIFICATION_BOUNCER_URL (or APNS_CERT_FILE)."
+                "Set ZULIP_SERVICES_URL (or APNS_CERT_FILE)."
             )
 
             from zerver.lib.push_notifications import initialize_push_notifications
@@ -4733,13 +4741,13 @@ class TestSendNotificationsToBouncer(PushNotificationTest):
         self.assertEqual(user.realm.push_notifications_enabled, False)
 
 
-@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+@activate_push_notification_service()
 class TestSendToPushBouncer(ZulipTestCase):
     def add_mock_response(
         self, body: bytes = orjson.dumps({"msg": "error"}), status: int = 200
     ) -> None:
-        assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+        assert settings.ZULIP_SERVICES_URL is not None
-        URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/register"
+        URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/register"
         responses.add(responses.POST, URL, body=body, status=status)
 
     @responses.activate
@@ -4829,11 +4837,11 @@ class TestPushApi(BouncerTestCase):
 
             # Use push notification bouncer and try to remove non-existing tokens.
             with (
-                self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"),
+                activate_push_notification_service(),
                 responses.RequestsMock() as resp,
             ):
-                assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
+                assert settings.ZULIP_SERVICES_URL is not None
-                URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/unregister"
+                URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/unregister"
                 resp.add_callback(responses.POST, URL, callback=self.request_callback)
                 result = self.client_delete(endpoint, {"token": "abcd1234"})
                 self.assert_json_error(result, "Token does not exist")
@@ -4867,7 +4875,7 @@ class TestPushApi(BouncerTestCase):
             self.assert_length(tokens, 1)
             self.assertEqual(tokens[0].token, token)
 
-        with self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"):
+        with activate_push_notification_service():
             self.add_mock_response()
             # Enable push notification bouncer and add tokens.
             for endpoint, token, appid in bouncer_requests:
@@ -4908,7 +4916,7 @@ class TestPushApi(BouncerTestCase):
 
         # Use push notification bouncer and test removing device tokens.
         # Tokens will be removed both locally and remotely.
-        with self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"):
+        with activate_push_notification_service():
             for endpoint, token, appid in bouncer_requests:
                 result = self.client_delete(endpoint, {"token": token})
                 self.assert_json_success(result)
@@ -4965,7 +4973,7 @@ class FCMSendTest(PushNotificationTest):
             send_android_push_notification_to_user(self.user_profile, {}, {})
             self.assertEqual(
                 "DEBUG:zerver.lib.push_notifications:"
-                "Skipping sending a FCM push notification since PUSH_NOTIFICATION_BOUNCER_URL "
+                "Skipping sending a FCM push notification since ZULIP_SERVICE_PUSH_NOTIFICATIONS "
                 "and ANDROID_FCM_CREDENTIALS_PATH are both unset",
                 logger.output[0],
             )

zerver/tests/test_realm.py

@@ -42,6 +42,7 @@ from zerver.lib.realm_description import get_realm_rendered_description, get_rea
 from zerver.lib.send_email import send_future_email
 from zerver.lib.streams import create_stream_if_needed
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import activate_push_notification_service
 from zerver.lib.upload import delete_message_attachments, upload_message_attachment
 from zerver.models import (
     Attachment,
@@ -1370,7 +1371,7 @@ class RealmTest(ZulipTestCase):
         ]
         self.assertEqual(sorted(user_group_names), sorted(expected_system_group_names))
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
+    @activate_push_notification_service()
     def test_do_create_realm_notify_bouncer(self) -> None:
         dummy_send_realms_only_response = {
             "result": "success",

zilencer/management/commands/populate_db.py

@@ -44,7 +44,7 @@ from zerver.lib.remote_server import get_realms_info_for_push_bouncer
 from zerver.lib.server_initialization import create_internal_realm, create_users
 from zerver.lib.storage import static_path
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
-from zerver.lib.types import ProfileFieldData
+from zerver.lib.types import AnalyticsDataUploadLevel, ProfileFieldData
 from zerver.lib.users import add_service
 from zerver.lib.utils import generate_api_key
 from zerver.models import (
@@ -79,7 +79,10 @@ from zilencer.views import update_remote_realm_data_for_server
 
 # Disable the push notifications bouncer to avoid enqueuing updates in
 # maybe_enqueue_audit_log_upload during early setup.
-settings.PUSH_NOTIFICATION_BOUNCER_URL = None
+settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS = False
+settings.ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False
+settings.ZULIP_SERVICE_SECURITY_ALERTS = False
+settings.ANALYTICS_DATA_UPLOAD_LEVEL = AnalyticsDataUploadLevel.NONE
 settings.USING_TORNADO = False
 # Disable using memcached caches to avoid 'unsupported pickle
 # protocol' errors if `populate_db` is run with a different Python

zproject/computed_settings.py

@@ -8,6 +8,7 @@ from urllib.parse import urljoin
 
 from scripts.lib.zulip_tools import get_tornado_ports
 from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor
+from zerver.lib.types import AnalyticsDataUploadLevel
 
 from .config import (
     DEPLOY_ROOT,
@@ -43,6 +44,7 @@ from .configured_settings import (
     LOCAL_UPLOADS_DIR,
     MEMCACHED_LOCATION,
     MEMCACHED_USERNAME,
+    PUSH_NOTIFICATION_BOUNCER_URL,
     RATE_LIMITING_RULES,
     REALM_HOSTS,
     REGISTER_LINK_DISABLED,
@@ -61,9 +63,14 @@ from .configured_settings import (
     SOCIAL_AUTH_SAML_SECURITY_CONFIG,
     SOCIAL_AUTH_SUBDOMAIN,
     STATIC_URL,
+    SUBMIT_USAGE_STATISTICS,
     TORNADO_PORTS,
     USING_PGROONGA,
     ZULIP_ADMINISTRATOR,
+    ZULIP_SERVICE_PUSH_NOTIFICATIONS,
+    ZULIP_SERVICE_SECURITY_ALERTS,
+    ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS,
+    ZULIP_SERVICES_URL,
 )
 
 ########################################################################
@@ -90,6 +97,68 @@ SERVER_GENERATION = int(time.time())
 ZULIP_ORG_KEY = get_secret("zulip_org_key")
 ZULIP_ORG_ID = get_secret("zulip_org_id")
 
+
+service_name_to_required_upload_level = {
+    "security_alerts": AnalyticsDataUploadLevel.BASIC,
+    "mobile_push": AnalyticsDataUploadLevel.BILLING,
+    "submit_usage_statistics": AnalyticsDataUploadLevel.ALL,
+}
+
+services: list[str] | None = None
+
+
+def services_append(service_name: str) -> None:
+    global services
+    if services is None:
+        services = []
+    services.append(service_name)
+
+
+if ZULIP_SERVICE_PUSH_NOTIFICATIONS:
+    services_append("mobile_push")
+    if ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS is None:
+        # This setting has special behavior where we want to activate
+        # it by default when push notifications are enabled - unless
+        # explicitly set otherwise in the config.
+        ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True
+
+if ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS:
+    services_append("submit_usage_statistics")
+if ZULIP_SERVICE_SECURITY_ALERTS:
+    services_append("security_alerts")
+
+if services is None and PUSH_NOTIFICATION_BOUNCER_URL is not None:
+    # ZULIP_SERVICE_* are the new settings that control the services
+    # enabled by the server, which in turn dictate the level of data
+    # uploaded to ZULIP_SERVICES_URL.
+    # As some older servers, predating the transition to the ZULIP_SERVICE_*
+    # settings, may have upgraded without redoing this part of their config,
+    # we need this block to set this level correctly based on the
+    # legacy settings.
+
+    # This is a setting that some servers from before 9.0 may have configured
+    # instead of the new ZULIP_SERVICE_* settings.
+    # Translate it to a correct configuration.
+    ZULIP_SERVICE_PUSH_NOTIFICATIONS = True
+    services_append("mobile_push")
+    ZULIP_SERVICES_URL = PUSH_NOTIFICATION_BOUNCER_URL
+    if SUBMIT_USAGE_STATISTICS:
+        ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True
+        services_append("submit_usage_statistics")
+
+if services is not None and set(services).intersection(
+    {"submit_usage_statistics", "security_alerts", "mobile_push"}
+):
+    # None of these make sense enabled without ZULIP_SERVICES_URL.
+    assert (
+        ZULIP_SERVICES_URL is not None
+    ), "ZULIP_SERVICES_URL is required when any services are enabled."
+
+ANALYTICS_DATA_UPLOAD_LEVEL = max(
+    [service_name_to_required_upload_level[service] for service in (services or [])],
+    default=AnalyticsDataUploadLevel.NONE,
+)
+
 if DEBUG:
     INTERNAL_IPS = ("127.0.0.1",)
 

zproject/default_settings.py

@@ -218,9 +218,29 @@ NAME_CHANGES_DISABLED = False
 AVATAR_CHANGES_DISABLED = False
 PASSWORD_MIN_LENGTH = 6
 PASSWORD_MIN_GUESSES = 10000
-PUSH_NOTIFICATION_BOUNCER_URL: str | None = None
+
+ZULIP_SERVICES_URL = "https://push.zulipchat.com"
+ZULIP_SERVICE_PUSH_NOTIFICATIONS = False
+
+# For this setting, we need to have None as the default value, so
+# that we can distinguish between the case of the setting not being
+# set at all and being disabled (set to False).
+# That's because unless the setting is explicitly configured, we want to
+# enable it in computed_settings when ZULIP_SERVICE_PUSH_NOTIFICATIONS
+# is enabled.
+ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS: bool | None = None
+ZULIP_SERVICE_SECURITY_ALERTS = False
+
 PUSH_NOTIFICATION_REDACT_CONTENT = False
+
+# Old setting kept around for backwards compatibility. Some old servers
+# may have it in their settings.py.
+PUSH_NOTIFICATION_BOUNCER_URL: str | None = None
+# Keep this default True, so that legacy deployments that configured PUSH_NOTIFICATION_BOUNCER_URL
+# without overriding SUBMIT_USAGE_STATISTICS get the original behavior. If a server configures
+# the modern ZULIP_SERVICES setting, all this will be ignored.
 SUBMIT_USAGE_STATISTICS = True
+
 PROMOTE_SPONSORING_ZULIP = True
 RATE_LIMITING = True
 RATE_LIMITING_AUTHENTICATE = True

zproject/dev_settings.py

@@ -205,7 +205,10 @@ SCIM_CONFIG: dict[str, SCIMConfigDict] = {
 
 SELF_HOSTING_MANAGEMENT_SUBDOMAIN = "selfhosting"
 DEVELOPMENT_DISABLE_PUSH_BOUNCER_DOMAIN_CHECK = True
-PUSH_NOTIFICATION_BOUNCER_URL = f"http://push.{EXTERNAL_HOST}"
+ZULIP_SERVICES_URL = f"http://push.{EXTERNAL_HOST}"
+
+ZULIP_SERVICE_PUSH_NOTIFICATIONS = True
+ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True
 
 # Breaks the UI if used, but enabled for development environment testing.
 ALLOW_GROUP_VALUED_SETTINGS = True

zproject/prod_settings_template.py

@@ -734,12 +734,19 @@ SOCIAL_AUTH_SAML_SUPPORT_CONTACT = {
 ## How long outgoing webhook requests time out after
 # OUTGOING_WEBHOOK_TIMEOUT_SECONDS = 10
 
-## Support for mobile push notifications.  Setting controls whether
+## Mobile push notifications require registering for the Zulip mobile
-## push notifications will be forwarded through a Zulip push
+## push notification service and configuring your server to use the
-## notification bouncer server to the mobile apps.  See
+## service here. For complete documentation, see:
-## https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html
+##
-## for information on how to sign up for and configure this.
+##   https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html
-# PUSH_NOTIFICATION_BOUNCER_URL = "https://push.zulipchat.com"
+##
+# ZULIP_SERVICE_PUSH_NOTIFICATIONS = True
+
+## By default, a Zulip server that has registered for Zulip services
+## submits both basic metadata (required for billing/free plan
+## eligiblity) as well as aggregate usage statistics. You can disable
+## submitting usage statistics here.
+# ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False
 
 ## Whether to redact the content of push notifications.  This is less
 ## usable, but avoids sending message content over the wire.  In the
@@ -747,13 +754,6 @@ SOCIAL_AUTH_SAML_SUPPORT_CONTACT = {
 ## notification encryption feature.
 # PUSH_NOTIFICATION_REDACT_CONTENT = False
 
-## Whether to submit basic usage statistics to help the Zulip core team.  Details at
-##
-##   https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html
-##
-## Defaults to True if and only if the Mobile Push Notifications Service is enabled.
-# SUBMIT_USAGE_STATISTICS = True
-
 ## Whether to lightly advertise sponsoring Zulip in the gear menu.
 # PROMOTE_SPONSORING_ZULIP = True
 

zproject/test_extra_settings.py

@@ -4,6 +4,7 @@ import ldap
 from django_auth_ldap.config import LDAPSearch
 
 from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor
+from zerver.lib.types import AnalyticsDataUploadLevel
 from zproject.settings_types import OIDCIdPConfigDict, SAMLIdPConfigDict, SCIMConfigDict
 
 from .config import DEPLOY_ROOT, get_from_file_if_exists
@@ -201,9 +202,23 @@ BIG_BLUE_BUTTON_URL = "https://bbb.example.com/bigbluebutton/"
 # By default two factor authentication is disabled in tests.
 # Explicitly set this to True within tests that must have this on.
 TWO_FACTOR_AUTHENTICATION_ENABLED = False
-PUSH_NOTIFICATION_BOUNCER_URL: str | None = None
 DEVELOPMENT_DISABLE_PUSH_BOUNCER_DOMAIN_CHECK = False
 
+# Disable all Zulip services by default. Tests can activate them by
+# overriding settings explicitly when they want to enable something,
+# often using activate_push_notification_service.
+ZULIP_SERVICE_PUSH_NOTIFICATIONS = False
+ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False
+ZULIP_SERVICE_SECURITY_ALERTS = False
+
+# Hack: This should be computed in computed_settings, but the transmission
+# of test settings overrides is wonky. See test_settings for more details.
+ANALYTICS_DATA_UPLOAD_LEVEL = AnalyticsDataUploadLevel.NONE
+
+# The most common value used by tests. Set it as the default so that it doesn't
+# have to be repeated every time.
+ZULIP_SERVICES_URL = "https://push.zulip.org.example.com"
+
 # Logging the emails while running the tests adds them
 # to /emails page.
 DEVELOPMENT_LOG_EMAILS = False