diff --git a/analytics/management/commands/update_analytics_counts.py b/analytics/management/commands/update_analytics_counts.py index 18de7989c5..ebae777b4a 100644 --- a/analytics/management/commands/update_analytics_counts.py +++ b/analytics/management/commands/update_analytics_counts.py @@ -11,7 +11,7 @@ from typing_extensions import override from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat from zerver.lib.management import ZulipBaseCommand, abort_unless_locked -from zerver.lib.remote_server import send_server_data_to_push_bouncer +from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data from zerver.lib.timestamp import floor_to_hour from zerver.models import Realm @@ -83,7 +83,10 @@ class Command(ZulipBaseCommand): ) logger.info("Finished updating analytics counts through %s", fill_to_time) - if settings.PUSH_NOTIFICATION_BOUNCER_URL: + if should_send_analytics_data(): + # Based on the specific value of the setting, the exact details to send + # will be decided. However, we proceed just based on this not being falsey. + # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so # that each server will report in at a somewhat consistent time. assert settings.ZULIP_ORG_ID diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index 00271b2f41..bc36358acf 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -8,7 +8,6 @@ import time_machine from django.apps import apps from django.db import models from django.db.models import Sum -from django.test import override_settings from django.utils.timezone import now as timezone_now from psycopg2.sql import SQL, Literal from typing_extensions import override @@ -58,6 +57,7 @@ from zerver.lib.push_notifications import ( hex_to_b64, ) from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import activate_push_notification_service from zerver.lib.timestamp import TimeZoneNotUTCError, ceiling_to_day, floor_to_day from zerver.lib.topic import DB_TOPIC_NAME from zerver.lib.user_counts import realm_user_count_by_role @@ -1373,7 +1373,7 @@ class TestLoggingCountStats(AnalyticsTestCase): self.assertTableState(UserCount, ["property", "value"], [["user test", 1]]) self.assertTableState(StreamCount, ["property", "value"], [["stream test", 1]]) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() def test_mobile_pushes_received_count(self) -> None: self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe" self.server = RemoteZulipServer.objects.create( diff --git a/corporate/tests/test_remote_billing.py b/corporate/tests/test_remote_billing.py index cb3afb8553..3a2b8e162b 100644 --- a/corporate/tests/test_remote_billing.py +++ b/corporate/tests/test_remote_billing.py @@ -5,7 +5,6 @@ from unittest import mock import responses import time_machine from django.conf import settings -from django.test import override_settings from django.utils.timezone import now as timezone_now from typing_extensions import override @@ -28,7 +27,7 @@ from zerver.lib.rate_limiter import RateLimitedIPAddr from zerver.lib.remote_server import send_server_data_to_push_bouncer from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import BouncerTestCase -from zerver.lib.test_helpers import ratelimit_rule +from zerver.lib.test_helpers import activate_push_notification_service, ratelimit_rule from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import Realm, UserProfile from zerver.models.realms import get_realm @@ -187,7 +186,7 @@ class RemoteRealmBillingTestCase(BouncerTestCase): return result -@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") +@activate_push_notification_service() class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase): @responses.activate def test_self_hosted_billing_endpoints(self) -> None: @@ -209,7 +208,7 @@ class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase): self_hosted_billing_url = "/self-hosted-billing/" self_hosted_billing_json_url = "/json/self-hosted-billing" - with self.settings(PUSH_NOTIFICATION_BOUNCER_URL=None): + with self.settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False): with self.settings(CORPORATE_ENABLED=True): result = self.client_get(self_hosted_billing_url) self.assertEqual(result.status_code, 404) @@ -278,13 +277,13 @@ class SelfHostedBillingEndpointBasicTest(RemoteRealmBillingTestCase): ) -@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") +@activate_push_notification_service() class RemoteBillingAuthenticationTest(RemoteRealmBillingTestCase): def test_self_hosted_config_error_page(self) -> None: self.login("desdemona") with ( - self.settings(CORPORATE_ENABLED=False, PUSH_NOTIFICATION_BOUNCER_URL=None), + self.settings(CORPORATE_ENABLED=False, ZULIP_SERVICE_PUSH_NOTIFICATIONS=False), self.assertLogs("django.request"), ): result = self.client_get("/self-hosted-billing/not-configured/") @@ -299,7 +298,7 @@ class RemoteBillingAuthenticationTest(RemoteRealmBillingTestCase): self.assertEqual(result.status_code, 404) # Also doesn't make sense on zulipchat.com (where CORPORATE_ENABLED is True). - with self.settings(CORPORATE_ENABLED=True, PUSH_NOTIFICATION_BOUNCER_URL=None): + with self.settings(CORPORATE_ENABLED=True, ZULIP_SERVICE_PUSH_NOTIFICATIONS=False): result = self.client_get("/self-hosted-billing/not-configured/") self.assertEqual(result.status_code, 404) diff --git a/corporate/tests/test_stripe.py b/corporate/tests/test_stripe.py index 958ecf0f7f..a76dfd07fc 100644 --- a/corporate/tests/test_stripe.py +++ b/corporate/tests/test_stripe.py @@ -21,7 +21,6 @@ import stripe import time_machine from django.conf import settings from django.core import signing -from django.test import override_settings from django.urls.resolvers import get_resolver from django.utils.crypto import get_random_string from django.utils.timezone import now as timezone_now @@ -90,6 +89,7 @@ from zerver.actions.realm_settings import do_deactivate_realm, do_reactivate_rea from zerver.actions.users import do_deactivate_user from zerver.lib.remote_server import send_server_data_to_push_bouncer from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import activate_push_notification_service from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.utils import assert_is_not_none from zerver.models import Message, Realm, RealmAuditLog, Recipient, UserProfile @@ -6579,7 +6579,7 @@ class TestRemoteBillingWriteAuditLog(StripeTestCase): ) -@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") +@activate_push_notification_service() class TestRemoteRealmBillingFlow(StripeTestCase, RemoteRealmBillingTestCase): @override def setUp(self) -> None: @@ -8308,7 +8308,7 @@ class TestRemoteRealmBillingFlow(StripeTestCase, RemoteRealmBillingTestCase): self.assertEqual(invoice_item1[key], value) -@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") +@activate_push_notification_service() class TestRemoteServerBillingFlow(StripeTestCase, RemoteServerTestCase): @override def setUp(self) -> None: diff --git a/docs/overview/changelog.md b/docs/overview/changelog.md index a105d13f04..038aba8c83 100644 --- a/docs/overview/changelog.md +++ b/docs/overview/changelog.md @@ -191,6 +191,29 @@ log][commit-log] for an up-to-date list of all changes. to give time to potentially reconfigure which channel to use. You can override the delay by running `./manage.py send_zulip_update_announcements --skip-delay` once you've done any necessary configuration updates. +- We've reworked how Zulip's mobile push notifications service is + configured to be easier to understand, more extensible, and avoid + hardcoding URLs unnecessarily. The old settings names are fully + supported with identical behavior, so no action is required before + upgrading. + + Once you've upgraded, while you're [updating your settings.py + documentation][update-settings-docs], we recommend updating + `/etc/zulip/settings.py` to use the modern settings names: Replacing + `PUSH_NOTIFICATIONS_BOUNCER_URL = "https://push.zulipchat.com"` with + `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` and renaming + `SUBMIT_USAGE_STATISTICS` to + `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS`, if you have either of those + settings enabled. It's important not to set both the old and new + settings: The modern settings will be ignored if the legacy ones are + present. + + The one minor functional change in this restructuring is that it is + now possible to configure sharing usage statistics with the Zulip + developers without attempting to send mobile push notifications via + the service, by setting `ZULIP_SERVICE_PUSH_NOTIFICATIONS = False` + and `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=True`. + - The Zulip server now contains a KaTeX server worker, designed to make bulk-rendering LaTeX efficient. It has minimal memory footprint, but can be disabled using the `katex_server` [deployment diff --git a/docs/production/mobile-push-notifications.md b/docs/production/mobile-push-notifications.md index 8dd02a79fd..902f1ac634 100644 --- a/docs/production/mobile-push-notifications.md +++ b/docs/production/mobile-push-notifications.md @@ -1,41 +1,57 @@ # Mobile push notification service -Zulip's iOS and Android [mobile apps](https://zulip.com/apps/) support receiving -push notifications from Zulip servers to let users know when new messages have -arrived. This is an important feature for having a great mobile app experience. +Zulip's iOS and Android [mobile apps](https://zulip.com/apps/) support +receiving push notifications from Zulip servers to notify users when +new messages have arrived. This is an important feature for having a +great mobile app experience. -To set up mobile push notifications, you will need to register your Zulip server -with the Zulip mobile push notification service. This service will forward push -notifications generated by your server to users' mobile apps. +The security model for mobile push notifications does not allow +self-hosted Zulip servers to directly send mobile notifications to the +Zulip mobile apps. The Zulip mobile push notification service solves +this problem by forwarding mobile push notifications generated by your +server to the Zulip mobile apps. -## How to sign up +## Signing up You can enable the mobile push notification service for your Zulip server as follows: +1. Check that your [server + version](https://zulip.com/help/view-zulip-version) is has Zulip + Server 9.0 or greater. For older versions, see the [Zulip 8.x + documentation](https://zulip.readthedocs.io/en/8.4/production/mobile-push-notifications.html). + 1. Make sure your server has outgoing HTTPS access to the public Internet. If that is restricted by a proxy, you will need to [configure Zulip to use your outgoing HTTP proxy](deployment.md#customizing-the-outgoing-http-proxy) first. +1. Set `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` in your + `/etc/zulip/settings.py` file. The [comments in + settings.py][update-settings-docs] should contain this line, + commented out with a `# `. Delete the `# ` at the start of the line + to enable the setting. + 1. Decide whether to share usage statistics with the Zulip team. By default, Zulip installations using the Mobile Push Notification Service submit additional usage statistics that help Zulip's maintainers allocate resources towards supporting self-hosted - installations ([details](#uploading-usage-statistics)). You can - disable submitting usage statistics now or at any time by setting - `SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`. + installations ([details](#uploading-usage-statistics)). + + You can disable submitting usage statistics now or at any time by + setting `ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=False` in + `/etc/zulip/settings.py` (the template contains a convenient + commented line that you can uncomment). Note that all systems using the service upload [basic metadata](#uploading-basic-metadata) about the organizations hosted by the installation. -1. Uncomment the - `PUSH_NOTIFICATION_BOUNCER_URL = 'https://push.zulipchat.com'` line - in your `/etc/zulip/settings.py` file (i.e., remove the `#` at the - start of the line), and [restart your Zulip - server](settings.md#making-changes). + [update-settings-docs]: ../production/upgrade.md#updating-settingspy-inline-documentation + +1. [Restart your Zulip server](settings.md#making-changes) so that + your configuration changes take effect. 1. Run the registration command. If you installed Zulip directly on the server (without Docker), run as root: @@ -248,7 +264,7 @@ Push Notifications Service itself. By default, Zulip installations that register for the Mobile Push Notifications Service upload the following usage statistics. You can disable these uploads any time by setting -`SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`. +`ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=False` in `/etc/zulip/settings.py`. - Totals for messages sent and read with subtotals for various combinations of clients and integrations. @@ -261,8 +277,8 @@ statistics. When enabled, usage statistics are submitted via an hourly cron job. If you'd like to access plan management immediately after -enabling `SUBMIT_USAGE_STATISTICS=True` on a pre-8.0 Zulip server, you -can run the analytics job manually via: +enabling `SUBMIT_USAGE_STATISTICS=True` (the legacy form of this setting) +on a pre-8.0 Zulip server, you can run the analytics job manually via: ``` /home/zulip/deployments/current/manage.py update_analytics_counts @@ -325,7 +341,7 @@ registration. ``` 1. Comment out the - `PUSH_NOTIFICATION_BOUNCER_URL = 'https://push.zulipchat.com'` line + `ZULIP_SERVICE_PUSH_NOTIFICATIONS = True` line in your `/etc/zulip/settings.py` file (i.e., add `# ` at the start of the line), and [restart your Zulip server](settings.md#making-changes). diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 45cd5f802c..a98198c6be 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -248,7 +248,7 @@ def send_apple_push_notification( if apns_context is None: logger.debug( "APNs: Dropping a notification because nothing configured. " - "Set PUSH_NOTIFICATION_BOUNCER_URL (or APNS_CERT_FILE)." + "Set ZULIP_SERVICES_URL (or APNS_CERT_FILE)." ) return 0 @@ -455,7 +455,7 @@ def send_android_push_notification( if not fcm_app: logger.debug( "Skipping sending a FCM push notification since " - "PUSH_NOTIFICATION_BOUNCER_URL and ANDROID_FCM_CREDENTIALS_PATH are both unset" + "ZULIP_SERVICE_PUSH_NOTIFICATIONS and ANDROID_FCM_CREDENTIALS_PATH are both unset" ) return 0 @@ -529,7 +529,7 @@ def send_android_push_notification( def uses_notification_bouncer() -> bool: - return settings.PUSH_NOTIFICATION_BOUNCER_URL is not None + return settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS is True def sends_notifications_directly() -> bool: diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index e231a171ae..95d674b735 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -27,6 +27,7 @@ from zerver.lib.exceptions import ( from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.queue import queue_event_on_commit from zerver.lib.redis_utils import get_redis_client +from zerver.lib.types import AnalyticsDataUploadLevel from zerver.models import Realm, RealmAuditLog from zerver.models.realms import OrgTypeEnum @@ -140,10 +141,10 @@ def send_to_push_bouncer( vs. client-side errors like an invalid token. """ - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None + assert settings.ZULIP_SERVICES_URL is not None assert settings.ZULIP_ORG_ID is not None assert settings.ZULIP_ORG_KEY is not None - url = urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL, "/api/v1/remotes/" + endpoint) + url = urljoin(settings.ZULIP_SERVICES_URL, "/api/v1/remotes/" + endpoint) api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID, settings.ZULIP_ORG_KEY) headers = {"User-agent": f"ZulipServer/{ZULIP_VERSION}"} @@ -286,7 +287,7 @@ def maybe_mark_pushes_disabled( if isinstance(e, JsonableError): logger.warning(e.msg) else: - logger.exception("Exception communicating with %s", settings.PUSH_NOTIFICATION_BOUNCER_URL) + logger.exception("Exception communicating with %s", settings.ZULIP_SERVICES_URL) # An exception was thrown talking to the push bouncer. There may # be certain transient failures that we could ignore here - @@ -381,6 +382,10 @@ def get_realms_info_for_push_bouncer(realm_id: int | None = None) -> list[RealmD return realm_info_list +def should_send_analytics_data() -> bool: # nocoverage + return settings.ANALYTICS_DATA_UPLOAD_LEVEL > AnalyticsDataUploadLevel.NONE + + def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) -> None: logger = logging.getLogger("zulip.analytics") # first, check what's latest @@ -396,7 +401,10 @@ def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) -> last_acked_installation_count_id = result["last_installation_count_id"] last_acked_realmauditlog_id = result["last_realmauditlog_id"] - if settings.SUBMIT_USAGE_STATISTICS and consider_usage_statistics: + if ( + settings.ANALYTICS_DATA_UPLOAD_LEVEL == AnalyticsDataUploadLevel.ALL + and consider_usage_statistics + ): # Only upload usage statistics, which is relatively expensive, # if called from the analytics cron job and the server has # uploading such statistics enabled. @@ -410,12 +418,20 @@ def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) -> installation_count_query = InstallationCount.objects.none() realm_count_query = RealmCount.objects.none() + if settings.ANALYTICS_DATA_UPLOAD_LEVEL >= AnalyticsDataUploadLevel.BILLING: + realmauditlog_query = RealmAuditLog.objects.filter( + event_type__in=RealmAuditLog.SYNCED_BILLING_EVENTS, id__gt=last_acked_realmauditlog_id + ) + else: + realmauditlog_query = RealmAuditLog.objects.none() + + # This code shouldn't be called at all if we're not configured to send any data. + assert settings.ANALYTICS_DATA_UPLOAD_LEVEL > AnalyticsDataUploadLevel.NONE + (realm_count_data, installation_count_data, realmauditlog_data) = build_analytics_data( realm_count_query=realm_count_query, installation_count_query=installation_count_query, - realmauditlog_query=RealmAuditLog.objects.filter( - event_type__in=RealmAuditLog.SYNCED_BILLING_EVENTS, id__gt=last_acked_realmauditlog_id - ), + realmauditlog_query=realmauditlog_query, ) record_count = len(realm_count_data) + len(installation_count_data) + len(realmauditlog_data) diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index ca86b1fc7d..7f6677d3b5 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -2566,8 +2566,8 @@ class BouncerTestCase(ZulipTestCase): # we can safely pick the first value. data = {k: v[0] for k, v in params.items()} assert request.url is not None # allow mypy to infer url is present. - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - local_url = request.url.replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "") + assert settings.ZULIP_SERVICES_URL is not None + local_url = request.url.replace(settings.ZULIP_SERVICES_URL, "") if request.method == "POST": result = self.uuid_post(self.server_uuid, local_url, data, subdomain="", **kwargs) elif request.method == "GET": @@ -2575,9 +2575,9 @@ class BouncerTestCase(ZulipTestCase): return (result.status_code, result.headers, result.content) def add_mock_response(self) -> None: - # Match any endpoint with the PUSH_NOTIFICATION_BOUNCER_URL. - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - COMPILED_URL = re.compile(settings.PUSH_NOTIFICATION_BOUNCER_URL + r".*") + # Match any endpoint with the ZULIP_SERVICES_URL. + assert settings.ZULIP_SERVICES_URL is not None + COMPILED_URL = re.compile(settings.ZULIP_SERVICES_URL + r".*") responses.add_callback(responses.POST, COMPILED_URL, callback=self.request_callback) responses.add_callback(responses.GET, COMPILED_URL, callback=self.request_callback) diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index f4b5080475..a62ab66cfd 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -37,6 +37,7 @@ from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.per_request_cache import flush_per_request_caches from zerver.lib.rate_limiter import RateLimitedIPAddr, rules from zerver.lib.request import RequestNotes +from zerver.lib.types import AnalyticsDataUploadLevel from zerver.lib.upload.s3 import S3UploadBackend from zerver.models import Client, Message, RealmUserDefault, Subscription, UserMessage, UserProfile from zerver.models.clients import clear_client_cache, get_client @@ -78,6 +79,53 @@ def stub_event_queue_user_events( yield +class activate_push_notification_service(override_settings): # noqa: N801 + """ + Activating the push notification service involves a few different settings + that are logically related, and ordinarily set correctly in computed_settings.py + based on the admin-configured settings. + Having tests deal with overriding all the necessary settings every time they + want to simulate using the push notification service would be too + cumbersome, so we provide a convenient helper. + Can be used as either a context manager or a decorator applied to a test method + or class, just like original override_settings. + """ + + def __init__( + self, zulip_services_url: str | None = None, submit_usage_statistics: bool = False + ) -> None: + if zulip_services_url is None: + zulip_services_url = settings.ZULIP_SERVICES_URL + assert zulip_services_url is not None + + # Ordinarily the ANALYTICS_DATA_UPLOAD_LEVEL setting is computed based on these + # ZULIP_SERVICE_* configured settings; but because these settings here won't get + # processed through computed_settings, we need to set ANALYTICS_DATA_UPLOAD_LEVEL + # manually. + # The logic here is: + # (1) If the currently active ANALYTICS_DATA_UPLOAD_LEVEL is lower than what's + # demanded to enable push notifications, then we need to override it to + # this minimum level (i.e. BILLING). + # (2) Otherwise, the test must have already somehow set up a higher level + # of data upload, so we should leave it alone. + if settings.ANALYTICS_DATA_UPLOAD_LEVEL < AnalyticsDataUploadLevel.BILLING: + analytics_data_upload_level = AnalyticsDataUploadLevel.BILLING + else: # nocoverage + analytics_data_upload_level = settings.ANALYTICS_DATA_UPLOAD_LEVEL + + # Finally, the data upload level can be elevated by the submit_usage_statistics + # argument. + if submit_usage_statistics: + analytics_data_upload_level = AnalyticsDataUploadLevel.ALL + + super().__init__( + ZULIP_SERVICES_URL=zulip_services_url, + ANALYTICS_DATA_UPLOAD_LEVEL=analytics_data_upload_level, + ZULIP_SERVICE_PUSH_NOTIFICATIONS=True, + ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS=submit_usage_statistics, + ) + + @contextmanager def cache_tries_captured() -> Iterator[list[tuple[str, str | list[str], str | None]]]: cache_queries: list[tuple[str, str | list[str], str | None]] = [] diff --git a/zerver/lib/types.py b/zerver/lib/types.py index eda4ab3c19..055c19d635 100644 --- a/zerver/lib/types.py +++ b/zerver/lib/types.py @@ -1,6 +1,7 @@ from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime +from enum import IntEnum from typing import Any, TypeAlias, TypeVar from django_stubs_ext import StrPromise @@ -326,3 +327,10 @@ class RawUserDict(TypedDict): class RemoteRealmDictValue(TypedDict): can_push: bool expected_end_timestamp: int | None + + +class AnalyticsDataUploadLevel(IntEnum): + NONE = 0 + BASIC = 1 + BILLING = 2 + ALL = 3 diff --git a/zerver/management/commands/register_server.py b/zerver/management/commands/register_server.py index bfbd00c841..6f30fc4ba3 100644 --- a/zerver/management/commands/register_server.py +++ b/zerver/management/commands/register_server.py @@ -58,16 +58,14 @@ class Command(ZulipBaseCommand): raise CommandError( "Missing zulip_org_key; run scripts/setup/generate_secrets.py to generate." ) - if settings.PUSH_NOTIFICATION_BOUNCER_URL is None: - if settings.DEVELOPMENT: - settings.PUSH_NOTIFICATION_BOUNCER_URL = ( - settings.EXTERNAL_URI_SCHEME + settings.EXTERNAL_HOST - ) - else: - raise CommandError( - "Please uncomment PUSH_NOTIFICATION_BOUNCER_URL " - "in /etc/zulip/settings.py (remove the '#')" - ) + if not settings.ZULIP_SERVICES_URL: + raise CommandError( + "ZULIP_SERVICES_URL is not set; was the default incorrectly overridden in /etc/zulip/settings.py?" + ) + if not settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS: + raise CommandError( + "Please set ZULIP_SERVICE_PUSH_NOTIFICATIONS to True in /etc/zulip/settings.py" + ) if options["deactivate"]: send_json_to_push_bouncer("POST", "server/deactivate", {}) @@ -139,15 +137,15 @@ class Command(ZulipBaseCommand): print("Mobile Push Notification Service registration successfully updated!") def _request_push_notification_bouncer_url(self, url: str, params: dict[str, Any]) -> Response: - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - registration_url = settings.PUSH_NOTIFICATION_BOUNCER_URL + url + assert settings.ZULIP_SERVICES_URL is not None + registration_url = settings.ZULIP_SERVICES_URL + url session = PushBouncerSession() try: response = session.post(registration_url, data=params) except requests.RequestException: raise CommandError( "Network error connecting to push notifications service " - f"({settings.PUSH_NOTIFICATION_BOUNCER_URL})", + f"({settings.ZULIP_SERVICES_URL})", ) try: response.raise_for_status() diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index cdf38f4f87..e993b6f329 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -23,7 +23,11 @@ from zerver.lib.home import ( ) from zerver.lib.soft_deactivation import do_soft_deactivate_users from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import get_user_messages, queries_captured +from zerver.lib.test_helpers import ( + activate_push_notification_service, + get_user_messages, + queries_captured, +) from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import DefaultStream, Draft, Realm, UserActivity, UserProfile from zerver.models.realms import get_realm @@ -973,7 +977,7 @@ class HomeTest(ZulipTestCase): self.assertEqual(page_params["narrow"], [dict(operator="stream", operand=stream_name)]) self.assertEqual(page_params["state_data"]["max_message_id"], -1) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() def test_get_billing_info(self) -> None: user = self.example_user("desdemona") user.role = UserProfile.ROLE_REALM_OWNER @@ -1129,7 +1133,7 @@ class HomeTest(ZulipTestCase): # If the server doesn't have the push bouncer configured, # remote billing should be shown anyway, as the billing endpoint # is supposed show a useful error page. - with self.settings(PUSH_NOTIFICATION_BOUNCER_URL=None, CORPORATE_ENABLED=False): + with self.settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False, CORPORATE_ENABLED=False): billing_info = get_billing_info(user) self.assertTrue(billing_info.show_remote_billing) diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 55986dd04c..27f044d6e5 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -12,7 +12,6 @@ import orjson from django.conf import settings from django.core.exceptions import ValidationError from django.db.models import Q, QuerySet -from django.test import override_settings from django.utils.timezone import now as timezone_now from typing_extensions import override @@ -47,6 +46,7 @@ from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( + activate_push_notification_service, create_s3_buckets, get_test_image_file, most_recent_message, @@ -1542,7 +1542,7 @@ class RealmImportExportTest(ExportFile): self.assertEqual(realm_user_default.default_language, "en") self.assertEqual(realm_user_default.twenty_four_hour_time, False) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() def test_import_realm_notify_bouncer(self) -> None: original_realm = Realm.objects.get(string_id="zulip") diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 76ea61f8b5..98fbf4d5b4 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -83,10 +83,12 @@ from zerver.lib.remote_server import ( from zerver.lib.response import json_response_from_error from zerver.lib.test_classes import BouncerTestCase, ZulipTestCase from zerver.lib.test_helpers import ( + activate_push_notification_service, mock_queue_publish, reset_email_visibility_to_everyone_in_zulip_realm, ) from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.types import AnalyticsDataUploadLevel from zerver.lib.user_counts import realm_user_count_by_role from zerver.models import ( Message, @@ -121,7 +123,7 @@ if settings.ZILENCER_ENABLED: class SendTestPushNotificationEndpointTest(BouncerTestCase): - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_send_test_push_notification_api_invalid_token(self) -> None: # What happens when the mobile device isn't registered with its server, @@ -167,7 +169,7 @@ class SendTestPushNotificationEndpointTest(BouncerTestCase): error_response = json_response_from_error(InvalidRemotePushDeviceTokenError()) responses.add( responses.POST, - f"{settings.PUSH_NOTIFICATION_BOUNCER_URL}/api/v1/remotes/push/test_notification", + f"{settings.ZULIP_SERVICES_URL}/api/v1/remotes/push/test_notification", body=error_response.content, status=error_response.status_code, ) @@ -293,7 +295,7 @@ class SendTestPushNotificationEndpointTest(BouncerTestCase): ) self.assert_json_success(result) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_send_test_push_notification_api_with_bouncer_config(self) -> None: """ @@ -1104,9 +1106,7 @@ class PushBouncerNotificationTest(BouncerTestCase): ) with ( - mock.patch( - "zerver.lib.push_notifications.uses_notification_bouncer", return_value=True - ), + activate_push_notification_service(), mock.patch("zerver.lib.remote_server.send_to_push_bouncer") as m, ): post_response = { @@ -1131,7 +1131,7 @@ class PushBouncerNotificationTest(BouncerTestCase): self.assertTrue(realm.push_notifications_enabled) self.assertEqual(realm.push_notifications_enabled_end_timestamp, None) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_register_token_realm_uuid_belongs_to_different_server(self) -> None: self.add_mock_response() @@ -1176,7 +1176,7 @@ class PushBouncerNotificationTest(BouncerTestCase): self.assert_length(RemotePushDeviceToken.objects.filter(token=token), 0) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_push_bouncer_api(self) -> None: """This is a variant of the below test_push_api, but using the full @@ -1221,8 +1221,8 @@ class PushBouncerNotificationTest(BouncerTestCase): result = self.client_delete(endpoint, {"token": "abcd1234"}, subdomain="zulip") self.assert_json_error(result, "Token does not exist") - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/register" + assert settings.ZULIP_SERVICES_URL is not None + URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/register" with responses.RequestsMock() as resp, self.assertLogs(level="ERROR") as error_log: resp.add(responses.POST, URL, body=ConnectionError(), status=502) with self.assertRaisesRegex( @@ -1382,11 +1382,11 @@ class AnalyticsBouncerTest(BouncerTestCase): return super().setUp() - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_analytics_failure_api(self) -> None: - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics" + assert settings.ZULIP_SERVICES_URL is not None + ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics" ANALYTICS_STATUS_URL = ANALYTICS_URL + "/status" with ( @@ -1447,7 +1447,7 @@ class AnalyticsBouncerTest(BouncerTestCase): send_server_data_to_push_bouncer() self.assertTrue( mock_warning.output[0].startswith( - f"ERROR:zulip.analytics:Exception communicating with {settings.PUSH_NOTIFICATION_BOUNCER_URL}\nTraceback", + f"ERROR:zulip.analytics:Exception communicating with {settings.ZULIP_SERVICES_URL}\nTraceback", ) ) self.assertTrue(resp.assert_call_count(ANALYTICS_STATUS_URL, 1)) @@ -1511,14 +1511,14 @@ class AnalyticsBouncerTest(BouncerTestCase): self.assertTrue(resp.assert_call_count(ANALYTICS_URL, 1)) self.assertPushNotificationsAre(False) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service(submit_usage_statistics=True) @responses.activate def test_analytics_api(self) -> None: """This is a variant of the below test_push_api, but using the full push notification bouncer flow """ - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics" + assert settings.ZULIP_SERVICES_URL is not None + ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics" ANALYTICS_STATUS_URL = ANALYTICS_URL + "/status" user = self.example_user("hamlet") end_time = self.TIME_ZERO @@ -1620,13 +1620,13 @@ class AnalyticsBouncerTest(BouncerTestCase): self.assertEqual(InstallationCount.objects.count(), 2) self.assertEqual(RealmAuditLog.objects.filter(id__gt=audit_log_max_id).count(), 2) - with self.settings(SUBMIT_USAGE_STATISTICS=False): - # With this setting off, we don't send RealmCounts and InstallationCounts. + with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.BILLING): + # With this setting, we don't send RealmCounts and InstallationCounts. send_server_data_to_push_bouncer() check_counts(2, 2, 0, 0, 1) - with self.settings(SUBMIT_USAGE_STATISTICS=True): - # With 'SUBMIT_USAGE_STATISTICS=True' but 'consider_usage_statistics=False', + with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.ALL): + # With ALL data upload enabled, but 'consider_usage_statistics=False', # we don't send RealmCount and InstallationCounts. send_server_data_to_push_bouncer(consider_usage_statistics=False) check_counts(3, 3, 0, 0, 1) @@ -1837,8 +1837,15 @@ class AnalyticsBouncerTest(BouncerTestCase): RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user.realm), }, ) + with self.settings(ANALYTICS_DATA_UPLOAD_LEVEL=AnalyticsDataUploadLevel.BASIC): + # With the BASIC level, RealmAuditLog rows are not sent. + send_server_data_to_push_bouncer() + check_counts(10, 10, 3, 2, 7) + + # Now, with ANALYTICS_DATA_UPLOAD_LEVEL back to the baseline for this test, + # the new RealmAuditLog event will be sent. send_server_data_to_push_bouncer() - check_counts(10, 10, 3, 2, 8) + check_counts(11, 11, 3, 2, 8) # Now create an InstallationCount with a property that's not supposed # to be tracked by the remote server - since the bouncer itself tracks @@ -1858,7 +1865,7 @@ class AnalyticsBouncerTest(BouncerTestCase): ) # The analytics endpoint call counts increase by 1, but the actual RemoteCounts remain unchanged, # since syncing the data failed. - check_counts(11, 11, 3, 2, 8) + check_counts(12, 12, 3, 2, 8) forbidden_installation_count.delete() (realm_count_data, installation_count_data, realmauditlog_data) = build_analytics_data( @@ -1899,7 +1906,7 @@ class AnalyticsBouncerTest(BouncerTestCase): ], ) # Only the request counts go up -- all of the other rows' duplicates are dropped - check_counts(12, 12, 3, 2, 8) + check_counts(13, 13, 3, 2, 8) # Test that only valid org_type values are accepted - integers defined in OrgTypeEnum. realms_data = get_realms_info_for_push_bouncer() @@ -1927,7 +1934,7 @@ class AnalyticsBouncerTest(BouncerTestCase): result, 'Invalid realms[0]["org_type"]: Value error, Not a valid org_type value' ) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service(submit_usage_statistics=True) @responses.activate def test_analytics_api_foreign_keys_to_remote_realm(self) -> None: self.add_mock_response() @@ -2075,7 +2082,7 @@ class AnalyticsBouncerTest(BouncerTestCase): for remote_realm_audit_log in RemoteRealmAuditLog.objects.filter(realm_id=user.realm.id): self.assertEqual(remote_realm_audit_log.remote_realm, remote_realm) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service(submit_usage_statistics=True) @responses.activate def test_analytics_api_invalid(self) -> None: """This is a variant of the below test_push_api, but using the full @@ -2098,7 +2105,7 @@ class AnalyticsBouncerTest(BouncerTestCase): self.assertEqual(m.output, ["WARNING:zulip.analytics:Invalid property invalid count stat"]) self.assertEqual(RemoteRealmCount.objects.count(), 0) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_remote_realm_duplicate_uuid(self) -> None: """ @@ -2141,7 +2148,7 @@ class AnalyticsBouncerTest(BouncerTestCase): # Servers on Zulip 2.0.6 and earlier only send realm_counts and installation_counts data, # and don't send realmauditlog_rows. Make sure that continues to work. - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_old_two_table_format(self) -> None: self.add_mock_response() @@ -2155,15 +2162,15 @@ class AnalyticsBouncerTest(BouncerTestCase): "version": '"2.0.6+git"', }, ) - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - ANALYTICS_URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/server/analytics" + assert settings.ZULIP_SERVICES_URL is not None + ANALYTICS_URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/server/analytics" self.assertTrue(responses.assert_call_count(ANALYTICS_URL, 1)) self.assertEqual(RemoteRealmCount.objects.count(), 1) self.assertEqual(RemoteInstallationCount.objects.count(), 0) self.assertEqual(RemoteRealmAuditLog.objects.count(), 0) # Make sure we aren't sending data we don't mean to, even if we don't store it. - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_only_sending_intended_realmauditlog_data(self) -> None: self.add_mock_response() @@ -2209,7 +2216,7 @@ class AnalyticsBouncerTest(BouncerTestCase): ): send_server_data_to_push_bouncer() - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_realmauditlog_data_mapping(self) -> None: self.add_mock_response() @@ -2236,7 +2243,7 @@ class AnalyticsBouncerTest(BouncerTestCase): # This verifies that the bouncer is backwards-compatible with remote servers using # TextField to store extra_data. - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_realmauditlog_string_extra_data(self) -> None: self.add_mock_response() @@ -2337,7 +2344,7 @@ class AnalyticsBouncerTest(BouncerTestCase): ) self.assertIn("Malformed audit log data", m.output[0]) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_realm_properties_after_send_analytics(self) -> None: self.add_mock_response() @@ -2647,7 +2654,7 @@ class AnalyticsBouncerTest(BouncerTestCase): ], ) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_deleted_realm(self) -> None: self.add_mock_response() @@ -2886,15 +2893,15 @@ class HandlePushNotificationTest(PushNotificationTest): @override def request_callback(self, request: PreparedRequest) -> tuple[int, ResponseHeaders, bytes]: assert request.url is not None # allow mypy to infer url is present. - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - local_url = request.url.replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "") + assert settings.ZULIP_SERVICES_URL is not None + local_url = request.url.replace(settings.ZULIP_SERVICES_URL, "") assert isinstance(request.body, bytes) result = self.uuid_post( self.server_uuid, local_url, request.body, content_type="application/json" ) return (result.status_code, result.headers, result.content) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_end_to_end(self) -> None: self.add_mock_response() @@ -2988,7 +2995,7 @@ class HandlePushNotificationTest(PushNotificationTest): ), ) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_end_to_end_failure_due_to_no_plan(self) -> None: self.add_mock_response() @@ -3062,7 +3069,7 @@ class HandlePushNotificationTest(PushNotificationTest): self.assertEqual(realm.push_notifications_enabled, True) self.assertEqual(realm.push_notifications_enabled_end_timestamp, None) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_unregistered_client(self) -> None: self.add_mock_response() @@ -3171,7 +3178,7 @@ class HandlePushNotificationTest(PushNotificationTest): # Local registrations have also been deleted: self.assertEqual(PushDeviceToken.objects.filter(kind=PushDeviceToken.APNS).count(), 0) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() @responses.activate def test_connection_error(self) -> None: self.setup_apns_tokens() @@ -3192,13 +3199,14 @@ class HandlePushNotificationTest(PushNotificationTest): "message_id": message.id, "trigger": NotificationTriggers.DIRECT_MESSAGE, } - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/notify" + assert settings.ZULIP_SERVICES_URL is not None + URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/notify" responses.add(responses.POST, URL, body=ConnectionError()) with self.assertRaises(PushNotificationBouncerRetryLaterError): handle_push_notification(self.user_profile.id, missed_message) @mock.patch("zerver.lib.push_notifications.push_notifications_configured", return_value=True) + @override_settings(ZULIP_SERVICE_PUSH_NOTIFICATIONS=False, ZULIP_SERVICES=set()) def test_read_message(self, mock_push_notifications: mock.MagicMock) -> None: user_profile = self.example_user("hamlet") message = self.get_message( @@ -3339,7 +3347,7 @@ class HandlePushNotificationTest(PushNotificationTest): "trigger": NotificationTriggers.DIRECT_MESSAGE, } with ( - self.settings(PUSH_NOTIFICATION_BOUNCER_URL=True), + activate_push_notification_service(), mock.patch( "zerver.lib.push_notifications.get_message_payload_apns", return_value={"apns": True}, @@ -3472,7 +3480,7 @@ class HandlePushNotificationTest(PushNotificationTest): ) with ( - self.settings(PUSH_NOTIFICATION_BOUNCER_URL=True), + activate_push_notification_service(), mock.patch("zerver.lib.push_notifications.send_notifications_to_bouncer") as mock_send, ): handle_remove_push_notification(user_profile.id, [message.id]) @@ -3951,7 +3959,7 @@ class TestAPNs(PushNotificationTest): notification_drop_log = ( "DEBUG:zerver.lib.push_notifications:" "APNs: Dropping a notification because nothing configured. " - "Set PUSH_NOTIFICATION_BOUNCER_URL (or APNS_CERT_FILE)." + "Set ZULIP_SERVICES_URL (or APNS_CERT_FILE)." ) from zerver.lib.push_notifications import initialize_push_notifications @@ -4733,13 +4741,13 @@ class TestSendNotificationsToBouncer(PushNotificationTest): self.assertEqual(user.realm.push_notifications_enabled, False) -@override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") +@activate_push_notification_service() class TestSendToPushBouncer(ZulipTestCase): def add_mock_response( self, body: bytes = orjson.dumps({"msg": "error"}), status: int = 200 ) -> None: - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/register" + assert settings.ZULIP_SERVICES_URL is not None + URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/register" responses.add(responses.POST, URL, body=body, status=status) @responses.activate @@ -4829,11 +4837,11 @@ class TestPushApi(BouncerTestCase): # Use push notification bouncer and try to remove non-existing tokens. with ( - self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"), + activate_push_notification_service(), responses.RequestsMock() as resp, ): - assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None - URL = settings.PUSH_NOTIFICATION_BOUNCER_URL + "/api/v1/remotes/push/unregister" + assert settings.ZULIP_SERVICES_URL is not None + URL = settings.ZULIP_SERVICES_URL + "/api/v1/remotes/push/unregister" resp.add_callback(responses.POST, URL, callback=self.request_callback) result = self.client_delete(endpoint, {"token": "abcd1234"}) self.assert_json_error(result, "Token does not exist") @@ -4867,7 +4875,7 @@ class TestPushApi(BouncerTestCase): self.assert_length(tokens, 1) self.assertEqual(tokens[0].token, token) - with self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"): + with activate_push_notification_service(): self.add_mock_response() # Enable push notification bouncer and add tokens. for endpoint, token, appid in bouncer_requests: @@ -4908,7 +4916,7 @@ class TestPushApi(BouncerTestCase): # Use push notification bouncer and test removing device tokens. # Tokens will be removed both locally and remotely. - with self.settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com"): + with activate_push_notification_service(): for endpoint, token, appid in bouncer_requests: result = self.client_delete(endpoint, {"token": token}) self.assert_json_success(result) @@ -4965,7 +4973,7 @@ class FCMSendTest(PushNotificationTest): send_android_push_notification_to_user(self.user_profile, {}, {}) self.assertEqual( "DEBUG:zerver.lib.push_notifications:" - "Skipping sending a FCM push notification since PUSH_NOTIFICATION_BOUNCER_URL " + "Skipping sending a FCM push notification since ZULIP_SERVICE_PUSH_NOTIFICATIONS " "and ANDROID_FCM_CREDENTIALS_PATH are both unset", logger.output[0], ) diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index a03f1caf48..9625edf72c 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -42,6 +42,7 @@ from zerver.lib.realm_description import get_realm_rendered_description, get_rea from zerver.lib.send_email import send_future_email from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import activate_push_notification_service from zerver.lib.upload import delete_message_attachments, upload_message_attachment from zerver.models import ( Attachment, @@ -1370,7 +1371,7 @@ class RealmTest(ZulipTestCase): ] self.assertEqual(sorted(user_group_names), sorted(expected_system_group_names)) - @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com") + @activate_push_notification_service() def test_do_create_realm_notify_bouncer(self) -> None: dummy_send_realms_only_response = { "result": "success", diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 52c07867dc..05ff4292e0 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -44,7 +44,7 @@ from zerver.lib.remote_server import get_realms_info_for_push_bouncer from zerver.lib.server_initialization import create_internal_realm, create_users from zerver.lib.storage import static_path from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS -from zerver.lib.types import ProfileFieldData +from zerver.lib.types import AnalyticsDataUploadLevel, ProfileFieldData from zerver.lib.users import add_service from zerver.lib.utils import generate_api_key from zerver.models import ( @@ -79,7 +79,10 @@ from zilencer.views import update_remote_realm_data_for_server # Disable the push notifications bouncer to avoid enqueuing updates in # maybe_enqueue_audit_log_upload during early setup. -settings.PUSH_NOTIFICATION_BOUNCER_URL = None +settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS = False +settings.ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False +settings.ZULIP_SERVICE_SECURITY_ALERTS = False +settings.ANALYTICS_DATA_UPLOAD_LEVEL = AnalyticsDataUploadLevel.NONE settings.USING_TORNADO = False # Disable using memcached caches to avoid 'unsupported pickle # protocol' errors if `populate_db` is run with a different Python diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 18c39fc6a1..be9114a40b 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -8,6 +8,7 @@ from urllib.parse import urljoin from scripts.lib.zulip_tools import get_tornado_ports from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor +from zerver.lib.types import AnalyticsDataUploadLevel from .config import ( DEPLOY_ROOT, @@ -43,6 +44,7 @@ from .configured_settings import ( LOCAL_UPLOADS_DIR, MEMCACHED_LOCATION, MEMCACHED_USERNAME, + PUSH_NOTIFICATION_BOUNCER_URL, RATE_LIMITING_RULES, REALM_HOSTS, REGISTER_LINK_DISABLED, @@ -61,9 +63,14 @@ from .configured_settings import ( SOCIAL_AUTH_SAML_SECURITY_CONFIG, SOCIAL_AUTH_SUBDOMAIN, STATIC_URL, + SUBMIT_USAGE_STATISTICS, TORNADO_PORTS, USING_PGROONGA, ZULIP_ADMINISTRATOR, + ZULIP_SERVICE_PUSH_NOTIFICATIONS, + ZULIP_SERVICE_SECURITY_ALERTS, + ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS, + ZULIP_SERVICES_URL, ) ######################################################################## @@ -90,6 +97,68 @@ SERVER_GENERATION = int(time.time()) ZULIP_ORG_KEY = get_secret("zulip_org_key") ZULIP_ORG_ID = get_secret("zulip_org_id") + +service_name_to_required_upload_level = { + "security_alerts": AnalyticsDataUploadLevel.BASIC, + "mobile_push": AnalyticsDataUploadLevel.BILLING, + "submit_usage_statistics": AnalyticsDataUploadLevel.ALL, +} + +services: list[str] | None = None + + +def services_append(service_name: str) -> None: + global services + if services is None: + services = [] + services.append(service_name) + + +if ZULIP_SERVICE_PUSH_NOTIFICATIONS: + services_append("mobile_push") + if ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS is None: + # This setting has special behavior where we want to activate + # it by default when push notifications are enabled - unless + # explicitly set otherwise in the config. + ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True + +if ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS: + services_append("submit_usage_statistics") +if ZULIP_SERVICE_SECURITY_ALERTS: + services_append("security_alerts") + +if services is None and PUSH_NOTIFICATION_BOUNCER_URL is not None: + # ZULIP_SERVICE_* are the new settings that control the services + # enabled by the server, which in turn dictate the level of data + # uploaded to ZULIP_SERVICES_URL. + # As some older servers, predating the transition to the ZULIP_SERVICE_* + # settings, may have upgraded without redoing this part of their config, + # we need this block to set this level correctly based on the + # legacy settings. + + # This is a setting that some servers from before 9.0 may have configured + # instead of the new ZULIP_SERVICE_* settings. + # Translate it to a correct configuration. + ZULIP_SERVICE_PUSH_NOTIFICATIONS = True + services_append("mobile_push") + ZULIP_SERVICES_URL = PUSH_NOTIFICATION_BOUNCER_URL + if SUBMIT_USAGE_STATISTICS: + ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True + services_append("submit_usage_statistics") + +if services is not None and set(services).intersection( + {"submit_usage_statistics", "security_alerts", "mobile_push"} +): + # None of these make sense enabled without ZULIP_SERVICES_URL. + assert ( + ZULIP_SERVICES_URL is not None + ), "ZULIP_SERVICES_URL is required when any services are enabled." + +ANALYTICS_DATA_UPLOAD_LEVEL = max( + [service_name_to_required_upload_level[service] for service in (services or [])], + default=AnalyticsDataUploadLevel.NONE, +) + if DEBUG: INTERNAL_IPS = ("127.0.0.1",) diff --git a/zproject/default_settings.py b/zproject/default_settings.py index e465952689..267bde4531 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -218,9 +218,29 @@ NAME_CHANGES_DISABLED = False AVATAR_CHANGES_DISABLED = False PASSWORD_MIN_LENGTH = 6 PASSWORD_MIN_GUESSES = 10000 -PUSH_NOTIFICATION_BOUNCER_URL: str | None = None + +ZULIP_SERVICES_URL = "https://push.zulipchat.com" +ZULIP_SERVICE_PUSH_NOTIFICATIONS = False + +# For this setting, we need to have None as the default value, so +# that we can distinguish between the case of the setting not being +# set at all and being disabled (set to False). +# That's because unless the setting is explicitly configured, we want to +# enable it in computed_settings when ZULIP_SERVICE_PUSH_NOTIFICATIONS +# is enabled. +ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS: bool | None = None +ZULIP_SERVICE_SECURITY_ALERTS = False + PUSH_NOTIFICATION_REDACT_CONTENT = False + +# Old setting kept around for backwards compatibility. Some old servers +# may have it in their settings.py. +PUSH_NOTIFICATION_BOUNCER_URL: str | None = None +# Keep this default True, so that legacy deployments that configured PUSH_NOTIFICATION_BOUNCER_URL +# without overriding SUBMIT_USAGE_STATISTICS get the original behavior. If a server configures +# the modern ZULIP_SERVICES setting, all this will be ignored. SUBMIT_USAGE_STATISTICS = True + PROMOTE_SPONSORING_ZULIP = True RATE_LIMITING = True RATE_LIMITING_AUTHENTICATE = True diff --git a/zproject/dev_settings.py b/zproject/dev_settings.py index f092300f56..d92e8773a6 100644 --- a/zproject/dev_settings.py +++ b/zproject/dev_settings.py @@ -205,7 +205,10 @@ SCIM_CONFIG: dict[str, SCIMConfigDict] = { SELF_HOSTING_MANAGEMENT_SUBDOMAIN = "selfhosting" DEVELOPMENT_DISABLE_PUSH_BOUNCER_DOMAIN_CHECK = True -PUSH_NOTIFICATION_BOUNCER_URL = f"http://push.{EXTERNAL_HOST}" +ZULIP_SERVICES_URL = f"http://push.{EXTERNAL_HOST}" + +ZULIP_SERVICE_PUSH_NOTIFICATIONS = True +ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = True # Breaks the UI if used, but enabled for development environment testing. ALLOW_GROUP_VALUED_SETTINGS = True diff --git a/zproject/prod_settings_template.py b/zproject/prod_settings_template.py index 0f2579e24f..ff08e1cf1d 100644 --- a/zproject/prod_settings_template.py +++ b/zproject/prod_settings_template.py @@ -734,12 +734,19 @@ SOCIAL_AUTH_SAML_SUPPORT_CONTACT = { ## How long outgoing webhook requests time out after # OUTGOING_WEBHOOK_TIMEOUT_SECONDS = 10 -## Support for mobile push notifications. Setting controls whether -## push notifications will be forwarded through a Zulip push -## notification bouncer server to the mobile apps. See -## https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html -## for information on how to sign up for and configure this. -# PUSH_NOTIFICATION_BOUNCER_URL = "https://push.zulipchat.com" +## Mobile push notifications require registering for the Zulip mobile +## push notification service and configuring your server to use the +## service here. For complete documentation, see: +## +## https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html +## +# ZULIP_SERVICE_PUSH_NOTIFICATIONS = True + +## By default, a Zulip server that has registered for Zulip services +## submits both basic metadata (required for billing/free plan +## eligiblity) as well as aggregate usage statistics. You can disable +## submitting usage statistics here. +# ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False ## Whether to redact the content of push notifications. This is less ## usable, but avoids sending message content over the wire. In the @@ -747,13 +754,6 @@ SOCIAL_AUTH_SAML_SUPPORT_CONTACT = { ## notification encryption feature. # PUSH_NOTIFICATION_REDACT_CONTENT = False -## Whether to submit basic usage statistics to help the Zulip core team. Details at -## -## https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html -## -## Defaults to True if and only if the Mobile Push Notifications Service is enabled. -# SUBMIT_USAGE_STATISTICS = True - ## Whether to lightly advertise sponsoring Zulip in the gear menu. # PROMOTE_SPONSORING_ZULIP = True diff --git a/zproject/test_extra_settings.py b/zproject/test_extra_settings.py index 6a83b58ee3..af455a3865 100644 --- a/zproject/test_extra_settings.py +++ b/zproject/test_extra_settings.py @@ -4,6 +4,7 @@ import ldap from django_auth_ldap.config import LDAPSearch from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor +from zerver.lib.types import AnalyticsDataUploadLevel from zproject.settings_types import OIDCIdPConfigDict, SAMLIdPConfigDict, SCIMConfigDict from .config import DEPLOY_ROOT, get_from_file_if_exists @@ -201,9 +202,23 @@ BIG_BLUE_BUTTON_URL = "https://bbb.example.com/bigbluebutton/" # By default two factor authentication is disabled in tests. # Explicitly set this to True within tests that must have this on. TWO_FACTOR_AUTHENTICATION_ENABLED = False -PUSH_NOTIFICATION_BOUNCER_URL: str | None = None DEVELOPMENT_DISABLE_PUSH_BOUNCER_DOMAIN_CHECK = False +# Disable all Zulip services by default. Tests can activate them by +# overriding settings explicitly when they want to enable something, +# often using activate_push_notification_service. +ZULIP_SERVICE_PUSH_NOTIFICATIONS = False +ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = False +ZULIP_SERVICE_SECURITY_ALERTS = False + +# Hack: This should be computed in computed_settings, but the transmission +# of test settings overrides is wonky. See test_settings for more details. +ANALYTICS_DATA_UPLOAD_LEVEL = AnalyticsDataUploadLevel.NONE + +# The most common value used by tests. Set it as the default so that it doesn't +# have to be repeated every time. +ZULIP_SERVICES_URL = "https://push.zulip.org.example.com" + # Logging the emails while running the tests adds them # to /emails page. DEVELOPMENT_LOG_EMAILS = False