diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 95d674b735..060a64722b 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -26,7 +26,7 @@ from zerver.lib.exceptions import ( ) from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.queue import queue_event_on_commit -from zerver.lib.redis_utils import get_redis_client +from zerver.lib.redis_utils import REDIS_KEY_PREFIX, get_redis_client from zerver.lib.types import AnalyticsDataUploadLevel from zerver.models import Realm, RealmAuditLog from zerver.models.realms import OrgTypeEnum @@ -489,3 +489,16 @@ def maybe_enqueue_audit_log_upload(realm: Realm) -> None: if uses_notification_bouncer(): event = {"type": "push_bouncer_update_for_realm", "realm_id": realm.id} queue_event_on_commit("deferred_work", event) + + +SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY = ( + "self_hosting_domain_takeover_challenge_verify" +) + + +def prepare_for_registration_takeover_challenge(verification_secret: str) -> None: + redis_client.set( + REDIS_KEY_PREFIX + SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY, + verification_secret, + ex=60 * 5, + ) diff --git a/zerver/management/commands/register_server.py b/zerver/management/commands/register_server.py index 03adb5a616..ce93797f55 100644 --- a/zerver/management/commands/register_server.py +++ b/zerver/management/commands/register_server.py @@ -13,6 +13,7 @@ from typing_extensions import override from zerver.lib.management import ZulipBaseCommand, check_config from zerver.lib.remote_server import ( PushBouncerSession, + prepare_for_registration_takeover_challenge, send_json_to_push_bouncer, send_server_data_to_push_bouncer, ) @@ -39,6 +40,11 @@ class Command(ZulipBaseCommand): action="store_true", help="Automatically rotate your server's zulip_org_key", ) + action.add_argument( + "--registration-takeover", + action="store_true", + help="Overwrite pre-existing registration for the hostname", + ) action.add_argument( "--deactivate", action="store_true", @@ -72,7 +78,7 @@ class Command(ZulipBaseCommand): print("Mobile Push Notification Service registration successfully deactivated!") return - request = { + request: dict[str, object] = { "zulip_org_id": settings.ZULIP_ORG_ID, "zulip_org_key": settings.ZULIP_ORG_KEY, "hostname": settings.EXTERNAL_HOST, @@ -82,6 +88,8 @@ class Command(ZulipBaseCommand): if not os.access(SECRETS_FILENAME, os.W_OK): raise CommandError(f"{SECRETS_FILENAME} is not writable by the current user.") request["new_org_key"] = get_random_string(64) + if options["registration_takeover"]: + request["registration_takeover"] = True print( "This command registers your server for the Mobile Push Notifications Service.\n" @@ -107,11 +115,16 @@ class Command(ZulipBaseCommand): # enough about what happened. return - response = self._request_push_notification_bouncer_url( - "/api/v1/remotes/server/register", request - ) + if not options["registration_takeover"]: + response = self._request_push_notification_bouncer_url( + "/api/v1/remotes/server/register", request + ) + else: + self.do_registration_takeover_flow(request) send_server_data_to_push_bouncer(consider_usage_statistics=False) + if options["registration_takeover"]: + return if response.json()["created"]: print( @@ -121,6 +134,8 @@ class Command(ZulipBaseCommand): else: if options["rotate_key"]: print(f"Success! Updating {SECRETS_FILENAME} with the new key...") + new_org_key = request["new_org_key"] + assert isinstance(new_org_key, str) subprocess.check_call( [ "crudini", @@ -129,11 +144,52 @@ class Command(ZulipBaseCommand): SECRETS_FILENAME, "secrets", "zulip_org_key", - request["new_org_key"], + new_org_key, ] ) print("Mobile Push Notification Service registration successfully updated!") + def do_registration_takeover_flow(self, params: dict[str, Any]) -> None: + params["registration_takeover"] = "true" + response = self._request_push_notification_bouncer_url( + "/api/v1/remotes/server/register", params + ) + verification_secret = response.json()["verification_secret"] + + prepare_for_registration_takeover_challenge(verification_secret) + response = self._request_push_notification_bouncer_url( + "/api/v1/remotes/server/register/verify_challenge", + dict(hostname=params["hostname"], verification_secret=verification_secret), + ) + + org_id = response.json()["zulip_org_id"] + org_key = response.json()["zulip_org_key"] + # Update the secrets file. + print("Success! Updating secrets file with received credentials.") + subprocess.check_call( + [ + "crudini", + "--inplace", + "--set", + SECRETS_FILENAME, + "secrets", + "zulip_org_id", + org_id, + ] + ) + subprocess.check_call( + [ + "crudini", + "--inplace", + "--set", + SECRETS_FILENAME, + "secrets", + "zulip_org_key", + org_key, + ] + ) + print("Mobile Push Notification Service registration successfully transferred.") + def _request_push_notification_bouncer_url(self, url: str, params: dict[str, Any]) -> Response: assert settings.ZULIP_SERVICES_URL is not None registration_url = settings.ZULIP_SERVICES_URL + url diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index c344ec5dd6..c1562e2535 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -1,10 +1,11 @@ from django.conf import settings -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect +from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, HttpResponseRedirect from django.shortcuts import render from django.urls import reverse from django.utils.translation import gettext as _ from zerver.decorator import human_users_only, zulip_login_required +from zerver.lib import redis_utils from zerver.lib.exceptions import ( JsonableError, MissingRemoteRealmError, @@ -21,16 +22,19 @@ from zerver.lib.push_notifications import ( uses_notification_bouncer, ) from zerver.lib.remote_server import ( + SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY, UserDataForRemoteBilling, get_realms_info_for_push_bouncer, send_server_data_to_push_bouncer, send_to_push_bouncer, ) from zerver.lib.response import json_success -from zerver.lib.typed_endpoint import ApnsAppId, typed_endpoint +from zerver.lib.typed_endpoint import ApnsAppId, typed_endpoint, typed_endpoint_without_parameters from zerver.models import PushDeviceToken, UserProfile from zerver.views.errors import config_error +redis_client = redis_utils.get_redis_client() + def validate_token(token_str: str, kind: int) -> None: if token_str == "" or len(token_str) > 4096: @@ -231,3 +235,15 @@ def self_hosting_auth_not_configured(request: HttpRequest) -> HttpResponse: go_back_to_url="/", go_back_to_url_name="the app", ) + + +@typed_endpoint_without_parameters +def self_hosting_registration_takeover_challenge_verify(request: HttpRequest) -> HttpResponse: + verification_secret_bytes = redis_client.get( + SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY + ) + if verification_secret_bytes is None: + return HttpResponseNotFound() + verification_secret = verification_secret_bytes.decode() + + return json_success(request, data={"verification_secret": verification_secret}) diff --git a/zilencer/auth.py b/zilencer/auth.py index b11635ffd2..be9be93d64 100644 --- a/zilencer/auth.py +++ b/zilencer/auth.py @@ -1,3 +1,4 @@ +import base64 import logging from collections.abc import Callable from functools import wraps @@ -5,6 +6,7 @@ from typing import Any, Concatenate import sentry_sdk from django.conf import settings +from django.core.signing import BadSignature, TimestampSigner from django.http import HttpRequest, HttpResponse from django.urls import path from django.urls.resolvers import URLPattern @@ -37,6 +39,29 @@ logger = logging.getLogger(__name__) ParamT = ParamSpec("ParamT") +REMOTE_SERVER_TAKEOVER_TOKEN_SALT = "remote_server_takeover" +REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS = 300 + + +def generate_registration_takeover_verification_secret(hostname: str) -> str: + signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT) + secret = base64.b16encode(signer.sign(hostname).encode()).decode() + return secret + + +def verify_registration_takeover_verification_secret(secret: str, hostname: str) -> bool: + signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT) + try: + signed_data = base64.b16decode(secret).decode() + hostname_from_secret = signer.unsign( + signed_data, max_age=REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS + ) + except BadSignature: + return False + if hostname_from_secret != hostname: + return False + return True + class InvalidZulipServerError(JsonableError): code = ErrorCode.INVALID_ZULIP_SERVER diff --git a/zilencer/urls.py b/zilencer/urls.py index d2dfa72eb8..55bc1f1293 100644 --- a/zilencer/urls.py +++ b/zilencer/urls.py @@ -15,6 +15,7 @@ from zilencer.views import ( remote_server_send_test_notification, unregister_all_remote_push_devices, unregister_remote_push_device, + verify_registration_takeover_challenge_ack_endpoint, ) i18n_urlpatterns: Any = [] @@ -28,6 +29,10 @@ push_bouncer_patterns = [ remote_server_path("remotes/push/test_notification", POST=remote_server_send_test_notification), # Push signup doesn't use the REST API, since there's no auth. path("remotes/server/register", register_remote_server), + path( + "remotes/server/register/verify_challenge", + verify_registration_takeover_challenge_ack_endpoint, + ), remote_server_path("remotes/server/deactivate", POST=deactivate_remote_server), # For receiving table data used in analytics and billing remote_server_path("remotes/server/analytics", POST=remote_server_post_analytics), diff --git a/zilencer/views.py b/zilencer/views.py index 52083f1f76..b431bf33c6 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -3,6 +3,7 @@ from collections import Counter from datetime import datetime, timedelta, timezone from email.headerregistry import Address from typing import Annotated, Any, TypedDict, TypeVar +from urllib.parse import urljoin from uuid import UUID import orjson @@ -43,6 +44,7 @@ from zerver.lib.exceptions import ( RemoteRealmServerMismatchError, RemoteServerDeactivatedError, ) +from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.push_notifications import ( InvalidRemotePushDeviceTokenError, UserPushIdentityCompat, @@ -62,6 +64,7 @@ from zerver.lib.response import json_success from zerver.lib.send_email import FromAddress from zerver.lib.timestamp import timestamp_to_datetime from zerver.lib.typed_endpoint import ( + ApiParamConfig, ApnsAppId, JsonBodyPayload, RequiredStringConstraint, @@ -73,7 +76,11 @@ from zerver.lib.types import RemoteRealmDictValue from zerver.models.realm_audit_logs import AuditLogEventType from zerver.models.realms import DisposableEmailError from zerver.views.push_notifications import validate_token -from zilencer.auth import InvalidZulipServerKeyError +from zilencer.auth import ( + InvalidZulipServerKeyError, + generate_registration_takeover_verification_secret, + verify_registration_takeover_verification_secret, +) from zilencer.models import ( RemoteInstallationCount, RemotePushDeviceToken, @@ -143,7 +150,15 @@ def register_remote_server( RequiredStringConstraint, AfterValidator(lambda s: check_string_fixed_length(s, RemoteZulipServer.API_KEY_LENGTH)), ] = None, + # TODO: Verify how to do a bool here properly. + registration_takeover_str: Annotated[ + str | None, ApiParamConfig(whence="registration_takeover") + ] = None, ) -> HttpResponse: + registration_takeover = False + if registration_takeover_str == "true": + registration_takeover = True + # StringConstraints validated the field lengths, but we still need to # validate the format of these fields. try: @@ -207,8 +222,16 @@ def register_remote_server( raise RemoteServerDeactivatedError if remote_server is None and RemoteZulipServer.objects.filter(hostname=hostname).exists(): - raise JsonableError( - _("A server with hostname {hostname} already exists").format(hostname=hostname) + if not registration_takeover: + raise JsonableError( + _("A server with hostname {hostname} already exists").format(hostname=hostname) + ) + verification_secret = generate_registration_takeover_verification_secret(hostname) + return json_success( + request, + data={ + "verification_secret": verification_secret, + }, ) with transaction.atomic(durable=True): @@ -239,6 +262,49 @@ def register_remote_server( return json_success(request, data={"created": created}) +@csrf_exempt +@typed_endpoint +def verify_registration_takeover_challenge_ack_endpoint( + request: HttpRequest, + *, + hostname: str, + verification_secret: str, +) -> HttpResponse: + """ + The host should POST to this endpoint to announce it is ready to serve the received + secret at {hostname}/zulip-services/verify. + If we successfully verify the secret, we will send the registration credentials + to the host, completing the whole flow. + """ + success = verify_registration_takeover_verification_secret(verification_secret, hostname) + if not success: + raise JsonableError(_("Verification failed")) + + try: + remote_server = RemoteZulipServer.objects.get(hostname=hostname) + except RemoteZulipServer.DoesNotExist: + # This should generally not happen, aside of some race conditions, since if we made + # a verification_secret for this hostname, that means we had a registration for it. + raise JsonableError(_("Registration not found for this hostname")) + + # TODO: Decide the correct timeout/retries. + session = OutgoingSession( + role="verify_registration_takeover_challenge", timeout=5, max_retries=3 + ) + url = urljoin(f"https://{hostname}", "/zulip-services/verify") + + response = session.get(url) + data = response.json() + presented_secret = data["verification_secret"] + if presented_secret != verification_secret: + raise JsonableError(_("Verification of the secret provided at the hostname failed")) + + return json_success( + request, + data={"zulip_org_id": str(remote_server.uuid), "zulip_org_key": remote_server.api_key}, + ) + + @typed_endpoint def register_remote_push_device( request: HttpRequest, diff --git a/zproject/urls.py b/zproject/urls.py index e68e696a77..24c108775b 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -101,6 +101,7 @@ from zerver.views.push_notifications import ( self_hosting_auth_json_endpoint, self_hosting_auth_not_configured, self_hosting_auth_redirect_endpoint, + self_hosting_registration_takeover_challenge_verify, send_test_push_notification_api, ) from zerver.views.reactions import add_reaction, remove_reaction @@ -859,6 +860,13 @@ urls += [ ), ] +urls += [ + path( + "zulip-services/verify", + self_hosting_registration_takeover_challenge_verify, + ), +] + if not settings.CORPORATE_ENABLED: # nocoverage # This conditional behavior cannot be tested directly, since # urls.py is not readily reloaded in Django tests. See the block