zilencer: Add flow for a server to reclaim its push notifs registration.

If the server controls the registration's hostname, it can reclaim its
registration credentials. This is useful, because self-hosted admins
frequently lose the credentials when moving their Zulip server to a
different machine / deployment method.

The flow is the following:
1. The host sends a POST request to /register specifying
   registration_takeover=true.
2. The bouncer responds with a signed token.
3. The host prepares to serve this token at /zulip-services/verify and
   sends a POST to /remotes/server/register/verify_challenge endpoint of
   the bouncer.
4. Upon receiving the POST request, the bouncer GETS
   https://{hostname}/zulip-services/verify,
   verifies the secret and responds to the original POST with the
   registration credentials.
5. The host can now save these credentials to it zulip-secrets.conf file
   and thus regains its push notifications registration.

zerver/lib/remote_server.py

@@ -26,7 +26,7 @@ from zerver.lib.exceptions import (
 )
 from zerver.lib.outgoing_http import OutgoingSession
 from zerver.lib.queue import queue_event_on_commit
-from zerver.lib.redis_utils import get_redis_client
+from zerver.lib.redis_utils import REDIS_KEY_PREFIX, get_redis_client
 from zerver.lib.types import AnalyticsDataUploadLevel
 from zerver.models import Realm, RealmAuditLog
 from zerver.models.realms import OrgTypeEnum
@@ -489,3 +489,16 @@ def maybe_enqueue_audit_log_upload(realm: Realm) -> None:
     if uses_notification_bouncer():
         event = {"type": "push_bouncer_update_for_realm", "realm_id": realm.id}
         queue_event_on_commit("deferred_work", event)
+
+
+SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY = (
+    "self_hosting_domain_takeover_challenge_verify"
+)
+
+
+def prepare_for_registration_takeover_challenge(verification_secret: str) -> None:
+    redis_client.set(
+        REDIS_KEY_PREFIX + SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY,
+        verification_secret,
+        ex=60 * 5,
+    )

zerver/management/commands/register_server.py

@@ -13,6 +13,7 @@ from typing_extensions import override
 from zerver.lib.management import ZulipBaseCommand, check_config
 from zerver.lib.remote_server import (
     PushBouncerSession,
+    prepare_for_registration_takeover_challenge,
     send_json_to_push_bouncer,
     send_server_data_to_push_bouncer,
 )
@@ -39,6 +40,11 @@ class Command(ZulipBaseCommand):
             action="store_true",
             help="Automatically rotate your server's zulip_org_key",
         )
+        action.add_argument(
+            "--registration-takeover",
+            action="store_true",
+            help="Overwrite pre-existing registration for the hostname",
+        )
         action.add_argument(
             "--deactivate",
             action="store_true",
@@ -72,7 +78,7 @@ class Command(ZulipBaseCommand):
             print("Mobile Push Notification Service registration successfully deactivated!")
             return
 
-        request = {
+        request: dict[str, object] = {
             "zulip_org_id": settings.ZULIP_ORG_ID,
             "zulip_org_key": settings.ZULIP_ORG_KEY,
             "hostname": settings.EXTERNAL_HOST,
@@ -82,6 +88,8 @@ class Command(ZulipBaseCommand):
             if not os.access(SECRETS_FILENAME, os.W_OK):
                 raise CommandError(f"{SECRETS_FILENAME} is not writable by the current user.")
             request["new_org_key"] = get_random_string(64)
+        if options["registration_takeover"]:
+            request["registration_takeover"] = True
 
         print(
             "This command registers your server for the Mobile Push Notifications Service.\n"
@@ -107,11 +115,16 @@ class Command(ZulipBaseCommand):
                 # enough about what happened.
                 return
 
-        response = self._request_push_notification_bouncer_url(
+        if not options["registration_takeover"]:
-            "/api/v1/remotes/server/register", request
+            response = self._request_push_notification_bouncer_url(
-        )
+                "/api/v1/remotes/server/register", request
+            )
+        else:
+            self.do_registration_takeover_flow(request)
 
         send_server_data_to_push_bouncer(consider_usage_statistics=False)
+        if options["registration_takeover"]:
+            return
 
         if response.json()["created"]:
             print(
@@ -121,6 +134,8 @@ class Command(ZulipBaseCommand):
         else:
             if options["rotate_key"]:
                 print(f"Success! Updating {SECRETS_FILENAME} with the new key...")
+                new_org_key = request["new_org_key"]
+                assert isinstance(new_org_key, str)
                 subprocess.check_call(
                     [
                         "crudini",
@@ -129,11 +144,52 @@ class Command(ZulipBaseCommand):
                         SECRETS_FILENAME,
                         "secrets",
                         "zulip_org_key",
-                        request["new_org_key"],
+                        new_org_key,
                     ]
                 )
             print("Mobile Push Notification Service registration successfully updated!")
 
+    def do_registration_takeover_flow(self, params: dict[str, Any]) -> None:
+        params["registration_takeover"] = "true"
+        response = self._request_push_notification_bouncer_url(
+            "/api/v1/remotes/server/register", params
+        )
+        verification_secret = response.json()["verification_secret"]
+
+        prepare_for_registration_takeover_challenge(verification_secret)
+        response = self._request_push_notification_bouncer_url(
+            "/api/v1/remotes/server/register/verify_challenge",
+            dict(hostname=params["hostname"], verification_secret=verification_secret),
+        )
+
+        org_id = response.json()["zulip_org_id"]
+        org_key = response.json()["zulip_org_key"]
+        # Update the secrets file.
+        print("Success! Updating secrets file with received credentials.")
+        subprocess.check_call(
+            [
+                "crudini",
+                "--inplace",
+                "--set",
+                SECRETS_FILENAME,
+                "secrets",
+                "zulip_org_id",
+                org_id,
+            ]
+        )
+        subprocess.check_call(
+            [
+                "crudini",
+                "--inplace",
+                "--set",
+                SECRETS_FILENAME,
+                "secrets",
+                "zulip_org_key",
+                org_key,
+            ]
+        )
+        print("Mobile Push Notification Service registration successfully transferred.")
+
     def _request_push_notification_bouncer_url(self, url: str, params: dict[str, Any]) -> Response:
         assert settings.ZULIP_SERVICES_URL is not None
         registration_url = settings.ZULIP_SERVICES_URL + url

zerver/views/push_notifications.py

@@ -1,10 +1,11 @@
 from django.conf import settings
-from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
 from zerver.decorator import human_users_only, zulip_login_required
+from zerver.lib import redis_utils
 from zerver.lib.exceptions import (
     JsonableError,
     MissingRemoteRealmError,
@@ -21,16 +22,19 @@ from zerver.lib.push_notifications import (
     uses_notification_bouncer,
 )
 from zerver.lib.remote_server import (
+    SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY,
     UserDataForRemoteBilling,
     get_realms_info_for_push_bouncer,
     send_server_data_to_push_bouncer,
     send_to_push_bouncer,
 )
 from zerver.lib.response import json_success
-from zerver.lib.typed_endpoint import ApnsAppId, typed_endpoint
+from zerver.lib.typed_endpoint import ApnsAppId, typed_endpoint, typed_endpoint_without_parameters
 from zerver.models import PushDeviceToken, UserProfile
 from zerver.views.errors import config_error
 
+redis_client = redis_utils.get_redis_client()
+
 
 def validate_token(token_str: str, kind: int) -> None:
     if token_str == "" or len(token_str) > 4096:
@@ -231,3 +235,15 @@ def self_hosting_auth_not_configured(request: HttpRequest) -> HttpResponse:
         go_back_to_url="/",
         go_back_to_url_name="the app",
     )
+
+
+@typed_endpoint_without_parameters
+def self_hosting_registration_takeover_challenge_verify(request: HttpRequest) -> HttpResponse:
+    verification_secret_bytes = redis_client.get(
+        SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY
+    )
+    if verification_secret_bytes is None:
+        return HttpResponseNotFound()
+    verification_secret = verification_secret_bytes.decode()
+
+    return json_success(request, data={"verification_secret": verification_secret})

zilencer/auth.py

@@ -1,3 +1,4 @@
+import base64
 import logging
 from collections.abc import Callable
 from functools import wraps
@@ -5,6 +6,7 @@ from typing import Any, Concatenate
 
 import sentry_sdk
 from django.conf import settings
+from django.core.signing import BadSignature, TimestampSigner
 from django.http import HttpRequest, HttpResponse
 from django.urls import path
 from django.urls.resolvers import URLPattern
@@ -37,6 +39,29 @@ logger = logging.getLogger(__name__)
 
 ParamT = ParamSpec("ParamT")
 
+REMOTE_SERVER_TAKEOVER_TOKEN_SALT = "remote_server_takeover"
+REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS = 300
+
+
+def generate_registration_takeover_verification_secret(hostname: str) -> str:
+    signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT)
+    secret = base64.b16encode(signer.sign(hostname).encode()).decode()
+    return secret
+
+
+def verify_registration_takeover_verification_secret(secret: str, hostname: str) -> bool:
+    signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT)
+    try:
+        signed_data = base64.b16decode(secret).decode()
+        hostname_from_secret = signer.unsign(
+            signed_data, max_age=REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS
+        )
+    except BadSignature:
+        return False
+    if hostname_from_secret != hostname:
+        return False
+    return True
+
 
 class InvalidZulipServerError(JsonableError):
     code = ErrorCode.INVALID_ZULIP_SERVER

zilencer/urls.py

@@ -15,6 +15,7 @@ from zilencer.views import (
     remote_server_send_test_notification,
     unregister_all_remote_push_devices,
     unregister_remote_push_device,
+    verify_registration_takeover_challenge_ack_endpoint,
 )
 
 i18n_urlpatterns: Any = []
@@ -28,6 +29,10 @@ push_bouncer_patterns = [
     remote_server_path("remotes/push/test_notification", POST=remote_server_send_test_notification),
     # Push signup doesn't use the REST API, since there's no auth.
     path("remotes/server/register", register_remote_server),
+    path(
+        "remotes/server/register/verify_challenge",
+        verify_registration_takeover_challenge_ack_endpoint,
+    ),
     remote_server_path("remotes/server/deactivate", POST=deactivate_remote_server),
     # For receiving table data used in analytics and billing
     remote_server_path("remotes/server/analytics", POST=remote_server_post_analytics),

zilencer/views.py

@@ -3,6 +3,7 @@ from collections import Counter
 from datetime import datetime, timedelta, timezone
 from email.headerregistry import Address
 from typing import Annotated, Any, TypedDict, TypeVar
+from urllib.parse import urljoin
 from uuid import UUID
 
 import orjson
@@ -43,6 +44,7 @@ from zerver.lib.exceptions import (
     RemoteRealmServerMismatchError,
     RemoteServerDeactivatedError,
 )
+from zerver.lib.outgoing_http import OutgoingSession
 from zerver.lib.push_notifications import (
     InvalidRemotePushDeviceTokenError,
     UserPushIdentityCompat,
@@ -62,6 +64,7 @@ from zerver.lib.response import json_success
 from zerver.lib.send_email import FromAddress
 from zerver.lib.timestamp import timestamp_to_datetime
 from zerver.lib.typed_endpoint import (
+    ApiParamConfig,
     ApnsAppId,
     JsonBodyPayload,
     RequiredStringConstraint,
@@ -73,7 +76,11 @@ from zerver.lib.types import RemoteRealmDictValue
 from zerver.models.realm_audit_logs import AuditLogEventType
 from zerver.models.realms import DisposableEmailError
 from zerver.views.push_notifications import validate_token
-from zilencer.auth import InvalidZulipServerKeyError
+from zilencer.auth import (
+    InvalidZulipServerKeyError,
+    generate_registration_takeover_verification_secret,
+    verify_registration_takeover_verification_secret,
+)
 from zilencer.models import (
     RemoteInstallationCount,
     RemotePushDeviceToken,
@@ -143,7 +150,15 @@ def register_remote_server(
         RequiredStringConstraint,
         AfterValidator(lambda s: check_string_fixed_length(s, RemoteZulipServer.API_KEY_LENGTH)),
     ] = None,
+    # TODO: Verify how to do a bool here properly.
+    registration_takeover_str: Annotated[
+        str | None, ApiParamConfig(whence="registration_takeover")
+    ] = None,
 ) -> HttpResponse:
+    registration_takeover = False
+    if registration_takeover_str == "true":
+        registration_takeover = True
+
     # StringConstraints validated the field lengths, but we still need to
     # validate the format of these fields.
     try:
@@ -207,8 +222,16 @@ def register_remote_server(
             raise RemoteServerDeactivatedError
 
     if remote_server is None and RemoteZulipServer.objects.filter(hostname=hostname).exists():
-        raise JsonableError(
+        if not registration_takeover:
-            _("A server with hostname {hostname} already exists").format(hostname=hostname)
+            raise JsonableError(
+                _("A server with hostname {hostname} already exists").format(hostname=hostname)
+            )
+        verification_secret = generate_registration_takeover_verification_secret(hostname)
+        return json_success(
+            request,
+            data={
+                "verification_secret": verification_secret,
+            },
         )
 
     with transaction.atomic(durable=True):
@@ -239,6 +262,49 @@ def register_remote_server(
     return json_success(request, data={"created": created})
 
 
+@csrf_exempt
+@typed_endpoint
+def verify_registration_takeover_challenge_ack_endpoint(
+    request: HttpRequest,
+    *,
+    hostname: str,
+    verification_secret: str,
+) -> HttpResponse:
+    """
+    The host should POST to this endpoint to announce it is ready to serve the received
+    secret at {hostname}/zulip-services/verify.
+    If we successfully verify the secret, we will send the registration credentials
+    to the host, completing the whole flow.
+    """
+    success = verify_registration_takeover_verification_secret(verification_secret, hostname)
+    if not success:
+        raise JsonableError(_("Verification failed"))
+
+    try:
+        remote_server = RemoteZulipServer.objects.get(hostname=hostname)
+    except RemoteZulipServer.DoesNotExist:
+        # This should generally not happen, aside of some race conditions, since if we made
+        # a verification_secret for this hostname, that means we had a registration for it.
+        raise JsonableError(_("Registration not found for this hostname"))
+
+    # TODO: Decide the correct timeout/retries.
+    session = OutgoingSession(
+        role="verify_registration_takeover_challenge", timeout=5, max_retries=3
+    )
+    url = urljoin(f"https://{hostname}", "/zulip-services/verify")
+
+    response = session.get(url)
+    data = response.json()
+    presented_secret = data["verification_secret"]
+    if presented_secret != verification_secret:
+        raise JsonableError(_("Verification of the secret provided at the hostname failed"))
+
+    return json_success(
+        request,
+        data={"zulip_org_id": str(remote_server.uuid), "zulip_org_key": remote_server.api_key},
+    )
+
+
 @typed_endpoint
 def register_remote_push_device(
     request: HttpRequest,

zproject/urls.py

@@ -101,6 +101,7 @@ from zerver.views.push_notifications import (
     self_hosting_auth_json_endpoint,
     self_hosting_auth_not_configured,
     self_hosting_auth_redirect_endpoint,
+    self_hosting_registration_takeover_challenge_verify,
     send_test_push_notification_api,
 )
 from zerver.views.reactions import add_reaction, remove_reaction
@@ -859,6 +860,13 @@ urls += [
     ),
 ]
 
+urls += [
+    path(
+        "zulip-services/verify",
+        self_hosting_registration_takeover_challenge_verify,
+    ),
+]
+
 if not settings.CORPORATE_ENABLED:  # nocoverage
     # This conditional behavior cannot be tested directly, since
     # urls.py is not readily reloaded in Django tests. See the block