zulip/zerver/lib/remote_server.py

import logging
from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
from urllib.parse import urljoin

import orjson
import requests
from django.conf import settings
from django.db.models import QuerySet
from django.utils.translation import gettext as _
from pydantic import UUID4, BaseModel, ConfigDict, Field, Json, field_validator

from analytics.models import InstallationCount, RealmCount
from version import API_FEATURE_LEVEL, ZULIP_VERSION
from zerver.actions.realm_settings import (
    do_set_push_notifications_enabled_end_timestamp,
    do_set_realm_property,
)
from zerver.lib.exceptions import (
    JsonableError,
    MissingRemoteRealmError,
    RemoteRealmServerMismatchError,
)
from zerver.lib.outgoing_http import OutgoingSession
from zerver.lib.queue import queue_event_on_commit
from zerver.models import OrgTypeEnum, Realm, RealmAuditLog


class PushBouncerSession(OutgoingSession):
    def __init__(self, timeout: int = 15) -> None:
        super().__init__(role="push_bouncer", timeout=timeout)


class PushNotificationBouncerError(Exception):
    pass


class PushNotificationBouncerRetryLaterError(JsonableError):
    http_status_code = 502


class PushNotificationBouncerServerError(PushNotificationBouncerRetryLaterError):
    http_status_code = 502


class RealmCountDataForAnalytics(BaseModel):
    property: str
    realm: int
    id: int
    end_time: float
    subgroup: Optional[str]
    value: int


class InstallationCountDataForAnalytics(BaseModel):
    property: str
    id: int
    end_time: float
    subgroup: Optional[str]
    value: int


class RealmAuditLogDataForAnalytics(BaseModel):
    id: int
    realm: int
    event_time: float
    backfilled: bool
    extra_data: Optional[Union[str, Dict[str, Any]]]
    event_type: int


class RealmDataForAnalytics(BaseModel):
    model_config = ConfigDict(extra="forbid")

    id: int
    host: str
    url: str
    name: str = ""
    org_type: int = 0
    date_created: float
    deactivated: bool
    is_system_bot_realm: bool = False

    authentication_methods: Dict[str, bool] = Field(default_factory=dict)

    uuid: UUID4
    uuid_owner_secret: str

    @field_validator("org_type")
    @classmethod
    def check_is_allowed_value(cls, value: int) -> int:
        if value not in [org_type.value for org_type in OrgTypeEnum]:
            raise ValueError("Not a valid org_type value")

        return value


class AnalyticsRequest(BaseModel):
    realm_counts: Json[List[RealmCountDataForAnalytics]]
    installation_counts: Json[List[InstallationCountDataForAnalytics]]
    realmauditlog_rows: Optional[Json[List[RealmAuditLogDataForAnalytics]]] = None
    realms: Json[List[RealmDataForAnalytics]]
    version: Optional[Json[str]]
    api_feature_level: Optional[Json[int]]


class UserDataForRemoteBilling(BaseModel):
    uuid: UUID4
    email: str
    full_name: str


def send_to_push_bouncer(
    method: str,
    endpoint: str,
    post_data: Union[bytes, Mapping[str, Union[str, int, None, bytes]]],
    extra_headers: Mapping[str, str] = {},
) -> Dict[str, object]:
    """While it does actually send the notice, this function has a lot of
    code and comments around error handling for the push notifications
    bouncer.  There are several classes of failures, each with its own
    potential solution:

    * Network errors with requests.request.  We raise an exception to signal
      it to the callers.

    * 500 errors from the push bouncer or other unexpected responses;
      we don't try to parse the response, but do make clear the cause.

    * 400 errors from the push bouncer.  Here there are 2 categories:
      Our server failed to connect to the push bouncer (should throw)
      vs. client-side errors like an invalid token.

    """
    assert settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
    assert settings.ZULIP_ORG_ID is not None
    assert settings.ZULIP_ORG_KEY is not None
    url = urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL, "/api/v1/remotes/" + endpoint)
    api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID, settings.ZULIP_ORG_KEY)

    headers = {"User-agent": f"ZulipServer/{ZULIP_VERSION}"}
    headers.update(extra_headers)

    if endpoint == "server/analytics":
        # Uploading audit log and/or analytics data can require the
        # bouncer to do a significant chunk of work in a few
        # situations; since this occurs in background jobs, set a long
        # timeout.
        session = PushBouncerSession(timeout=90)
    else:
        session = PushBouncerSession()

    try:
        res = session.request(
            method,
            url,
            data=post_data,
            auth=api_auth,
            verify=True,
            headers=headers,
        )
    except (
        requests.exceptions.Timeout,
        requests.exceptions.SSLError,
        requests.exceptions.ConnectionError,
    ) as e:
        raise PushNotificationBouncerRetryLaterError(
            f"{type(e).__name__} while trying to connect to push notification bouncer"
        )

    if res.status_code >= 500:
        # 500s should be resolved by the people who run the push
        # notification bouncer service, and they'll get an appropriate
        # error notification from the server. We raise an exception to signal
        # to the callers that the attempt failed and they can retry.
        error_msg = "Received 500 from push notification bouncer"
        logging.warning(error_msg)
        raise PushNotificationBouncerServerError(error_msg)
    elif res.status_code >= 400:
        # If JSON parsing errors, just let that exception happen
        result_dict = orjson.loads(res.content)
        msg = result_dict["msg"]
        if "code" in result_dict and result_dict["code"] == "INVALID_ZULIP_SERVER":
            # Invalid Zulip server credentials should email this server's admins
            raise PushNotificationBouncerError(
                _("Push notifications bouncer error: {error}").format(error=msg)
            )
        elif (
            endpoint == "push/test_notification"
            and "code" in result_dict
            and result_dict["code"] == "INVALID_REMOTE_PUSH_DEVICE_TOKEN"
        ):
            # This error from the notification debugging endpoint should just be directly
            # communicated to the device.
            # TODO: Extend this to use a more general mechanism when we add more such error responses.
            from zerver.lib.push_notifications import InvalidRemotePushDeviceTokenError

            raise InvalidRemotePushDeviceTokenError
        elif (
            endpoint == "server/billing"
            and "code" in result_dict
            and result_dict["code"] == "MISSING_REMOTE_REALM"
        ):  # nocoverage
            # The callers requesting this endpoint want the exception to propagate
            # so they can catch it.
            raise MissingRemoteRealmError
        elif (
            endpoint == "server/billing"
            and "code" in result_dict
            and result_dict["code"] == "REMOTE_REALM_SERVER_MISMATCH_ERROR"
        ):  # nocoverage
            # The callers requesting this endpoint want the exception to propagate
            # so they can catch it.
            raise RemoteRealmServerMismatchError
        else:
            # But most other errors coming from the push bouncer
            # server are client errors (e.g. never-registered token)
            # and should be handled as such.
            raise JsonableError(msg)
    elif res.status_code != 200:
        # Anything else is unexpected and likely suggests a bug in
        # this version of Zulip, so we throw an exception that will
        # email the server admins.
        raise PushNotificationBouncerError(
            f"Push notification bouncer returned unexpected status code {res.status_code}"
        )

    # If we don't throw an exception, it's a successful bounce!
    return orjson.loads(res.content)


def send_json_to_push_bouncer(
    method: str, endpoint: str, post_data: Mapping[str, object]
) -> Dict[str, object]:
    return send_to_push_bouncer(
        method,
        endpoint,
        orjson.dumps(post_data),
        extra_headers={"Content-type": "application/json"},
    )


def build_analytics_data(
    realm_count_query: QuerySet[RealmCount],
    installation_count_query: QuerySet[InstallationCount],
    realmauditlog_query: QuerySet[RealmAuditLog],
) -> Tuple[
    List[RealmCountDataForAnalytics],
    List[InstallationCountDataForAnalytics],
    List[RealmAuditLogDataForAnalytics],
]:
    # We limit the batch size on the client side to avoid OOM kills timeouts, etc.
    MAX_CLIENT_BATCH_SIZE = 10000
    realm_count_data = [
        RealmCountDataForAnalytics(
            property=row.property,
            realm=row.realm.id,
            id=row.id,
            end_time=row.end_time.timestamp(),
            subgroup=row.subgroup,
            value=row.value,
        )
        for row in realm_count_query.order_by("id")[0:MAX_CLIENT_BATCH_SIZE]
    ]
    installation_count_data = [
        InstallationCountDataForAnalytics(
            property=row.property,
            id=row.id,
            end_time=row.end_time.timestamp(),
            subgroup=row.subgroup,
            value=row.value,
        )
        for row in installation_count_query.order_by("id")[0:MAX_CLIENT_BATCH_SIZE]
    ]
    zerver_realmauditlog = [
        RealmAuditLogDataForAnalytics(
            id=row.id,
            realm=row.realm.id,
            event_time=row.event_time.timestamp(),
            backfilled=row.backfilled,
            # Note that we don't need to add extra_data_json here because
            # the view remote_server_post_analytics populates extra_data_json
            # from the provided extra_data.
            extra_data=row.extra_data,
            event_type=row.event_type,
        )
        for row in realmauditlog_query.order_by("id")[0:MAX_CLIENT_BATCH_SIZE]
    ]

    return realm_count_data, installation_count_data, zerver_realmauditlog


def get_realms_info_for_push_bouncer(realm_id: Optional[int] = None) -> List[RealmDataForAnalytics]:
    realms = Realm.objects.order_by("id")
    if realm_id is not None:  # nocoverage
        realms = realms.filter(id=realm_id)

    realm_info_list = [
        RealmDataForAnalytics(
            id=realm.id,
            uuid=realm.uuid,
            uuid_owner_secret=realm.uuid_owner_secret,
            host=realm.host,
            url=realm.uri,
            deactivated=realm.deactivated,
            date_created=realm.date_created.timestamp(),
            org_type=realm.org_type,
            name=realm.name,
            authentication_methods=realm.authentication_methods_dict(),
            is_system_bot_realm=realm.string_id == settings.SYSTEM_BOT_REALM,
        )
        for realm in realms
    ]

    return realm_info_list


def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) -> None:
    logger = logging.getLogger("zulip.analytics")
    # first, check what's latest
    try:
        result = send_to_push_bouncer("GET", "server/analytics/status", {})
    except PushNotificationBouncerRetryLaterError as e:
        logger.warning(e.msg, exc_info=True)
        return

    # Gather only entries with IDs greater than the last ID received by the push bouncer.
    # We don't re-send old data that's already been submitted.
    last_acked_realm_count_id = result["last_realm_count_id"]
    last_acked_installation_count_id = result["last_installation_count_id"]
    last_acked_realmauditlog_id = result["last_realmauditlog_id"]

    if settings.SUBMIT_USAGE_STATISTICS and consider_usage_statistics:
        # Only upload usage statistics, which is relatively expensive,
        # if called from the analytics cron job and the server has
        # uploading such statistics enabled.
        installation_count_query = InstallationCount.objects.filter(
            id__gt=last_acked_installation_count_id
        )
        realm_count_query = RealmCount.objects.filter(id__gt=last_acked_realm_count_id)
    else:
        installation_count_query = InstallationCount.objects.none()
        realm_count_query = RealmCount.objects.none()

    (realm_count_data, installation_count_data, realmauditlog_data) = build_analytics_data(
        realm_count_query=realm_count_query,
        installation_count_query=installation_count_query,
        realmauditlog_query=RealmAuditLog.objects.filter(
            event_type__in=RealmAuditLog.SYNCED_BILLING_EVENTS, id__gt=last_acked_realmauditlog_id
        ),
    )

    record_count = len(realm_count_data) + len(installation_count_data) + len(realmauditlog_data)
    request = AnalyticsRequest.model_construct(
        realm_counts=realm_count_data,
        installation_counts=installation_count_data,
        realmauditlog_rows=realmauditlog_data,
        realms=get_realms_info_for_push_bouncer(),
        version=ZULIP_VERSION,
        api_feature_level=API_FEATURE_LEVEL,
    )

    # Send the actual request, and process the response.
    try:
        response = send_to_push_bouncer(
            "POST", "server/analytics", request.model_dump(round_trip=True)
        )
    except PushNotificationBouncerServerError:  # nocoverage
        # 50x errors from the bouncer cannot be addressed by the
        # administrator of this server, and may be localized to
        # this endpoint; don't rashly mark push notifications as
        # disabled when they are likely working perfectly fine.
        return
    except Exception as e:
        if isinstance(e, JsonableError):
            # Log exceptions with server error messages to the analytics log.
            logger.warning(e.msg)

        # An exception was thrown trying to ask the bouncer service whether we can send
        # push notifications or not. There may be certain transient failures that we could
        # ignore here, but the default explanation is that there is something wrong either
        # with our credentials being corrupted or our ability to reach the bouncer service
        # over the network, so we immediately move to reporting push notifications as likely not working,
        # as whatever failed here is likely to also fail when trying to send a push notification.
        for realm in Realm.objects.filter(push_notifications_enabled=True):
            do_set_realm_property(realm, "push_notifications_enabled", False, acting_user=None)
            do_set_push_notifications_enabled_end_timestamp(realm, None, acting_user=None)
        if not isinstance(e, JsonableError):
            # Log this generic error only if we haven't already logged specific error above.
            logger.exception("Exception asking push bouncer if notifications will work.")
        return

    assert isinstance(response["realms"], dict)  # for mypy
    realms = response["realms"]
    for realm_uuid, data in realms.items():
        try:
            realm = Realm.objects.get(uuid=realm_uuid)
        except Realm.DoesNotExist:
            # This occurs if the installation's database was rebuilt
            # from scratch or a realm was hard-deleted from the local
            # database, after generating secrets and talking to the
            # bouncer.
            logger.warning("Received unexpected realm UUID from bouncer %s", realm_uuid)
            continue

        do_set_realm_property(
            realm, "push_notifications_enabled", data["can_push"], acting_user=None
        )
        do_set_push_notifications_enabled_end_timestamp(
            realm, data["expected_end_timestamp"], acting_user=None
        )

    logger.info("Reported %d records", record_count)


def maybe_enqueue_audit_log_upload(realm: Realm) -> None:
    # Update the push notifications service, either with the fact that
    # the realm now exists or updates to its audit log of users.
    #
    # Done via a queue worker so that networking failures cannot have
    # any impact on the success operation of the local server's
    # ability to do operations that trigger these updates.
    from zerver.lib.push_notifications import uses_notification_bouncer

    if uses_notification_bouncer():
        event = {"type": "push_bouncer_update_for_realm", "realm_id": realm.id}
        queue_event_on_commit("deferred_work", event)