zulip/zerver/views/realm_export.py

from datetime import timedelta
from typing import Annotated

from django.conf import settings
from django.db import transaction
from django.http import HttpRequest, HttpResponse
from django.utils.timezone import now as timezone_now
from django.utils.translation import gettext as _
from pydantic import Json

from analytics.models import RealmCount
from zerver.actions.realm_export import do_delete_realm_export, notify_realm_export
from zerver.decorator import require_realm_admin
from zerver.lib.exceptions import JsonableError
from zerver.lib.export import get_realm_exports_serialized
from zerver.lib.queue import queue_json_publish
from zerver.lib.response import json_success
from zerver.lib.typed_endpoint import typed_endpoint
from zerver.lib.typed_endpoint_validators import check_int_in_validator
from zerver.models import RealmExport, UserProfile


@transaction.atomic(durable=True)
@require_realm_admin
@typed_endpoint
def export_realm(
    request: HttpRequest,
    user: UserProfile,
    *,
    export_type: Json[
        Annotated[
            int,
            check_int_in_validator(
                [RealmExport.EXPORT_PUBLIC, RealmExport.EXPORT_FULL_WITH_CONSENT]
            ),
        ]
    ] = RealmExport.EXPORT_PUBLIC,
) -> HttpResponse:
    realm = user.realm
    EXPORT_LIMIT = 5

    # Exporting organizations with a huge amount of history can
    # potentially consume a lot of disk or otherwise have accidental
    # DoS risk; for that reason, we require large exports to be done
    # manually on the command line.
    #
    # It's very possible that higher limits would be completely safe.
    MAX_MESSAGE_HISTORY = 250000
    MAX_UPLOAD_QUOTA = 10 * 1024 * 1024 * 1024

    # Filter based upon the number of events that have occurred in the delta
    # If we are at the limit, the incoming request is rejected
    event_time_delta = timezone_now() - timedelta(days=7)
    limit_check = RealmExport.objects.filter(
        realm=realm, date_requested__gte=event_time_delta
    ).count()
    if limit_check >= EXPORT_LIMIT:
        raise JsonableError(_("Exceeded rate limit."))

    # The RealmCount analytics table lets us efficiently get an estimate
    # for the number of messages in an organization. It won't match the
    # actual number of messages in the export, because this measures the
    # number of messages that went to DMs / Group DMs / public or private
    # channels at the time they were sent.
    # Thus, messages that were deleted or moved between channels and
    # private messages for which the users didn't consent for export will be
    # treated differently for this check vs. in the export code.
    realm_count_query = RealmCount.objects.filter(
        realm=realm, property="messages_sent:message_type:day"
    )
    if export_type == RealmExport.EXPORT_PUBLIC:
        realm_count_query.filter(subgroup="public_stream")
    exportable_messages_estimate = sum(realm_count.value for realm_count in realm_count_query)

    if (
        exportable_messages_estimate > MAX_MESSAGE_HISTORY
        or user.realm.currently_used_upload_space_bytes() > MAX_UPLOAD_QUOTA
    ):
        raise JsonableError(
            _("Please request a manual export from {email}.").format(
                email=settings.ZULIP_ADMINISTRATOR,
            )
        )

    row = RealmExport.objects.create(
        realm=realm,
        type=export_type,
        acting_user=user,
        status=RealmExport.REQUESTED,
        date_requested=timezone_now(),
    )

    # Allow for UI updates on a pending export
    notify_realm_export(realm)

    # Using the deferred_work queue processor to avoid
    # killing the process after 60s
    event = {
        "type": "realm_export",
        "user_profile_id": user.id,
        "realm_export_id": row.id,
    }
    transaction.on_commit(lambda: queue_json_publish("deferred_work", event))
    return json_success(request, data={"id": row.id})


@require_realm_admin
def get_realm_exports(request: HttpRequest, user: UserProfile) -> HttpResponse:
    realm_exports = get_realm_exports_serialized(user.realm)
    return json_success(request, data={"exports": realm_exports})


@require_realm_admin
def delete_realm_export(request: HttpRequest, user: UserProfile, export_id: int) -> HttpResponse:
    try:
        export_row = RealmExport.objects.get(id=export_id)
    except RealmExport.DoesNotExist:
        raise JsonableError(_("Invalid data export ID"))

    if export_row.status == RealmExport.DELETED:
        raise JsonableError(_("Export already deleted"))
    if export_row.status == RealmExport.FAILED:
        raise JsonableError(_("Export failed, nothing to delete"))
    if export_row.status in [RealmExport.REQUESTED, RealmExport.STARTED]:
        raise JsonableError(_("Export still in progress"))
    do_delete_realm_export(export_row, user)
    return json_success(request)


@require_realm_admin
def get_users_export_consents(request: HttpRequest, user: UserProfile) -> HttpResponse:
    rows = UserProfile.objects.filter(realm=user.realm, is_active=True, is_bot=False).values(
        "id", "allow_private_data_export"
    )
    export_consents = [
        {"user_id": row["id"], "consented": row["allow_private_data_export"]} for row in rows
    ]
    return json_success(request, data={"export_consents": export_consents})