from typing import Any, Dict, List, Optional, Union import datetime import logging from django.core.exceptions import ValidationError from django.core.validators import validate_email, URLValidator from django.db import IntegrityError, transaction from django.http import HttpRequest, HttpResponse from django.utils import timezone from django.utils.timezone import utc as timezone_utc from django.utils.translation import ugettext as _, ugettext as err_ from django.views.decorators.csrf import csrf_exempt from analytics.lib.counts import COUNT_STATS from zerver.decorator import require_post, InvalidZulipServerKeyError from zerver.lib.exceptions import JsonableError from zerver.lib.push_notifications import send_android_push_notification, \ send_apple_push_notification from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_int, check_string, \ check_capped_string, check_string_fixed_length, check_float, check_none_or, \ check_dict_only, check_list, check_bool from zerver.models import UserProfile from zerver.views.push_notifications import validate_token from zilencer.models import RemotePushDeviceToken, RemoteZulipServer, \ RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog def validate_entity(entity: Union[UserProfile, RemoteZulipServer]) -> RemoteZulipServer: if not isinstance(entity, RemoteZulipServer): raise JsonableError(err_("Must validate with valid Zulip server API key")) return entity def validate_bouncer_token_request(entity: Union[UserProfile, RemoteZulipServer], token: bytes, kind: int) -> RemoteZulipServer: if kind not in [RemotePushDeviceToken.APNS, RemotePushDeviceToken.GCM]: raise JsonableError(err_("Invalid token type")) server = validate_entity(entity) validate_token(token, kind) return server @csrf_exempt @require_post @has_request_variables def register_remote_server( request: HttpRequest, zulip_org_id: str=REQ(str_validator=check_string_fixed_length(RemoteZulipServer.UUID_LENGTH)), zulip_org_key: str=REQ(str_validator=check_string_fixed_length(RemoteZulipServer.API_KEY_LENGTH)), hostname: str=REQ(str_validator=check_capped_string(RemoteZulipServer.HOSTNAME_MAX_LENGTH)), contact_email: str=REQ(str_validator=check_string), new_org_key: Optional[str]=REQ(str_validator=check_string_fixed_length( RemoteZulipServer.API_KEY_LENGTH), default=None), ) -> HttpResponse: # REQ validated the the field lengths, but we still need to # validate the format of these fields. try: # TODO: Ideally we'd not abuse the URL validator this way url_validator = URLValidator() url_validator('http://' + hostname) except ValidationError: raise JsonableError(_('%s is not a valid hostname') % (hostname,)) try: validate_email(contact_email) except ValidationError as e: raise JsonableError(e.message) remote_server, created = RemoteZulipServer.objects.get_or_create( uuid=zulip_org_id, defaults={'hostname': hostname, 'contact_email': contact_email, 'api_key': zulip_org_key}) if not created: if remote_server.api_key != zulip_org_key: raise InvalidZulipServerKeyError(zulip_org_id) else: remote_server.hostname = hostname remote_server.contact_email = contact_email if new_org_key is not None: remote_server.api_key = new_org_key remote_server.save() return json_success({'created': created}) @has_request_variables def register_remote_push_device(request: HttpRequest, entity: Union[UserProfile, RemoteZulipServer], user_id: int=REQ(), token: bytes=REQ(), token_kind: int=REQ(validator=check_int), ios_app_id: Optional[str]=None) -> HttpResponse: server = validate_bouncer_token_request(entity, token, token_kind) try: with transaction.atomic(): RemotePushDeviceToken.objects.create( user_id=user_id, server=server, kind=token_kind, token=token, ios_app_id=ios_app_id, # last_updated is to be renamed to date_created. last_updated=timezone.now()) except IntegrityError: pass return json_success() @has_request_variables def unregister_remote_push_device(request: HttpRequest, entity: Union[UserProfile, RemoteZulipServer], token: bytes=REQ(), token_kind: int=REQ(validator=check_int), user_id: int=REQ(), ios_app_id: Optional[str]=None) -> HttpResponse: server = validate_bouncer_token_request(entity, token, token_kind) deleted = RemotePushDeviceToken.objects.filter(token=token, kind=token_kind, user_id=user_id, server=server).delete() if deleted[0] == 0: return json_error(err_("Token does not exist")) return json_success() @has_request_variables def remote_server_notify_push(request: HttpRequest, entity: Union[UserProfile, RemoteZulipServer], payload: Dict[str, Any]=REQ(argument_type='body')) -> HttpResponse: server = validate_entity(entity) user_id = payload['user_id'] gcm_payload = payload['gcm_payload'] apns_payload = payload['apns_payload'] gcm_options = payload.get('gcm_options', {}) android_devices = list(RemotePushDeviceToken.objects.filter( user_id=user_id, kind=RemotePushDeviceToken.GCM, server=server )) apple_devices = list(RemotePushDeviceToken.objects.filter( user_id=user_id, kind=RemotePushDeviceToken.APNS, server=server )) if android_devices: send_android_push_notification(android_devices, gcm_payload, gcm_options, remote=True) if apple_devices: send_apple_push_notification(user_id, apple_devices, apns_payload, remote=True) return json_success() def validate_incoming_table_data(server: RemoteZulipServer, model: Any, rows: List[Dict[str, Any]], is_count_stat: bool=False) -> None: last_id = get_last_id_from_server(server, model) for row in rows: if is_count_stat and row['property'] not in COUNT_STATS: raise JsonableError(_("Invalid property %s") % (row['property'],)) if row['id'] <= last_id: raise JsonableError(_("Data is out of order.")) last_id = row['id'] def batch_create_table_data(server: RemoteZulipServer, model: Any, row_objects: Union[List[RemoteRealmCount], List[RemoteInstallationCount]]) -> None: BATCH_SIZE = 1000 while len(row_objects) > 0: try: model.objects.bulk_create(row_objects[:BATCH_SIZE]) except IntegrityError: logging.warning("Invalid data saving %s for server %s/%s" % ( model._meta.db_table, server.hostname, server.uuid)) raise JsonableError(_("Invalid data.")) row_objects = row_objects[BATCH_SIZE:] @has_request_variables def remote_server_post_analytics(request: HttpRequest, entity: Union[UserProfile, RemoteZulipServer], realm_counts: List[Dict[str, Any]]=REQ( validator=check_list(check_dict_only([ ('property', check_string), ('realm', check_int), ('id', check_int), ('end_time', check_float), ('subgroup', check_none_or(check_string)), ('value', check_int), ]))), installation_counts: List[Dict[str, Any]]=REQ( validator=check_list(check_dict_only([ ('property', check_string), ('id', check_int), ('end_time', check_float), ('subgroup', check_none_or(check_string)), ('value', check_int), ]))), realmauditlog_rows: Optional[List[Dict[str, Any]]]=REQ( validator=check_list(check_dict_only([ ('id', check_int), ('realm', check_int), ('event_time', check_float), ('backfilled', check_bool), ('extra_data', check_none_or(check_string)), ('event_type', check_int), ])), default=None)) -> HttpResponse: server = validate_entity(entity) validate_incoming_table_data(server, RemoteRealmCount, realm_counts, True) validate_incoming_table_data(server, RemoteInstallationCount, installation_counts, True) if realmauditlog_rows is not None: validate_incoming_table_data(server, RemoteRealmAuditLog, realmauditlog_rows) row_objects = [RemoteRealmCount( property=row['property'], realm_id=row['realm'], remote_id=row['id'], server=server, end_time=datetime.datetime.fromtimestamp(row['end_time'], tz=timezone_utc), subgroup=row['subgroup'], value=row['value']) for row in realm_counts] batch_create_table_data(server, RemoteRealmCount, row_objects) row_objects = [RemoteInstallationCount( property=row['property'], remote_id=row['id'], server=server, end_time=datetime.datetime.fromtimestamp(row['end_time'], tz=timezone_utc), subgroup=row['subgroup'], value=row['value']) for row in installation_counts] batch_create_table_data(server, RemoteInstallationCount, row_objects) if realmauditlog_rows is not None: row_objects = [RemoteRealmAuditLog( realm_id=row['realm'], remote_id=row['id'], server=server, event_time=datetime.datetime.fromtimestamp(row['event_time'], tz=timezone_utc), backfilled=row['backfilled'], extra_data=row['extra_data'], event_type=row['event_type']) for row in realmauditlog_rows] batch_create_table_data(server, RemoteRealmAuditLog, row_objects) return json_success() def get_last_id_from_server(server: RemoteZulipServer, model: Any) -> int: last_count = model.objects.filter(server=server).order_by("remote_id").last() if last_count is not None: return last_count.remote_id return 0 @has_request_variables def remote_server_check_analytics(request: HttpRequest, entity: Union[UserProfile, RemoteZulipServer]) -> HttpResponse: server = validate_entity(entity) result = { 'last_realm_count_id': get_last_id_from_server(server, RemoteRealmCount), 'last_installation_count_id': get_last_id_from_server( server, RemoteInstallationCount), 'last_realmauditlog_id': get_last_id_from_server( server, RemoteRealmAuditLog), } return json_success(result)