From cddee49e75aca31c505ec0c2173f8d7ebdfd2d2f Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Thu, 27 Oct 2016 14:55:31 -0700 Subject: [PATCH] Add support infrastructure for push notification bouncer service. This is an incomplete cleaned-up continuation of Lisa Neigut's push notification bouncer work. It supports registration and deregistration of individual push tokens with a central push notification bouncer server. It still is missing a few things before we can complete this effort: * A registration form for server admins to configure their server for this service, with tests. * Code (and tests) for actually bouncing the notifications. --- zerver/decorator.py | 44 ++++- zerver/lib/push_notifications.py | 64 +++++++ zerver/lib/test_classes.py | 15 +- zerver/tests/test_push_notifications.py | 177 +++++++++++++++++- zerver/views/push_notifications.py | 5 +- .../migrations/0002_remote_zulip_server.py | 41 ++++ zilencer/models.py | 25 ++- zilencer/urls.py | 4 + zilencer/views.py | 56 +++++- zproject/settings.py | 1 + 10 files changed, 414 insertions(+), 18 deletions(-) create mode 100644 zilencer/migrations/0002_remote_zulip_server.py diff --git a/zerver/decorator.py b/zerver/decorator.py index b5c94f8f40..f18e151a65 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -30,9 +30,18 @@ from io import BytesIO from zerver.lib.mandrill_client import get_mandrill_client from six.moves import zip, urllib -from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text +from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text, cast from zerver.lib.str_utils import force_bytes +# This is a hack to ensure that RemoteZulipServer always exists even +# if Zilencer isn't enabled. +if settings.ZILENCER_ENABLED: + from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer +else: + from mock import Mock + get_remote_server_by_uuid = Mock() + RemoteZulipServer = Mock() # type: ignore # https://github.com/JukkaL/mypy/issues/1188 + FuncT = TypeVar('FuncT', bound=Callable[..., Any]) ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse]) @@ -155,14 +164,20 @@ def process_client(request, user_profile, is_json_view=False, client_name=None): update_user_activity(request, user_profile) def validate_api_key(request, role, api_key, is_webhook=False): - # type: (HttpRequest, Text, Text, bool) -> UserProfile + # type: (HttpRequest, Text, Text, bool) -> Union[UserProfile, RemoteZulipServer] # Remove whitespace to protect users from trivial errors. role, api_key = role.strip(), api_key.strip() - try: - profile = get_user_profile_by_email(role) - except UserProfile.DoesNotExist: - raise JsonableError(_("Invalid user: %s") % (role,)) + if "@" in role: + try: + profile = get_user_profile_by_email(role) # type: Union[UserProfile, RemoteZulipServer] + except UserProfile.DoesNotExist: + raise JsonableError(_("Invalid user: %s") % (role,)) + else: + try: + profile = get_remote_server_by_uuid(role) + except RemoteZulipServer.DoesNotExist: + raise JsonableError(_("Invalid Zulip server: %s") % (role,)) if api_key != profile.api_key: if len(api_key) != 32: @@ -171,6 +186,14 @@ def validate_api_key(request, role, api_key, is_webhook=False): else: reason = _("Invalid API key for role '%s'") raise JsonableError(reason % (role,)) + + # early exit for RemoteZulipServer instances + if settings.ZILENCER_ENABLED and isinstance(profile, RemoteZulipServer): + if not check_subdomain(get_subdomain(request), ""): + raise JsonableError(_("This API key only works on the root subdomain")) + return profile + + profile = cast(UserProfile, profile) # is UserProfile if not profile.is_active: raise JsonableError(_("Account not active")) if profile.is_incoming_webhook and not is_webhook: @@ -389,13 +412,18 @@ def authenticated_rest_api_view(is_webhook=False): # Now we try to do authentication or die try: - # role is a UserProfile + # profile is a Union[UserProfile, RemoteZulipServer] profile = validate_api_key(request, role, api_key, is_webhook) except JsonableError as e: return json_unauthorized(e.error) request.user = profile process_client(request, profile) - request._email = profile.email + if isinstance(profile, UserProfile): + request._email = profile.email + else: + assert isinstance(profile, RemoteZulipServer) # type: ignore # https://github.com/python/mypy/issues/2957 + request._email = "zulip-server:" + role + profile.rate_limits = "" # Apply rate limiting return rate_limit()(view_func)(request, profile, *args, **kwargs) return _wrapped_func_arguments diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index a19cd3ab77..ccc85b3fd1 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -1,8 +1,10 @@ from __future__ import absolute_import import random +import requests from typing import Any, Dict, List, Optional, SupportsInt, Text +from version import ZULIP_VERSION from zerver.models import PushDeviceToken, Message, Recipient, UserProfile, \ UserMessage, get_display_recipient, receives_offline_notifications, \ receives_online_notifications @@ -20,12 +22,14 @@ from gcm import GCM from django.conf import settings from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ +from six.moves import urllib import base64 import binascii import logging import os import time +import ujson from functools import partial # APNS error codes @@ -343,6 +347,22 @@ def handle_push_notification(user_profile_id, missed_message): def add_push_device_token(user_profile, token_str, kind, ios_app_id=None): # type: (UserProfile, str, int, Optional[str]) -> None + # If we're sending things to the push notification bouncer + # register this user with them here + if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None: + post_data = { + 'server_uuid': settings.ZULIP_ORG_ID, + 'user_id': user_profile.id, + 'token': token_str, + 'token_kind': kind, + } + + if kind == PushDeviceToken.APNS: + post_data['ios_app_id'] = ios_app_id + + send_to_push_bouncer('POST', 'register', post_data) + return + # If another user was previously logged in on the same device and didn't # properly log out, the token will still be registered to the wrong account PushDeviceToken.objects.filter(token=token_str).exclude(user=user_profile).delete() @@ -359,8 +379,52 @@ def add_push_device_token(user_profile, token_str, kind, ios_app_id=None): def remove_push_device_token(user_profile, token_str, kind): # type: (UserProfile, str, int) -> None + + # If we're sending things to the push notification bouncer + # register this user with them here + if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None: + # TODO: Make this a remove item + post_data = { + 'server_uuid': settings.ZULIP_ORG_ID, + 'user_id': user_profile.id, + 'token': token_str, + 'token_kind': kind, + } + send_to_push_bouncer("POST", "unregister", post_data) + return + try: token = PushDeviceToken.objects.get(token=token_str, kind=kind) token.delete() except PushDeviceToken.DoesNotExist: raise JsonableError(_("Token does not exist")) + + +def send_to_push_bouncer(method, endpoint, post_data): + # type: (str, str, Dict[str, Any]) -> None + url = urllib.parse.urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL, + '/api/v1/remotes/push/' + endpoint) + api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID, + settings.ZULIP_ORG_KEY) + + res = requests.request(method, + url, + data=ujson.dumps(post_data), + auth=api_auth, + timeout=30, + verify=True, + headers={"User-agent": "ZulipServer/%s" % (ZULIP_VERSION,)}) + + # TODO: Think more carefully about how this error hanlding should work. + if res.status_code >= 500: + raise JsonableError(_("Error received from push notification bouncer")) + elif res.status_code >= 400: + try: + msg = ujson.loads(res.content)['msg'] + except Exception: + raise JsonableError(_("Error received from push notification bouncer")) + raise JsonableError(msg) + elif res.status_code != 200: + raise JsonableError(_("Error received from push notification bouncer")) + + # If we don't throw an exception, it's a successful bounce! diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index 8e42665316..fd93473a62 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -46,6 +46,7 @@ from zerver.models import ( ) from zerver.lib.request import JsonableError +from zilencer.models import get_remote_server_by_uuid import base64 @@ -268,9 +269,21 @@ class ZulipTestCase(TestCase): API_KEYS[email] = get_user_profile_by_email(email).api_key return API_KEYS[email] + def get_server_api_key(self, server_uuid): + # type: (Text) -> Text + if server_uuid not in API_KEYS: + API_KEYS[server_uuid] = get_remote_server_by_uuid(server_uuid).api_key + + return API_KEYS[server_uuid] + def api_auth(self, email): # type: (Text) -> Dict[str, Text] - credentials = u"%s:%s" % (email, self.get_api_key(email)) + if "@" not in email: + api_key = self.get_server_api_key(email) + else: + api_key = self.get_api_key(email) + + credentials = u"%s:%s" % (email, api_key) return { 'HTTP_AUTHORIZATION': u'Basic ' + base64.b64encode(credentials.encode('utf-8')).decode('utf-8') } diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index d7390fbbc7..6ab1e7a324 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -1,21 +1,30 @@ +from __future__ import absolute_import +from __future__ import print_function + import mock from mock import call import time from typing import Any, Dict, Union, SupportsInt, Text import gcm +import ujson -from django.test import TestCase +from django.test import TestCase, override_settings from django.conf import settings +from django.http import HttpResponse from zerver.models import PushDeviceToken, UserProfile, Message from zerver.models import get_user_profile_by_email, receives_online_notifications, \ receives_offline_notifications from zerver.lib import push_notifications as apn +from zerver.lib.response import json_success from zerver.lib.test_classes import ( ZulipTestCase, ) +from zilencer.models import RemoteZulipServer, RemotePushDeviceToken +from django.utils.timezone import now + class MockRedis(object): data = {} # type: Dict[str, Any] @@ -40,6 +49,172 @@ class MockRedis(object): # type: (*Any, **Any) -> None pass +class PushBouncerNotificationTest(ZulipTestCase): + server_uuid = "1234-abcd" + + def setUp(self): + # type: () -> None + server = RemoteZulipServer(uuid=self.server_uuid, + api_key="magic_secret_api_key", + hostname="demo.example.com", + last_updated=now()) + server.save() + + def tearDown(self): + # type: () -> None + RemoteZulipServer.objects.filter(uuid=self.server_uuid).delete() + + def bounce_request(self, *args, **kwargs): + # type: (*Any, **Any) -> HttpResponse + """This method is used to carry out the push notification bouncer + requests using the Django test browser, rather than python-requests. + """ + # args[0] is method, args[1] is URL. + local_url = args[1].replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "") + if args[0] == "POST": + result = self.client_post(local_url, + ujson.loads(kwargs['data']), + **self.get_auth()) + else: + raise AssertionError("Unsupported method for bounce_request") + return result + + def test_unregister_remote_push_user_params(self): + # type: () -> None + token = "111222" + token_kind = PushDeviceToken.GCM + + endpoint = '/api/v1/remotes/push/unregister' + result = self.client_post(endpoint, {'token_kind': token_kind}, + **self.get_auth()) + self.assert_json_error(result, "Missing 'token' argument") + result = self.client_post(endpoint, {'token': token}, + **self.get_auth()) + self.assert_json_error(result, "Missing 'token_kind' argument") + result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind}, + **self.api_auth("hamlet@zulip.com")) + self.assert_json_error(result, "Must validate with valid Zulip server API key") + + def test_register_remote_push_user_paramas(self): + # type: () -> None + token = "111222" + user_id = 11 + token_kind = PushDeviceToken.GCM + + endpoint = '/api/v1/remotes/push/register' + + result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind}, + **self.get_auth()) + self.assert_json_error(result, "Missing 'token' argument") + result = self.client_post(endpoint, {'user_id': user_id, 'token': token}, + **self.get_auth()) + self.assert_json_error(result, "Missing 'token_kind' argument") + result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind}, + **self.get_auth()) + self.assert_json_error(result, "Missing 'user_id' argument") + result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind, + 'token': token}, + **self.api_auth("hamlet@zulip.com")) + self.assert_json_error(result, "Must validate with valid Zulip server API key") + + def test_remote_push_user_endpoints(self): + # type: () -> None + endpoints = [ + ('/api/v1/remotes/push/register', 'register'), + ('/api/v1/remotes/push/unregister', 'unregister'), + ] + + for endpoint, method in endpoints: + payload = self.get_generic_payload(method) + + # Verify correct results are success + result = self.client_post(endpoint, payload, **self.get_auth()) + self.assert_json_success(result) + + remote_tokens = RemotePushDeviceToken.objects.filter(token=payload['token']) + token_count = 1 if method == 'register' else 0 + self.assertEqual(len(remote_tokens), token_count) + + # Try adding/removing tokens that are too big... + broken_token = "x" * 5000 # too big + payload['token'] = broken_token + result = self.client_post(endpoint, payload, **self.get_auth()) + self.assert_json_error(result, 'Empty or invalid length token') + + @override_settings(PUSH_NOTIFICATION_BOUNCER_URL='https://push.zulip.org.example.com') + @mock.patch('zerver.lib.push_notifications.requests.request') + def test_push_bouncer_api(self, mock): + # type: (Any) -> None + """This is a variant of the below test_push_api, but using the full + push notification bouncer flow + """ + mock.side_effect = self.bounce_request + email = "cordelia@zulip.com" + user = get_user_profile_by_email(email) + self.login(email) + server = RemoteZulipServer.objects.get(uuid=self.server_uuid) + + endpoints = [ + ('/json/users/me/apns_device_token', 'apple-token'), + ('/json/users/me/android_gcm_reg_id', 'android-token'), + ] + + # Test error handling + for endpoint, _ in endpoints: + # Try adding/removing tokens that are too big... + broken_token = "x" * 5000 # too big + result = self.client_post(endpoint, {'token': broken_token}) + self.assert_json_error(result, 'Empty or invalid length token') + + result = self.client_delete(endpoint, {'token': broken_token}) + self.assert_json_error(result, 'Empty or invalid length token') + + # Try to remove a non-existent token... + result = self.client_delete(endpoint, {'token': 'non-existent token'}) + self.assert_json_error(result, 'Token does not exist') + + # Add tokens + for endpoint, token in endpoints: + # Test that we can push twice + result = self.client_post(endpoint, {'token': token}) + self.assert_json_success(result) + + result = self.client_post(endpoint, {'token': token}) + self.assert_json_success(result) + + tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token, + server=server)) + self.assertEqual(len(tokens), 1) + self.assertEqual(tokens[0].token, token) + + # User should have tokens for both devices now. + tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, + server=server)) + self.assertEqual(len(tokens), 2) + + # Remove tokens + for endpoint, token in endpoints: + result = self.client_delete(endpoint, {'token': token}) + self.assert_json_success(result) + tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token, + server=server)) + self.assertEqual(len(tokens), 0) + + def get_generic_payload(self, method='register'): + # type: (Text) -> Dict[str, Any] + user_id = 10 + token = "111222" + token_kind = PushDeviceToken.GCM + + return {'user_id': user_id, + 'token': token, + 'token_kind': token_kind} + + def get_auth(self): + # type: () -> Dict[str, Text] + # Auth on this user + return self.api_auth(self.server_uuid) + class PushNotificationTest(TestCase): def setUp(self): # type: () -> None diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index 14a4414549..441695d750 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -1,6 +1,9 @@ from __future__ import absolute_import -from typing import Optional +import requests +import json + +from typing import Optional, Text from django.conf import settings from django.http import HttpRequest, HttpResponse diff --git a/zilencer/migrations/0002_remote_zulip_server.py b/zilencer/migrations/0002_remote_zulip_server.py new file mode 100644 index 0000000000..d1f9a96637 --- /dev/null +++ b/zilencer/migrations/0002_remote_zulip_server.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('zilencer', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='RemotePushDeviceToken', + fields=[ + ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)), + ('user_id', models.BigIntegerField()), + ('kind', models.PositiveSmallIntegerField(choices=[(1, 'apns'), (2, 'gcm')])), + ('token', models.CharField(unique=True, max_length=4096)), + ('last_updated', models.DateTimeField(auto_now=True)), + ('ios_app_id', models.TextField(null=True)), + ], + ), + migrations.CreateModel( + name='RemoteZulipServer', + fields=[ + ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)), + ('uuid', models.CharField(unique=True, max_length=36)), + ('api_key', models.CharField(max_length=64)), + ('hostname', models.CharField(unique=True, max_length=128)), + ('contact_email', models.EmailField(max_length=254, blank=True)), + ('last_updated', models.DateTimeField(verbose_name='last updated')), + ], + ), + migrations.AddField( + model_name='remotepushdevicetoken', + name='server', + field=models.ForeignKey(to='zilencer.RemoteZulipServer'), + ), + ] diff --git a/zilencer/models.py b/zilencer/models.py index e8bea24f20..8749e14919 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -1,12 +1,27 @@ from django.db import models from django.db.models import Manager -from typing import Dict, Text +from typing import Dict, Optional, Text import zerver.models +import datetime -def get_deployment_by_domain(realm_str): - # type: (Text) -> Deployment - return Deployment.objects.get(realms__string_id=realm_str) +def get_remote_server_by_uuid(uuid): + # type: (Text) -> RemoteZulipServer + return RemoteZulipServer.objects.get(uuid=uuid) + +class RemoteZulipServer(models.Model): + uuid = models.CharField(max_length=36, unique=True) # type: Text + api_key = models.CharField(max_length=64) # type: Text + + hostname = models.CharField(max_length=128, unique=True) # type: Text + contact_email = models.EmailField(blank=True, null=False) # type: Text + last_updated = models.DateTimeField('last updated') # type: datetime.datetime + +# Variant of PushDeviceToken for a remote server. +class RemotePushDeviceToken(zerver.models.AbstractPushDeviceToken): + server = models.ForeignKey(RemoteZulipServer) # type: RemoteZulipServer + # The user id on the remote server for this device device this is + user_id = models.BigIntegerField() # type: int class Deployment(models.Model): realms = models.ManyToManyField(zerver.models.Realm, @@ -32,5 +47,5 @@ class Deployment(models.Model): # TODO: This only does the right thing for prod because prod authenticates to # staging with the zulip.com deployment key, while staging is technically the # deployment for the zulip.com realm. - # This also doesn't necessarily handle other multi-realm deployments correctly. + # This also doesn't necessarily handle other multi-realm deployments correctly return self.realms.order_by('pk')[0].domain diff --git a/zilencer/urls.py b/zilencer/urls.py index 12d6131f3d..e2f847efbd 100644 --- a/zilencer/urls.py +++ b/zilencer/urls.py @@ -12,6 +12,10 @@ i18n_urlpatterns = [] # type: Any v1_api_and_json_patterns = [ url('^deployment/report_error$', rest_dispatch, {'POST': 'zerver.views.report.report_error'}), + url('^remotes/push/register$', rest_dispatch, + {'POST': 'zilencer.views.remote_server_register_push'}), + url('^remotes/push/unregister$', rest_dispatch, + {'POST': 'zilencer.views.remote_server_unregister_push'}), ] urlpatterns = [ diff --git a/zilencer/views.py b/zilencer/views.py index 959de186d9..dbc54a5843 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -1,17 +1,69 @@ from __future__ import absolute_import from django.utils.translation import ugettext as _ +from django.utils import timezone from django.http import HttpResponse, HttpRequest -from zilencer.models import Deployment +from zilencer.models import Deployment, RemotePushDeviceToken, RemoteZulipServer from zerver.decorator import has_request_variables, REQ from zerver.lib.error_notify import do_report_error +from zerver.lib.push_notifications import send_android_push_notification, \ + send_apple_push_notification +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict +from zerver.models import UserProfile, PushDeviceToken, Realm -from typing import Any, Dict, Text +from typing import Any, Dict, Optional, Union, Text, cast @has_request_variables def report_error(request, deployment, type=REQ(), report=REQ(validator=check_dict([]))): # type: (HttpRequest, Deployment, Text, Dict[str, Any]) -> HttpResponse return do_report_error(deployment.name, type, report) + +@has_request_variables +def remote_server_register_push(request, entity, user_id=REQ(), + token=REQ(), token_kind=REQ(), ios_app_id=None): + # type: (HttpRequest, Union[UserProfile, RemoteZulipServer], int, Text, int, Optional[Text]) -> HttpResponse + if not isinstance(entity, RemoteZulipServer): + return json_error(_("Must validate with valid Zulip server API key")) + if token == '' or len(token) > 4096: + return json_error(_("Empty or invalid length token")) + + server = cast(RemoteZulipServer, entity) + + # If a user logged out on a device and failed to unregister, + # we should delete any other user associations for this token + # & RemoteServer pair + RemotePushDeviceToken.objects.filter( + token=token, kind=token_kind, server=server).exclude(user_id=user_id).delete() + + # Save or update + remote_token, created = RemotePushDeviceToken.objects.update_or_create( + user_id=user_id, + server=server, + kind=token_kind, + token=token, + defaults=dict( + ios_app_id=ios_app_id, + last_updated=timezone.now())) + + return json_success() + +@has_request_variables +def remote_server_unregister_push(request, entity, token=REQ(), + token_kind=REQ(), ios_app_id=None): + # type: (HttpRequest, Union[UserProfile, RemoteZulipServer], Text, int, Optional[Text]) -> HttpResponse + if not isinstance(entity, RemoteZulipServer): + return json_error(_("Must validate with valid Zulip server API key")) + if token == '' or len(token) > 4096: + return json_error(_("Empty or invalid length token")) + + server = cast(RemoteZulipServer, entity) + deleted = RemotePushDeviceToken.objects.filter(token=token, + kind=token_kind, + server=server).delete() + if deleted[0] == 0: + return json_error(_("Token does not exist")) + + return json_success() diff --git a/zproject/settings.py b/zproject/settings.py index 8fac07fa3c..e90124bdf1 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -213,6 +213,7 @@ DEFAULT_SETTINGS = {'TWITTER_CONSUMER_KEY': '', 'PASSWORD_MIN_LENGTH': 6, 'PASSWORD_MIN_ZXCVBN_QUALITY': 0.5, 'OFFLINE_THRESHOLD_SECS': 5 * 60, + 'PUSH_NOTIFICATION_BOUNCER_URL': None, } for setting_name, setting_val in six.iteritems(DEFAULT_SETTINGS):