Add support infrastructure for push notification bouncer service.

This is an incomplete cleaned-up continuation of Lisa Neigut's push
notification bouncer work.  It supports registration and
deregistration of individual push tokens with a central push
notification bouncer server.

It still is missing a few things before we can complete this effort:
* A registration form for server admins to configure their server for
  this service, with tests.
* Code (and tests) for actually bouncing the notifications.
This commit is contained in:
Tim Abbott 2016-10-27 14:55:31 -07:00
parent ae788b2e98
commit cddee49e75
10 changed files with 414 additions and 18 deletions

View File

@ -30,9 +30,18 @@ from io import BytesIO
from zerver.lib.mandrill_client import get_mandrill_client
from six.moves import zip, urllib
from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text
from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text, cast
from zerver.lib.str_utils import force_bytes
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else:
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock() # type: ignore # https://github.com/JukkaL/mypy/issues/1188
FuncT = TypeVar('FuncT', bound=Callable[..., Any])
ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse])
@ -155,14 +164,20 @@ def process_client(request, user_profile, is_json_view=False, client_name=None):
update_user_activity(request, user_profile)
def validate_api_key(request, role, api_key, is_webhook=False):
# type: (HttpRequest, Text, Text, bool) -> UserProfile
# type: (HttpRequest, Text, Text, bool) -> Union[UserProfile, RemoteZulipServer]
# Remove whitespace to protect users from trivial errors.
role, api_key = role.strip(), api_key.strip()
try:
profile = get_user_profile_by_email(role)
except UserProfile.DoesNotExist:
raise JsonableError(_("Invalid user: %s") % (role,))
if "@" in role:
try:
profile = get_user_profile_by_email(role) # type: Union[UserProfile, RemoteZulipServer]
except UserProfile.DoesNotExist:
raise JsonableError(_("Invalid user: %s") % (role,))
else:
try:
profile = get_remote_server_by_uuid(role)
except RemoteZulipServer.DoesNotExist:
raise JsonableError(_("Invalid Zulip server: %s") % (role,))
if api_key != profile.api_key:
if len(api_key) != 32:
@ -171,6 +186,14 @@ def validate_api_key(request, role, api_key, is_webhook=False):
else:
reason = _("Invalid API key for role '%s'")
raise JsonableError(reason % (role,))
# early exit for RemoteZulipServer instances
if settings.ZILENCER_ENABLED and isinstance(profile, RemoteZulipServer):
if not check_subdomain(get_subdomain(request), ""):
raise JsonableError(_("This API key only works on the root subdomain"))
return profile
profile = cast(UserProfile, profile) # is UserProfile
if not profile.is_active:
raise JsonableError(_("Account not active"))
if profile.is_incoming_webhook and not is_webhook:
@ -389,13 +412,18 @@ def authenticated_rest_api_view(is_webhook=False):
# Now we try to do authentication or die
try:
# role is a UserProfile
# profile is a Union[UserProfile, RemoteZulipServer]
profile = validate_api_key(request, role, api_key, is_webhook)
except JsonableError as e:
return json_unauthorized(e.error)
request.user = profile
process_client(request, profile)
request._email = profile.email
if isinstance(profile, UserProfile):
request._email = profile.email
else:
assert isinstance(profile, RemoteZulipServer) # type: ignore # https://github.com/python/mypy/issues/2957
request._email = "zulip-server:" + role
profile.rate_limits = ""
# Apply rate limiting
return rate_limit()(view_func)(request, profile, *args, **kwargs)
return _wrapped_func_arguments

View File

@ -1,8 +1,10 @@
from __future__ import absolute_import
import random
import requests
from typing import Any, Dict, List, Optional, SupportsInt, Text
from version import ZULIP_VERSION
from zerver.models import PushDeviceToken, Message, Recipient, UserProfile, \
UserMessage, get_display_recipient, receives_offline_notifications, \
receives_online_notifications
@ -20,12 +22,14 @@ from gcm import GCM
from django.conf import settings
from django.utils.timezone import now as timezone_now
from django.utils.translation import ugettext as _
from six.moves import urllib
import base64
import binascii
import logging
import os
import time
import ujson
from functools import partial
# APNS error codes
@ -343,6 +347,22 @@ def handle_push_notification(user_profile_id, missed_message):
def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
# type: (UserProfile, str, int, Optional[str]) -> None
# If we're sending things to the push notification bouncer
# register this user with them here
if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None:
post_data = {
'server_uuid': settings.ZULIP_ORG_ID,
'user_id': user_profile.id,
'token': token_str,
'token_kind': kind,
}
if kind == PushDeviceToken.APNS:
post_data['ios_app_id'] = ios_app_id
send_to_push_bouncer('POST', 'register', post_data)
return
# If another user was previously logged in on the same device and didn't
# properly log out, the token will still be registered to the wrong account
PushDeviceToken.objects.filter(token=token_str).exclude(user=user_profile).delete()
@ -359,8 +379,52 @@ def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
def remove_push_device_token(user_profile, token_str, kind):
# type: (UserProfile, str, int) -> None
# If we're sending things to the push notification bouncer
# register this user with them here
if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None:
# TODO: Make this a remove item
post_data = {
'server_uuid': settings.ZULIP_ORG_ID,
'user_id': user_profile.id,
'token': token_str,
'token_kind': kind,
}
send_to_push_bouncer("POST", "unregister", post_data)
return
try:
token = PushDeviceToken.objects.get(token=token_str, kind=kind)
token.delete()
except PushDeviceToken.DoesNotExist:
raise JsonableError(_("Token does not exist"))
def send_to_push_bouncer(method, endpoint, post_data):
# type: (str, str, Dict[str, Any]) -> None
url = urllib.parse.urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL,
'/api/v1/remotes/push/' + endpoint)
api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID,
settings.ZULIP_ORG_KEY)
res = requests.request(method,
url,
data=ujson.dumps(post_data),
auth=api_auth,
timeout=30,
verify=True,
headers={"User-agent": "ZulipServer/%s" % (ZULIP_VERSION,)})
# TODO: Think more carefully about how this error hanlding should work.
if res.status_code >= 500:
raise JsonableError(_("Error received from push notification bouncer"))
elif res.status_code >= 400:
try:
msg = ujson.loads(res.content)['msg']
except Exception:
raise JsonableError(_("Error received from push notification bouncer"))
raise JsonableError(msg)
elif res.status_code != 200:
raise JsonableError(_("Error received from push notification bouncer"))
# If we don't throw an exception, it's a successful bounce!

View File

@ -46,6 +46,7 @@ from zerver.models import (
)
from zerver.lib.request import JsonableError
from zilencer.models import get_remote_server_by_uuid
import base64
@ -268,9 +269,21 @@ class ZulipTestCase(TestCase):
API_KEYS[email] = get_user_profile_by_email(email).api_key
return API_KEYS[email]
def get_server_api_key(self, server_uuid):
# type: (Text) -> Text
if server_uuid not in API_KEYS:
API_KEYS[server_uuid] = get_remote_server_by_uuid(server_uuid).api_key
return API_KEYS[server_uuid]
def api_auth(self, email):
# type: (Text) -> Dict[str, Text]
credentials = u"%s:%s" % (email, self.get_api_key(email))
if "@" not in email:
api_key = self.get_server_api_key(email)
else:
api_key = self.get_api_key(email)
credentials = u"%s:%s" % (email, api_key)
return {
'HTTP_AUTHORIZATION': u'Basic ' + base64.b64encode(credentials.encode('utf-8')).decode('utf-8')
}

View File

@ -1,21 +1,30 @@
from __future__ import absolute_import
from __future__ import print_function
import mock
from mock import call
import time
from typing import Any, Dict, Union, SupportsInt, Text
import gcm
import ujson
from django.test import TestCase
from django.test import TestCase, override_settings
from django.conf import settings
from django.http import HttpResponse
from zerver.models import PushDeviceToken, UserProfile, Message
from zerver.models import get_user_profile_by_email, receives_online_notifications, \
receives_offline_notifications
from zerver.lib import push_notifications as apn
from zerver.lib.response import json_success
from zerver.lib.test_classes import (
ZulipTestCase,
)
from zilencer.models import RemoteZulipServer, RemotePushDeviceToken
from django.utils.timezone import now
class MockRedis(object):
data = {} # type: Dict[str, Any]
@ -40,6 +49,172 @@ class MockRedis(object):
# type: (*Any, **Any) -> None
pass
class PushBouncerNotificationTest(ZulipTestCase):
server_uuid = "1234-abcd"
def setUp(self):
# type: () -> None
server = RemoteZulipServer(uuid=self.server_uuid,
api_key="magic_secret_api_key",
hostname="demo.example.com",
last_updated=now())
server.save()
def tearDown(self):
# type: () -> None
RemoteZulipServer.objects.filter(uuid=self.server_uuid).delete()
def bounce_request(self, *args, **kwargs):
# type: (*Any, **Any) -> HttpResponse
"""This method is used to carry out the push notification bouncer
requests using the Django test browser, rather than python-requests.
"""
# args[0] is method, args[1] is URL.
local_url = args[1].replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "")
if args[0] == "POST":
result = self.client_post(local_url,
ujson.loads(kwargs['data']),
**self.get_auth())
else:
raise AssertionError("Unsupported method for bounce_request")
return result
def test_unregister_remote_push_user_params(self):
# type: () -> None
token = "111222"
token_kind = PushDeviceToken.GCM
endpoint = '/api/v1/remotes/push/unregister'
result = self.client_post(endpoint, {'token_kind': token_kind},
**self.get_auth())
self.assert_json_error(result, "Missing 'token' argument")
result = self.client_post(endpoint, {'token': token},
**self.get_auth())
self.assert_json_error(result, "Missing 'token_kind' argument")
result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind},
**self.api_auth("hamlet@zulip.com"))
self.assert_json_error(result, "Must validate with valid Zulip server API key")
def test_register_remote_push_user_paramas(self):
# type: () -> None
token = "111222"
user_id = 11
token_kind = PushDeviceToken.GCM
endpoint = '/api/v1/remotes/push/register'
result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind},
**self.get_auth())
self.assert_json_error(result, "Missing 'token' argument")
result = self.client_post(endpoint, {'user_id': user_id, 'token': token},
**self.get_auth())
self.assert_json_error(result, "Missing 'token_kind' argument")
result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind},
**self.get_auth())
self.assert_json_error(result, "Missing 'user_id' argument")
result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind,
'token': token},
**self.api_auth("hamlet@zulip.com"))
self.assert_json_error(result, "Must validate with valid Zulip server API key")
def test_remote_push_user_endpoints(self):
# type: () -> None
endpoints = [
('/api/v1/remotes/push/register', 'register'),
('/api/v1/remotes/push/unregister', 'unregister'),
]
for endpoint, method in endpoints:
payload = self.get_generic_payload(method)
# Verify correct results are success
result = self.client_post(endpoint, payload, **self.get_auth())
self.assert_json_success(result)
remote_tokens = RemotePushDeviceToken.objects.filter(token=payload['token'])
token_count = 1 if method == 'register' else 0
self.assertEqual(len(remote_tokens), token_count)
# Try adding/removing tokens that are too big...
broken_token = "x" * 5000 # too big
payload['token'] = broken_token
result = self.client_post(endpoint, payload, **self.get_auth())
self.assert_json_error(result, 'Empty or invalid length token')
@override_settings(PUSH_NOTIFICATION_BOUNCER_URL='https://push.zulip.org.example.com')
@mock.patch('zerver.lib.push_notifications.requests.request')
def test_push_bouncer_api(self, mock):
# type: (Any) -> None
"""This is a variant of the below test_push_api, but using the full
push notification bouncer flow
"""
mock.side_effect = self.bounce_request
email = "cordelia@zulip.com"
user = get_user_profile_by_email(email)
self.login(email)
server = RemoteZulipServer.objects.get(uuid=self.server_uuid)
endpoints = [
('/json/users/me/apns_device_token', 'apple-token'),
('/json/users/me/android_gcm_reg_id', 'android-token'),
]
# Test error handling
for endpoint, _ in endpoints:
# Try adding/removing tokens that are too big...
broken_token = "x" * 5000 # too big
result = self.client_post(endpoint, {'token': broken_token})
self.assert_json_error(result, 'Empty or invalid length token')
result = self.client_delete(endpoint, {'token': broken_token})
self.assert_json_error(result, 'Empty or invalid length token')
# Try to remove a non-existent token...
result = self.client_delete(endpoint, {'token': 'non-existent token'})
self.assert_json_error(result, 'Token does not exist')
# Add tokens
for endpoint, token in endpoints:
# Test that we can push twice
result = self.client_post(endpoint, {'token': token})
self.assert_json_success(result)
result = self.client_post(endpoint, {'token': token})
self.assert_json_success(result)
tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token,
server=server))
self.assertEqual(len(tokens), 1)
self.assertEqual(tokens[0].token, token)
# User should have tokens for both devices now.
tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id,
server=server))
self.assertEqual(len(tokens), 2)
# Remove tokens
for endpoint, token in endpoints:
result = self.client_delete(endpoint, {'token': token})
self.assert_json_success(result)
tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token,
server=server))
self.assertEqual(len(tokens), 0)
def get_generic_payload(self, method='register'):
# type: (Text) -> Dict[str, Any]
user_id = 10
token = "111222"
token_kind = PushDeviceToken.GCM
return {'user_id': user_id,
'token': token,
'token_kind': token_kind}
def get_auth(self):
# type: () -> Dict[str, Text]
# Auth on this user
return self.api_auth(self.server_uuid)
class PushNotificationTest(TestCase):
def setUp(self):
# type: () -> None

View File

@ -1,6 +1,9 @@
from __future__ import absolute_import
from typing import Optional
import requests
import json
from typing import Optional, Text
from django.conf import settings
from django.http import HttpRequest, HttpResponse

View File

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('zilencer', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='RemotePushDeviceToken',
fields=[
('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
('user_id', models.BigIntegerField()),
('kind', models.PositiveSmallIntegerField(choices=[(1, 'apns'), (2, 'gcm')])),
('token', models.CharField(unique=True, max_length=4096)),
('last_updated', models.DateTimeField(auto_now=True)),
('ios_app_id', models.TextField(null=True)),
],
),
migrations.CreateModel(
name='RemoteZulipServer',
fields=[
('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
('uuid', models.CharField(unique=True, max_length=36)),
('api_key', models.CharField(max_length=64)),
('hostname', models.CharField(unique=True, max_length=128)),
('contact_email', models.EmailField(max_length=254, blank=True)),
('last_updated', models.DateTimeField(verbose_name='last updated')),
],
),
migrations.AddField(
model_name='remotepushdevicetoken',
name='server',
field=models.ForeignKey(to='zilencer.RemoteZulipServer'),
),
]

View File

@ -1,12 +1,27 @@
from django.db import models
from django.db.models import Manager
from typing import Dict, Text
from typing import Dict, Optional, Text
import zerver.models
import datetime
def get_deployment_by_domain(realm_str):
# type: (Text) -> Deployment
return Deployment.objects.get(realms__string_id=realm_str)
def get_remote_server_by_uuid(uuid):
# type: (Text) -> RemoteZulipServer
return RemoteZulipServer.objects.get(uuid=uuid)
class RemoteZulipServer(models.Model):
uuid = models.CharField(max_length=36, unique=True) # type: Text
api_key = models.CharField(max_length=64) # type: Text
hostname = models.CharField(max_length=128, unique=True) # type: Text
contact_email = models.EmailField(blank=True, null=False) # type: Text
last_updated = models.DateTimeField('last updated') # type: datetime.datetime
# Variant of PushDeviceToken for a remote server.
class RemotePushDeviceToken(zerver.models.AbstractPushDeviceToken):
server = models.ForeignKey(RemoteZulipServer) # type: RemoteZulipServer
# The user id on the remote server for this device device this is
user_id = models.BigIntegerField() # type: int
class Deployment(models.Model):
realms = models.ManyToManyField(zerver.models.Realm,
@ -32,5 +47,5 @@ class Deployment(models.Model):
# TODO: This only does the right thing for prod because prod authenticates to
# staging with the zulip.com deployment key, while staging is technically the
# deployment for the zulip.com realm.
# This also doesn't necessarily handle other multi-realm deployments correctly.
# This also doesn't necessarily handle other multi-realm deployments correctly
return self.realms.order_by('pk')[0].domain

View File

@ -12,6 +12,10 @@ i18n_urlpatterns = [] # type: Any
v1_api_and_json_patterns = [
url('^deployment/report_error$', rest_dispatch,
{'POST': 'zerver.views.report.report_error'}),
url('^remotes/push/register$', rest_dispatch,
{'POST': 'zilencer.views.remote_server_register_push'}),
url('^remotes/push/unregister$', rest_dispatch,
{'POST': 'zilencer.views.remote_server_unregister_push'}),
]
urlpatterns = [

View File

@ -1,17 +1,69 @@
from __future__ import absolute_import
from django.utils.translation import ugettext as _
from django.utils import timezone
from django.http import HttpResponse, HttpRequest
from zilencer.models import Deployment
from zilencer.models import Deployment, RemotePushDeviceToken, RemoteZulipServer
from zerver.decorator import has_request_variables, REQ
from zerver.lib.error_notify import do_report_error
from zerver.lib.push_notifications import send_android_push_notification, \
send_apple_push_notification
from zerver.lib.response import json_error, json_success
from zerver.lib.validator import check_dict
from zerver.models import UserProfile, PushDeviceToken, Realm
from typing import Any, Dict, Text
from typing import Any, Dict, Optional, Union, Text, cast
@has_request_variables
def report_error(request, deployment, type=REQ(), report=REQ(validator=check_dict([]))):
# type: (HttpRequest, Deployment, Text, Dict[str, Any]) -> HttpResponse
return do_report_error(deployment.name, type, report)
@has_request_variables
def remote_server_register_push(request, entity, user_id=REQ(),
token=REQ(), token_kind=REQ(), ios_app_id=None):
# type: (HttpRequest, Union[UserProfile, RemoteZulipServer], int, Text, int, Optional[Text]) -> HttpResponse
if not isinstance(entity, RemoteZulipServer):
return json_error(_("Must validate with valid Zulip server API key"))
if token == '' or len(token) > 4096:
return json_error(_("Empty or invalid length token"))
server = cast(RemoteZulipServer, entity)
# If a user logged out on a device and failed to unregister,
# we should delete any other user associations for this token
# & RemoteServer pair
RemotePushDeviceToken.objects.filter(
token=token, kind=token_kind, server=server).exclude(user_id=user_id).delete()
# Save or update
remote_token, created = RemotePushDeviceToken.objects.update_or_create(
user_id=user_id,
server=server,
kind=token_kind,
token=token,
defaults=dict(
ios_app_id=ios_app_id,
last_updated=timezone.now()))
return json_success()
@has_request_variables
def remote_server_unregister_push(request, entity, token=REQ(),
token_kind=REQ(), ios_app_id=None):
# type: (HttpRequest, Union[UserProfile, RemoteZulipServer], Text, int, Optional[Text]) -> HttpResponse
if not isinstance(entity, RemoteZulipServer):
return json_error(_("Must validate with valid Zulip server API key"))
if token == '' or len(token) > 4096:
return json_error(_("Empty or invalid length token"))
server = cast(RemoteZulipServer, entity)
deleted = RemotePushDeviceToken.objects.filter(token=token,
kind=token_kind,
server=server).delete()
if deleted[0] == 0:
return json_error(_("Token does not exist"))
return json_success()

View File

@ -213,6 +213,7 @@ DEFAULT_SETTINGS = {'TWITTER_CONSUMER_KEY': '',
'PASSWORD_MIN_LENGTH': 6,
'PASSWORD_MIN_ZXCVBN_QUALITY': 0.5,
'OFFLINE_THRESHOLD_SECS': 5 * 60,
'PUSH_NOTIFICATION_BOUNCER_URL': None,
}
for setting_name, setting_val in six.iteritems(DEFAULT_SETTINGS):