Add support infrastructure for push notification bouncer service.

This is an incomplete cleaned-up continuation of Lisa Neigut's push
notification bouncer work.  It supports registration and
deregistration of individual push tokens with a central push
notification bouncer server.

It still is missing a few things before we can complete this effort:
* A registration form for server admins to configure their server for
  this service, with tests.
* Code (and tests) for actually bouncing the notifications.

zerver/decorator.py

@@ -30,9 +30,18 @@ from io import BytesIO
 from zerver.lib.mandrill_client import get_mandrill_client
 from six.moves import zip, urllib
 
-from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text
+from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text, cast
 from zerver.lib.str_utils import force_bytes
 
+# This is a hack to ensure that RemoteZulipServer always exists even
+# if Zilencer isn't enabled.
+if settings.ZILENCER_ENABLED:
+    from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
+else:
+    from mock import Mock
+    get_remote_server_by_uuid = Mock()
+    RemoteZulipServer = Mock() # type: ignore # https://github.com/JukkaL/mypy/issues/1188
+
 FuncT = TypeVar('FuncT', bound=Callable[..., Any])
 ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse])
 
@@ -155,14 +164,20 @@ def process_client(request, user_profile, is_json_view=False, client_name=None):
     update_user_activity(request, user_profile)
 
 def validate_api_key(request, role, api_key, is_webhook=False):
-    # type: (HttpRequest, Text, Text, bool) -> UserProfile
+    # type: (HttpRequest, Text, Text, bool) -> Union[UserProfile, RemoteZulipServer]
     # Remove whitespace to protect users from trivial errors.
     role, api_key = role.strip(), api_key.strip()
 
-    try:
+    if "@" in role:
-        profile = get_user_profile_by_email(role)
+        try:
-    except UserProfile.DoesNotExist:
+            profile = get_user_profile_by_email(role) # type: Union[UserProfile, RemoteZulipServer]
-        raise JsonableError(_("Invalid user: %s") % (role,))
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid user: %s") % (role,))
+    else:
+        try:
+            profile = get_remote_server_by_uuid(role)
+        except RemoteZulipServer.DoesNotExist:
+            raise JsonableError(_("Invalid Zulip server: %s") % (role,))
 
     if api_key != profile.api_key:
         if len(api_key) != 32:
@@ -171,6 +186,14 @@ def validate_api_key(request, role, api_key, is_webhook=False):
         else:
             reason = _("Invalid API key for role '%s'")
         raise JsonableError(reason % (role,))
+
+    # early exit for RemoteZulipServer instances
+    if settings.ZILENCER_ENABLED and isinstance(profile, RemoteZulipServer):
+        if not check_subdomain(get_subdomain(request), ""):
+            raise JsonableError(_("This API key only works on the root subdomain"))
+        return profile
+
+    profile = cast(UserProfile, profile) # is UserProfile
     if not profile.is_active:
         raise JsonableError(_("Account not active"))
     if profile.is_incoming_webhook and not is_webhook:
@@ -389,13 +412,18 @@ def authenticated_rest_api_view(is_webhook=False):
 
             # Now we try to do authentication or die
             try:
-                # role is a UserProfile
+                # profile is a Union[UserProfile, RemoteZulipServer]
                 profile = validate_api_key(request, role, api_key, is_webhook)
             except JsonableError as e:
                 return json_unauthorized(e.error)
             request.user = profile
             process_client(request, profile)
-            request._email = profile.email
+            if isinstance(profile, UserProfile):
+                request._email = profile.email
+            else:
+                assert isinstance(profile, RemoteZulipServer)  # type: ignore # https://github.com/python/mypy/issues/2957
+                request._email = "zulip-server:" + role
+                profile.rate_limits = ""
             # Apply rate limiting
             return rate_limit()(view_func)(request, profile, *args, **kwargs)
         return _wrapped_func_arguments

zerver/lib/push_notifications.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import
 
 import random
+import requests
 from typing import Any, Dict, List, Optional, SupportsInt, Text
 
+from version import ZULIP_VERSION
 from zerver.models import PushDeviceToken, Message, Recipient, UserProfile, \
     UserMessage, get_display_recipient, receives_offline_notifications, \
     receives_online_notifications
@@ -20,12 +22,14 @@ from gcm import GCM
 from django.conf import settings
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import ugettext as _
+from six.moves import urllib
 
 import base64
 import binascii
 import logging
 import os
 import time
+import ujson
 from functools import partial
 
 # APNS error codes
@@ -343,6 +347,22 @@ def handle_push_notification(user_profile_id, missed_message):
 def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
     # type: (UserProfile, str, int, Optional[str]) -> None
 
+    # If we're sending things to the push notification bouncer
+    # register this user with them here
+    if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None:
+        post_data = {
+            'server_uuid': settings.ZULIP_ORG_ID,
+            'user_id': user_profile.id,
+            'token': token_str,
+            'token_kind': kind,
+        }
+
+        if kind == PushDeviceToken.APNS:
+            post_data['ios_app_id'] = ios_app_id
+
+        send_to_push_bouncer('POST', 'register', post_data)
+        return
+
     # If another user was previously logged in on the same device and didn't
     # properly log out, the token will still be registered to the wrong account
     PushDeviceToken.objects.filter(token=token_str).exclude(user=user_profile).delete()
@@ -359,8 +379,52 @@ def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
 
 def remove_push_device_token(user_profile, token_str, kind):
     # type: (UserProfile, str, int) -> None
+
+    # If we're sending things to the push notification bouncer
+    # register this user with them here
+    if settings.PUSH_NOTIFICATION_BOUNCER_URL is not None:
+        # TODO: Make this a remove item
+        post_data = {
+            'server_uuid': settings.ZULIP_ORG_ID,
+            'user_id': user_profile.id,
+            'token': token_str,
+            'token_kind': kind,
+        }
+        send_to_push_bouncer("POST", "unregister", post_data)
+        return
+
     try:
         token = PushDeviceToken.objects.get(token=token_str, kind=kind)
         token.delete()
     except PushDeviceToken.DoesNotExist:
         raise JsonableError(_("Token does not exist"))
+
+
+def send_to_push_bouncer(method, endpoint, post_data):
+    # type: (str, str, Dict[str, Any]) -> None
+    url = urllib.parse.urljoin(settings.PUSH_NOTIFICATION_BOUNCER_URL,
+                               '/api/v1/remotes/push/' + endpoint)
+    api_auth = requests.auth.HTTPBasicAuth(settings.ZULIP_ORG_ID,
+                                           settings.ZULIP_ORG_KEY)
+
+    res = requests.request(method,
+                           url,
+                           data=ujson.dumps(post_data),
+                           auth=api_auth,
+                           timeout=30,
+                           verify=True,
+                           headers={"User-agent": "ZulipServer/%s" % (ZULIP_VERSION,)})
+
+    # TODO: Think more carefully about how this error hanlding should work.
+    if res.status_code >= 500:
+        raise JsonableError(_("Error received from push notification bouncer"))
+    elif res.status_code >= 400:
+        try:
+            msg = ujson.loads(res.content)['msg']
+        except Exception:
+            raise JsonableError(_("Error received from push notification bouncer"))
+        raise JsonableError(msg)
+    elif res.status_code != 200:
+        raise JsonableError(_("Error received from push notification bouncer"))
+
+    # If we don't throw an exception, it's a successful bounce!

zerver/lib/test_classes.py

@@ -46,6 +46,7 @@ from zerver.models import (
 )
 
 from zerver.lib.request import JsonableError
+from zilencer.models import get_remote_server_by_uuid
 
 
 import base64
@@ -268,9 +269,21 @@ class ZulipTestCase(TestCase):
             API_KEYS[email] = get_user_profile_by_email(email).api_key
         return API_KEYS[email]
 
+    def get_server_api_key(self, server_uuid):
+        # type: (Text) -> Text
+        if server_uuid not in API_KEYS:
+            API_KEYS[server_uuid] = get_remote_server_by_uuid(server_uuid).api_key
+
+        return API_KEYS[server_uuid]
+
     def api_auth(self, email):
         # type: (Text) -> Dict[str, Text]
-        credentials = u"%s:%s" % (email, self.get_api_key(email))
+        if "@" not in email:
+            api_key = self.get_server_api_key(email)
+        else:
+            api_key = self.get_api_key(email)
+
+        credentials = u"%s:%s" % (email, api_key)
         return {
             'HTTP_AUTHORIZATION': u'Basic ' + base64.b64encode(credentials.encode('utf-8')).decode('utf-8')
         }

zerver/tests/test_push_notifications.py

@@ -1,21 +1,30 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 import mock
 from mock import call
 import time
 from typing import Any, Dict, Union, SupportsInt, Text
 
 import gcm
+import ujson
 
-from django.test import TestCase
+from django.test import TestCase, override_settings
 from django.conf import settings
+from django.http import HttpResponse
 
 from zerver.models import PushDeviceToken, UserProfile, Message
 from zerver.models import get_user_profile_by_email, receives_online_notifications, \
     receives_offline_notifications
 from zerver.lib import push_notifications as apn
+from zerver.lib.response import json_success
 from zerver.lib.test_classes import (
     ZulipTestCase,
 )
 
+from zilencer.models import RemoteZulipServer, RemotePushDeviceToken
+from django.utils.timezone import now
+
 class MockRedis(object):
     data = {}  # type: Dict[str, Any]
 
@@ -40,6 +49,172 @@ class MockRedis(object):
         # type: (*Any, **Any) -> None
         pass
 
+class PushBouncerNotificationTest(ZulipTestCase):
+    server_uuid = "1234-abcd"
+
+    def setUp(self):
+        # type: () -> None
+        server = RemoteZulipServer(uuid=self.server_uuid,
+                                   api_key="magic_secret_api_key",
+                                   hostname="demo.example.com",
+                                   last_updated=now())
+        server.save()
+
+    def tearDown(self):
+        # type: () -> None
+        RemoteZulipServer.objects.filter(uuid=self.server_uuid).delete()
+
+    def bounce_request(self, *args, **kwargs):
+        # type: (*Any, **Any) -> HttpResponse
+        """This method is used to carry out the push notification bouncer
+        requests using the Django test browser, rather than python-requests.
+        """
+        # args[0] is method, args[1] is URL.
+        local_url = args[1].replace(settings.PUSH_NOTIFICATION_BOUNCER_URL, "")
+        if args[0] == "POST":
+            result = self.client_post(local_url,
+                                      ujson.loads(kwargs['data']),
+                                      **self.get_auth())
+        else:
+            raise AssertionError("Unsupported method for bounce_request")
+        return result
+
+    def test_unregister_remote_push_user_params(self):
+        # type: () -> None
+        token = "111222"
+        token_kind = PushDeviceToken.GCM
+
+        endpoint = '/api/v1/remotes/push/unregister'
+        result = self.client_post(endpoint, {'token_kind': token_kind},
+                                  **self.get_auth())
+        self.assert_json_error(result, "Missing 'token' argument")
+        result = self.client_post(endpoint, {'token': token},
+                                  **self.get_auth())
+        self.assert_json_error(result, "Missing 'token_kind' argument")
+        result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind},
+                                  **self.api_auth("hamlet@zulip.com"))
+        self.assert_json_error(result, "Must validate with valid Zulip server API key")
+
+    def test_register_remote_push_user_paramas(self):
+        # type: () -> None
+        token = "111222"
+        user_id = 11
+        token_kind = PushDeviceToken.GCM
+
+        endpoint = '/api/v1/remotes/push/register'
+
+        result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind},
+                                  **self.get_auth())
+        self.assert_json_error(result, "Missing 'token' argument")
+        result = self.client_post(endpoint, {'user_id': user_id, 'token': token},
+                                  **self.get_auth())
+        self.assert_json_error(result, "Missing 'token_kind' argument")
+        result = self.client_post(endpoint, {'token': token, 'token_kind': token_kind},
+                                  **self.get_auth())
+        self.assert_json_error(result, "Missing 'user_id' argument")
+        result = self.client_post(endpoint, {'user_id': user_id, 'token_kind': token_kind,
+                                             'token': token},
+                                  **self.api_auth("hamlet@zulip.com"))
+        self.assert_json_error(result, "Must validate with valid Zulip server API key")
+
+    def test_remote_push_user_endpoints(self):
+        # type: () -> None
+        endpoints = [
+            ('/api/v1/remotes/push/register', 'register'),
+            ('/api/v1/remotes/push/unregister', 'unregister'),
+        ]
+
+        for endpoint, method in endpoints:
+            payload = self.get_generic_payload(method)
+
+            # Verify correct results are success
+            result = self.client_post(endpoint, payload, **self.get_auth())
+            self.assert_json_success(result)
+
+            remote_tokens = RemotePushDeviceToken.objects.filter(token=payload['token'])
+            token_count = 1 if method == 'register' else 0
+            self.assertEqual(len(remote_tokens), token_count)
+
+            # Try adding/removing tokens that are too big...
+            broken_token = "x" * 5000 # too big
+            payload['token'] = broken_token
+            result = self.client_post(endpoint, payload, **self.get_auth())
+            self.assert_json_error(result, 'Empty or invalid length token')
+
+    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL='https://push.zulip.org.example.com')
+    @mock.patch('zerver.lib.push_notifications.requests.request')
+    def test_push_bouncer_api(self, mock):
+        # type: (Any) -> None
+        """This is a variant of the below test_push_api, but using the full
+        push notification bouncer flow
+        """
+        mock.side_effect = self.bounce_request
+        email = "cordelia@zulip.com"
+        user = get_user_profile_by_email(email)
+        self.login(email)
+        server = RemoteZulipServer.objects.get(uuid=self.server_uuid)
+
+        endpoints = [
+            ('/json/users/me/apns_device_token', 'apple-token'),
+            ('/json/users/me/android_gcm_reg_id', 'android-token'),
+        ]
+
+        # Test error handling
+        for endpoint, _ in endpoints:
+            # Try adding/removing tokens that are too big...
+            broken_token = "x" * 5000 # too big
+            result = self.client_post(endpoint, {'token': broken_token})
+            self.assert_json_error(result, 'Empty or invalid length token')
+
+            result = self.client_delete(endpoint, {'token': broken_token})
+            self.assert_json_error(result, 'Empty or invalid length token')
+
+            # Try to remove a non-existent token...
+            result = self.client_delete(endpoint, {'token': 'non-existent token'})
+            self.assert_json_error(result, 'Token does not exist')
+
+        # Add tokens
+        for endpoint, token in endpoints:
+            # Test that we can push twice
+            result = self.client_post(endpoint, {'token': token})
+            self.assert_json_success(result)
+
+            result = self.client_post(endpoint, {'token': token})
+            self.assert_json_success(result)
+
+            tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token,
+                                                               server=server))
+            self.assertEqual(len(tokens), 1)
+            self.assertEqual(tokens[0].token, token)
+
+        # User should have tokens for both devices now.
+        tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id,
+                                                           server=server))
+        self.assertEqual(len(tokens), 2)
+
+        # Remove tokens
+        for endpoint, token in endpoints:
+            result = self.client_delete(endpoint, {'token': token})
+            self.assert_json_success(result)
+            tokens = list(RemotePushDeviceToken.objects.filter(user_id=user.id, token=token,
+                                                               server=server))
+            self.assertEqual(len(tokens), 0)
+
+    def get_generic_payload(self, method='register'):
+        # type: (Text) -> Dict[str, Any]
+        user_id = 10
+        token = "111222"
+        token_kind = PushDeviceToken.GCM
+
+        return {'user_id': user_id,
+                'token': token,
+                'token_kind': token_kind}
+
+    def get_auth(self):
+        # type: () -> Dict[str, Text]
+        # Auth on this user
+        return self.api_auth(self.server_uuid)
+
 class PushNotificationTest(TestCase):
     def setUp(self):
         # type: () -> None

zerver/views/push_notifications.py

@@ -1,6 +1,9 @@
 from __future__ import absolute_import
 
-from typing import Optional
+import requests
+import json
+
+from typing import Optional, Text
 
 from django.conf import settings
 from django.http import HttpRequest, HttpResponse

zilencer/migrations/0002_remote_zulip_server.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zilencer', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RemotePushDeviceToken',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('user_id', models.BigIntegerField()),
+                ('kind', models.PositiveSmallIntegerField(choices=[(1, 'apns'), (2, 'gcm')])),
+                ('token', models.CharField(unique=True, max_length=4096)),
+                ('last_updated', models.DateTimeField(auto_now=True)),
+                ('ios_app_id', models.TextField(null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='RemoteZulipServer',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('uuid', models.CharField(unique=True, max_length=36)),
+                ('api_key', models.CharField(max_length=64)),
+                ('hostname', models.CharField(unique=True, max_length=128)),
+                ('contact_email', models.EmailField(max_length=254, blank=True)),
+                ('last_updated', models.DateTimeField(verbose_name='last updated')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='remotepushdevicetoken',
+            name='server',
+            field=models.ForeignKey(to='zilencer.RemoteZulipServer'),
+        ),
+    ]

zilencer/models.py

@@ -1,12 +1,27 @@
 from django.db import models
 from django.db.models import Manager
-from typing import Dict, Text
+from typing import Dict, Optional, Text
 
 import zerver.models
+import datetime
 
-def get_deployment_by_domain(realm_str):
+def get_remote_server_by_uuid(uuid):
-    # type: (Text) -> Deployment
+    # type: (Text) -> RemoteZulipServer
-    return Deployment.objects.get(realms__string_id=realm_str)
+    return RemoteZulipServer.objects.get(uuid=uuid)
+
+class RemoteZulipServer(models.Model):
+    uuid = models.CharField(max_length=36, unique=True) # type: Text
+    api_key = models.CharField(max_length=64) # type: Text
+
+    hostname = models.CharField(max_length=128, unique=True) # type: Text
+    contact_email = models.EmailField(blank=True, null=False) # type: Text
+    last_updated = models.DateTimeField('last updated') # type: datetime.datetime
+
+# Variant of PushDeviceToken for a remote server.
+class RemotePushDeviceToken(zerver.models.AbstractPushDeviceToken):
+    server = models.ForeignKey(RemoteZulipServer)  # type: RemoteZulipServer
+    # The user id on the remote server for this device device this is
+    user_id = models.BigIntegerField()  # type: int
 
 class Deployment(models.Model):
     realms = models.ManyToManyField(zerver.models.Realm,
@@ -32,5 +47,5 @@ class Deployment(models.Model):
         # TODO: This only does the right thing for prod because prod authenticates to
         # staging with the zulip.com deployment key, while staging is technically the
         # deployment for the zulip.com realm.
-        # This also doesn't necessarily handle other multi-realm deployments correctly.
+        # This also doesn't necessarily handle other multi-realm deployments correctly
         return self.realms.order_by('pk')[0].domain

zilencer/urls.py

@@ -12,6 +12,10 @@ i18n_urlpatterns = [] # type: Any
 v1_api_and_json_patterns = [
     url('^deployment/report_error$', rest_dispatch,
         {'POST': 'zerver.views.report.report_error'}),
+    url('^remotes/push/register$', rest_dispatch,
+        {'POST': 'zilencer.views.remote_server_register_push'}),
+    url('^remotes/push/unregister$', rest_dispatch,
+        {'POST': 'zilencer.views.remote_server_unregister_push'}),
 ]
 
 urlpatterns = [

zilencer/views.py

@@ -1,17 +1,69 @@
 from __future__ import absolute_import
 
 from django.utils.translation import ugettext as _
+from django.utils import timezone
 from django.http import HttpResponse, HttpRequest
 
-from zilencer.models import Deployment
+from zilencer.models import Deployment, RemotePushDeviceToken, RemoteZulipServer
 
 from zerver.decorator import has_request_variables, REQ
 from zerver.lib.error_notify import do_report_error
+from zerver.lib.push_notifications import send_android_push_notification, \
+    send_apple_push_notification
+from zerver.lib.response import json_error, json_success
 from zerver.lib.validator import check_dict
+from zerver.models import UserProfile, PushDeviceToken, Realm
 
-from typing import Any, Dict, Text
+from typing import Any, Dict, Optional, Union, Text, cast
 
 @has_request_variables
 def report_error(request, deployment, type=REQ(), report=REQ(validator=check_dict([]))):
     # type: (HttpRequest, Deployment, Text, Dict[str, Any]) -> HttpResponse
     return do_report_error(deployment.name, type, report)
+
+@has_request_variables
+def remote_server_register_push(request, entity, user_id=REQ(),
+                                token=REQ(), token_kind=REQ(), ios_app_id=None):
+    # type: (HttpRequest, Union[UserProfile, RemoteZulipServer], int, Text, int, Optional[Text]) -> HttpResponse
+    if not isinstance(entity, RemoteZulipServer):
+        return json_error(_("Must validate with valid Zulip server API key"))
+    if token == '' or len(token) > 4096:
+        return json_error(_("Empty or invalid length token"))
+
+    server = cast(RemoteZulipServer, entity)
+
+    # If a user logged out on a device and failed to unregister,
+    # we should delete any other user associations for this token
+    # & RemoteServer pair
+    RemotePushDeviceToken.objects.filter(
+        token=token, kind=token_kind, server=server).exclude(user_id=user_id).delete()
+
+    # Save or update
+    remote_token, created = RemotePushDeviceToken.objects.update_or_create(
+        user_id=user_id,
+        server=server,
+        kind=token_kind,
+        token=token,
+        defaults=dict(
+            ios_app_id=ios_app_id,
+            last_updated=timezone.now()))
+
+    return json_success()
+
+@has_request_variables
+def remote_server_unregister_push(request, entity, token=REQ(),
+                                  token_kind=REQ(), ios_app_id=None):
+    # type: (HttpRequest, Union[UserProfile, RemoteZulipServer], Text, int, Optional[Text]) -> HttpResponse
+    if not isinstance(entity, RemoteZulipServer):
+        return json_error(_("Must validate with valid Zulip server API key"))
+    if token == '' or len(token) > 4096:
+        return json_error(_("Empty or invalid length token"))
+
+    server = cast(RemoteZulipServer, entity)
+    deleted = RemotePushDeviceToken.objects.filter(token=token,
+                                                   kind=token_kind,
+                                                   server=server).delete()
+    if deleted[0] == 0:
+        return json_error(_("Token does not exist"))
+
+    return json_success()

zproject/settings.py

@@ -213,6 +213,7 @@ DEFAULT_SETTINGS = {'TWITTER_CONSUMER_KEY': '',
                     'PASSWORD_MIN_LENGTH': 6,
                     'PASSWORD_MIN_ZXCVBN_QUALITY': 0.5,
                     'OFFLINE_THRESHOLD_SECS': 5 * 60,
+                    'PUSH_NOTIFICATION_BOUNCER_URL': None,
                     }
 
 for setting_name, setting_val in six.iteritems(DEFAULT_SETTINGS):