[schema] Support for authenticating Deployments via the API.

Here we introduce a new Django app, zilencer. The intent is to not have
this app enabled on LOCALSERVER instances, and for it to grow to include
all the functionality we want to have in our central server that isn't
relevant for local deployments.

Currently we have to modify functions in zerver/* to match; in the
future, it would be cool to have the relevant shared code broken out
into a separate library.

This commit inclues both the migration to create the models as well as a
data migration that (for non-LOCALSERVER) creates a single default
Deployment for zulip.com.

To apply this migration to your system, run:
   ./manage.py migrate zilencer

(imported from commit 86d5497ac120e03fa7f298a9cc08b192d5939b43)

zerver/decorator.py

@@ -18,12 +18,17 @@ from zerver.lib.utils import statsd
 from zerver.exceptions import RateLimited
 from zerver.lib.rate_limiter import incr_ratelimit, is_ratelimited, \
      api_calls_left
+
+from zilencer.models import get_deployment_by_domain, Deployment
 from functools import wraps
 import base64
 import logging
 import cProfile
 from zerver.lib.mandrill_client import get_mandrill_client
 
+def get_deployment_or_userprofile(role):
+    return get_user_profile_by_email(role) if "@" in role else get_deployment_by_domain(role)
+
 class _RespondAsynchronously(object):
     pass
 
@@ -75,24 +80,26 @@ def process_client(request, user_profile, default):
 
     update_user_activity(request, user_profile)
 
-def validate_api_key(email, api_key):
+def validate_api_key(role, api_key):
     # Remove whitespace to protect users from trivial errors.
-    email, api_key = email.strip(), api_key.strip()
+    role, api_key = role.strip(), api_key.strip()
 
     try:
-        user_profile = get_user_profile_by_email(email)
+        profile = get_deployment_or_userprofile(role)
     except UserProfile.DoesNotExist:
-        raise JsonableError("Invalid user: %s" % (email,))
+        raise JsonableError("Invalid user: %s" % (role,))
+    except Deployment.DoesNotExist:
+        raise JsonableError("Invalid deployment: %s" % (role,))
 
-    if api_key != user_profile.api_key:
+    if api_key != profile.api_key:
         if len(api_key) != 32:
             reason = "Incorrect API key length (keys should be 32 characters long)"
         else:
             reason = "Invalid API key"
-        raise JsonableError(reason + " for user '%s'" % (email,))
-    if not user_profile.is_active:
-        raise JsonableError("User account is not active")
-    return user_profile
+        raise JsonableError(reason + " for role '%s'" % (role,))
+    if not profile.is_active:
+        raise JsonableError("Account not active")
+    return profile
 
 # Use this for webhook views that don't get an email passed in.
 def api_key_only_webhook_view(view_func):
@@ -158,7 +165,7 @@ def authenticated_rest_api_view(view_func):
             # case insensitive per RFC 1945
             if auth_type.lower() != "basic":
                 return json_error("Only Basic authentication is supported.")
-            email, api_key = base64.b64decode(encoded_value).split(":")
+            role, api_key = base64.b64decode(encoded_value).split(":")
         except ValueError:
             return json_error("Invalid authorization header for basic auth")
         except KeyError:
@@ -166,15 +173,19 @@ def authenticated_rest_api_view(view_func):
 
         # Now we try to do authentication or die
         try:
-            user_profile = validate_api_key(email, api_key)
+            # Could be a UserProfile or a Deployment
+            profile = validate_api_key(role, api_key)
         except JsonableError, e:
             return json_unauthorized(e.error)
-        request.user = user_profile
-        request._email = user_profile.email
-        process_client(request, user_profile, "API")
+        request.user = profile
+        process_client(request, profile, "API")
+        if isinstance(profile, UserProfile):
+            request._email = profile.email
+        else:
+            request._email = "deployment:" + role
+            profile.rate_limits = ""
         # Apply rate limiting
-        limited_func = rate_limit()(view_func)
-        return limited_func(request, user_profile, *args, **kwargs)
+        return rate_limit()(view_func)(request, profile, *args, **kwargs)
     return _wrapped_view_func
 
 def process_as_post(view_func):
@@ -422,7 +433,7 @@ def rate_limit_user(request, user, domain):
     request._ratelimit_over_limit = ratelimited
     # Abort this request if the user is over her rate limits
     if ratelimited:
-        statsd.incr("ratelimiter.limited.%s" % user.id)
+        statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
         raise RateLimited()
 
     incr_ratelimit(user, domain)

zerver/lib/rate_limiter.py

@@ -20,7 +20,7 @@ def _rules_for_user(user):
 
 def redis_key(user, domain):
     """Return the redis keys for this user"""
-    return ["ratelimit:%s:%s:%s" % (user.id, domain, keytype) for keytype in ['list', 'zset', 'block']]
+    return ["ratelimit:%s:%s:%s:%s" % (type(user), user.id, domain, keytype) for keytype in ['list', 'zset', 'block']]
 
 def max_api_calls(user):
     "Returns the API rate limit for the highest limit"

zerver/management/commands/populate_db.py

@@ -19,6 +19,7 @@ from zerver.lib.bulk_create import bulk_create_realms, \
 from zerver.lib.timestamp import timestamp_to_datetime
 from zerver.models import MAX_MESSAGE_LENGTH
 from zerver.models import DefaultStream, get_stream
+from zilencer.models import Deployment
 
 import ujson
 import datetime
@@ -119,6 +120,12 @@ class Command(BaseCommand):
             for realm in Realm.objects.all():
                 realms[realm.domain] = realm
 
+            if not settings.LOCALSERVER:
+                # Associate initial deployment with Realm
+                dep = Deployment.objects.all()[0]
+                dep.realms = [realms["zulip.com"]]
+                dep.save()
+
             # Create test Users (UserProfiles are automatically created,
             # as are subscriptions to the ability to receive personals).
             names = [("Othello, the Moor of Venice", "othello@zulip.com"), ("Iago", "iago@zulip.com"),

zerver/models.py

@@ -105,6 +105,17 @@ class Realm(models.Model):
     def get_emoji(self):
         return get_realm_emoji_uncached(self)
 
+    @property
+    def deployment(self):
+        try:
+            return self._deployments.all()[0]
+        except IndexError:
+            return None
+
+    @deployment.setter
+    def set_deployments(self, value):
+        self._deployments = [value]
+
     class Meta:
         permissions = (
             ('administer', "Administer a realm"),

zilencer/README.md

@@ -0,0 +1,9 @@
+ZILENCER -- The Zulip License Manager
+========
+
+This app is the place for storing state about various deployments of Zulip that
+exist in the world.
+
+Models for our centralized Zulip-the-company services like billing, local
+server tracking, etc. that should not be hosted by local server instances
+belong in this app.

zilencer/management/commands/create_deployment.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import
+from optparse import make_option
+import re
+import sys
+
+from django.core.management.base import BaseCommand
+
+from zerver.models import get_realm
+from zerver.lib.create_user import random_api_key
+from zerver.management.commands.create_realm import Command as CreateRealm
+
+from zilencer.models import Deployment
+
+class Command(BaseCommand):
+    help = """Create a deployment and accompanying realm."""
+
+    option_list = CreateRealm.option_list + (
+        make_option('--no-realm',
+                    dest='no_realm',
+                    action='store_true',
+                    default=False,
+                    help='Do not create a new realm; associate with an existing one.' + \
+                           ' In this case, only the domain needs to be specified.'),
+        )
+
+    def handle(self, *args, **options):
+        if options["domain"] is None:
+            print >>sys.stderr, "\033[1;31mPlease provide a domain.\033[0m\n"
+            self.print_help("python manage.py", "create_realm")
+            exit(1)
+
+        if not options["no_realm"]:
+            CreateRealm().handle(*args, **options)
+            print # Newline
+
+        realm = get_realm(options["domain"])
+        if realm is None:
+            print >>sys.stderr, "\033[1;31mRealm does not exist!\033[0m\n"
+            exit(2)
+
+        dep = Deployment()
+        dep.api_key = random_api_key()
+        dep.save()
+        dep.realms = [realm]
+        dep.save()
+        print "Deployment created."
+

zilencer/migrations/0001_initial.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+from django.conf import settings
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Deployment'
+        db.create_table(u'zilencer_deployment', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('is_active', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('api_key', self.gf('django.db.models.fields.CharField')(max_length=32, null=True)),
+        ))
+        db.send_create_signal(u'zilencer', ['Deployment'])
+
+        # Adding M2M table for field realms on 'Deployment'
+        db.create_table(u'zilencer_deployment_realms', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('deployment', models.ForeignKey(orm[u'zilencer.deployment'], null=False)),
+            ('realm', models.ForeignKey(orm[u'zerver.realm'], null=False))
+        ))
+        db.create_unique(u'zilencer_deployment_realms', ['deployment_id', 'realm_id'])
+
+        if not settings.LOCALSERVER:
+            try:
+                dep = orm['zilencer.Deployment']()
+                dep.api_key = settings.DEPLOYMENT_ROLE_KEY
+                dep.save()
+                dep.realms = [orm['zerver.Realm'].objects.get(domain="zulip.com")]
+                dep.save()
+
+                dep = orm['zilencer.Deployment']()
+                dep.api_key = settings.DEPLOYMENT_ROLE_KEY
+                dep.save()
+                dep.realms = orm['zerver.Realm'].objects.annotate(dc=models.Count("_deployments")).filter(dc=0)
+                dep.save()
+            except orm['zerver.Realm'].DoesNotExist:
+                pass
+
+
+
+    def backwards(self, orm):
+        # Deleting model 'Deployment'
+        db.delete_table(u'zilencer_deployment')
+
+        # Removing M2M table for field realms on 'Deployment'
+        db.delete_table('zilencer_deployment_realms')
+
+
+    models = {
+        u'zerver.realm': {
+            'Meta': {'object_name': 'Realm'},
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True'}),
+            'notifications_stream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': u"orm['zerver.Stream']"}),
+            'restricted_to_domain': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        u'zerver.stream': {
+            'Meta': {'unique_together': "(('name', 'realm'),)", 'object_name': 'Stream'},
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email_token': ('django.db.models.fields.CharField', [], {'default': "'1946c400b4b841499b527216b8bc3db6'", 'max_length': '32'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'invite_only': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '60', 'db_index': 'True'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"})
+        },
+        u'zilencer.deployment': {
+            'Meta': {'object_name': 'Deployment'},
+            'api_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'realms': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'_deployments'", 'symmetrical': 'False', 'to': u"orm['zerver.Realm']"})
+        }
+    }
+
+    complete_apps = ['zilencer']

zilencer/models.py

@@ -0,0 +1,13 @@
+from django.db import models
+import zerver.models
+
+def get_deployment_by_domain(domain):
+    return Deployment.objects.get(realms__domain=domain)
+
+class Deployment(models.Model):
+    realms = models.ManyToManyField(zerver.models.Realm, related_name="_deployments")
+    is_active = models.BooleanField(default=True)
+
+    # TODO: This should really become the public portion of a keypair, and
+    # it should be settable only with an initial bearer "activation key"
+    api_key = models.CharField(max_length=32, null=True)

zproject/local_settings.py

@@ -29,8 +29,9 @@ RABBITMQ_PASSWORD = 'xxxxxxxxxxxxxxxx'
 MAILCHIMP_API_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-us4'
 ZULIP_FRIENDS_LIST_ID = '84b2f3da6b'
 
-# This can be filled in automatically from the database
-FEEDBACK_BOT_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+# This can be filled in automatically from the database, maybe
+DEPLOYMENT_ROLE_NAME = 'zulip.com'
+DEPLOYMENT_ROLE_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
 
 # This comes from our mandrill accounts page
 MANDRILL_API_KEY = 'xxxxxxxxxxxxxxxxxxxxxx'

zproject/local_settings_template.py

@@ -29,8 +29,10 @@ RABBITMQ_PASSWORD = ''
 # TODO: Make USING_MAILCHIMP do something (and default to False)
 USING_MAILCHIMP = False
 
-# This can be filled in automatically from the database
-FEEDBACK_BOT_KEY = ''
+
+# These credentials are for communication with the central Zulip deployment manager
+DEPLOYMENT_ROLE_NAME = ''
+DEPLOYMENT_ROLE_KEY = ''
 
 # TODO: Make USING_MANDRILL do something (and default to False)
 USING_MANDRILL = False

zproject/settings.py

@@ -164,6 +164,7 @@ INSTALLED_APPS = (
     'guardian',
     'pipeline',
     'zerver',
+    'zilencer',
 )
 
 LOCAL_STATSD = (False)