presence: Add realm_id to UserPresence.

This index is intended to optimize the performance of the very
frequently run query of "what is the presence status of all users in a
realm?".

Main changes:
    - add realm_id to UserPresence
    - add index for realm_id
    - backfill realm_id for old rows
    - change all writes to UserPresence to include
      realm_id

The index is of this form:

    "zerver_userpresence_realm_id_5c4ef5a9" btree (realm_id)

We will create an index on (realm_id, timestamp) in a
future commit, but I think it's a bit faster if you do
the backfill before the index.

There's also a minor tweak to the populate_db script.

zerver/lib/actions.py

@@ -4063,11 +4063,18 @@ def do_update_user_presence(user_profile: UserProfile,
                             log_time: datetime.datetime,
                             status: int) -> None:
     client = consolidate_client(client)
+
+    defaults = dict(
+        timestamp=log_time,
+        status=status,
+        realm_id=user_profile.realm_id
+    )
+
     (presence, created) = UserPresence.objects.get_or_create(
         user_profile = user_profile,
         client = client,
-        defaults = {'timestamp': log_time,
-                    'status': status})
+        defaults = defaults
+    )
 
     stale_status = (log_time - presence.timestamp) > datetime.timedelta(minutes=1, seconds=10)
     was_idle = presence.status == UserPresence.IDLE

zerver/migrations/0266_userpresence_realm.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.28 on 2020-02-08 20:19
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0265_remove_stream_is_announcement_only'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userpresence',
+            name='realm',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'),
+        ),
+    ]

zerver/migrations/0267_backfill_userpresence_realm_id.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0266_userpresence_realm'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+            UPDATE zerver_userpresence
+            SET realm_id = zerver_userprofile.realm_id
+            FROM zerver_userprofile
+            WHERE zerver_userprofile.id = zerver_userpresence.user_profile_id;
+            """,
+            reverse_sql='UPDATE zerver_userpresence SET realm_id = NULL'),
+    ]

zerver/models.py

@@ -2329,6 +2329,7 @@ class UserPresence(models.Model):
         unique_together = ("user_profile", "client")
 
     user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)  # type: UserProfile
+    realm = models.ForeignKey(Realm, null=True)  # type: Optional[Realm]
     client = models.ForeignKey(Client, on_delete=CASCADE)  # type: Client
 
     # The time we heard this update from the client.

zerver/tests/test_message_edit_notifications.py

@@ -271,6 +271,7 @@ class EditMessageSideEffectsTest(ZulipTestCase):
         cordelia = self.example_user('cordelia')
         UserPresence.objects.create(
             user_profile_id=cordelia.id,
+            realm_id=cordelia.realm_id,
             status=UserPresence.ACTIVE,
             client=get_client('web'),
             timestamp=timezone_now(),

zerver/tests/test_messages.py

@@ -3842,10 +3842,11 @@ class MissedMessageTest(ZulipTestCase):
             )
             self.assertEqual(sorted(user_ids), sorted(presence_idle_user_ids))
 
-        def set_presence(user_id: int, client_name: str, ago: int) -> None:
+        def set_presence(user: UserProfile, client_name: str, ago: int) -> None:
             when = timezone_now() - datetime.timedelta(seconds=ago)
             UserPresence.objects.create(
-                user_profile_id=user_id,
+                user_profile_id=user.id,
+                realm_id=user.realm_id,
                 client=get_client(client_name),
                 timestamp=when,
             )
@@ -3857,10 +3858,10 @@ class MissedMessageTest(ZulipTestCase):
         user_flags[hamlet.id] = ['mentioned']
         assert_missing([hamlet.id])
 
-        set_presence(hamlet.id, 'iPhone', ago=5000)
+        set_presence(hamlet, 'iPhone', ago=5000)
         assert_missing([hamlet.id])
 
-        set_presence(hamlet.id, 'webapp', ago=15)
+        set_presence(hamlet, 'webapp', ago=15)
         assert_missing([])
 
         message_type = 'private'

zilencer/management/commands/populate_db.py

@@ -464,6 +464,7 @@ class Command(BaseCommand):
                 if user.full_name[0] <= 'H':
                     client = get_client("ZulipAndroid")
                 UserPresence.objects.get_or_create(user_profile=user,
+                                                   realm_id=user.realm_id,
                                                    client=client,
                                                    timestamp=date,
                                                    status=status)