[schema] Collect data on when users are active on site.

These engagement data will be useful both for making pretty graphs of
how addicted our users are as well as for allowing us to check whether
a new deployment is actually using the product or not.

This measures "number of minutes during which each user had checked
the app within the previous 15 minutes".  It should correctly not
count server-initiated reloads.

It's possible that we should use something less aggressive than
mousemove; I'm a little torn on that because you really can check the
app for new messages without doing anything active.

This is somewhat tested but there are a few outstanding issues:

* Mobile apps don't report these data.  It should be as easy as having
  them send in update_active_status queries with new_user_input=true.

* The semantics of this should be better documented (e.g. the
  management script should print out the spec above)x.

(imported from commit ec8b2dc96b180e1951df00490707ae916887178e)
This commit is contained in:
Tim Abbott 2013-09-06 15:52:12 -04:00 committed by Leo Franchi
parent 5bcc403ea4
commit 9fcca3df4e
9 changed files with 315 additions and 8 deletions

View File

@ -19,6 +19,16 @@ exports.IDLE = "idle";
exports.has_focus = true;
// We initialize this to true, to count new page loads, but set it to
// false in the onload function in reload.js if this was a
// server-initiated-reload to avoid counting a server-initiated reload
// as user activity.
exports.new_user_input = true;
$("html").on("mousemove", function () {
exports.new_user_input = true;
});
var user_info = {};
function sort_users(users, user_info) {
@ -83,7 +93,8 @@ function status_from_timestamp(baseline_time, presence) {
function focus_ping() {
$.post('/json/update_active_status',
{status: (exports.has_focus) ? exports.ACTIVE : exports.IDLE}, function (data) {
{status: (exports.has_focus) ? exports.ACTIVE : exports.IDLE,
new_user_input: exports.new_user_input}, function (data) {
if (data === undefined || data.presences === undefined) {
// We sometimes receive no data even on successful
// requests; we should figure out why but this will
@ -100,6 +111,8 @@ function focus_ping() {
$('#zephyr-mirror-error').hide();
}
exports.new_user_input = false;
// Ping returns the active peer list
_.each(data.presences, function (presence, this_email) {
if (page_params.email !== this_email) {

View File

@ -102,6 +102,7 @@ function process_hotkey(e) {
var row, focused_message_edit_content, focused_message_edit_save, message_edit_form;
var event_name = get_event_name(e);
activity.new_user_input = true;
if (event_name === "tab") {
// The alert word configuration is on the settings page,

View File

@ -89,6 +89,7 @@ $(function () {
page_params.initial_pointer = pointer;
}
activity.new_user_input = false;
hashchange.changehash(vars.oldhash);
});

View File

@ -12,7 +12,7 @@ from zerver.models import Realm, RealmEmoji, Stream, UserProfile, UserActivity,
to_dict_cache_key, get_realm, stringify_message_dict, bulk_get_recipients, \
email_to_domain, email_to_username, display_recipient_cache_key, \
get_stream_cache_key, to_dict_cache_key_id, is_super_user, \
get_active_user_profiles_by_realm
get_active_user_profiles_by_realm, UserActivityInterval
from django.db import transaction, IntegrityError
from django.db.models import F, Q
from django.core.exceptions import ValidationError
@ -1077,7 +1077,14 @@ def do_update_user_presence(user_profile, client, log_time, status):
# user without delay
send_presence_changed(user_profile, presence)
def update_user_presence(user_profile, client, log_time, status):
def update_user_activity_interval(user_profile, log_time):
event={'type': 'user_activity_interval',
'user_profile_id': user_profile.id,
'time': datetime_to_timestamp(log_time)}
queue_json_publish("user_activity", event, process_user_activity_interval_event)
def update_user_presence(user_profile, client, log_time, status,
new_user_input):
event={'type': 'user_presence',
'user_profile_id': user_profile.id,
'status': status,
@ -1086,6 +1093,9 @@ def update_user_presence(user_profile, client, log_time, status):
queue_json_publish("user_activity", event, process_user_presence_event)
if ujson.loads(new_user_input):
update_user_activity_interval(user_profile, log_time)
def do_update_message_flags(user_profile, operation, flag, messages, all):
flagattr = getattr(UserMessage.flags, flag)
@ -1119,6 +1129,23 @@ def process_user_presence_event(event):
status = event["status"]
return do_update_user_presence(user_profile, client, log_time, status)
def process_user_activity_interval_event(event):
user_profile = get_user_profile_by_id(event["user_profile_id"])
log_time = timestamp_to_datetime(event["time"])
effective_end = log_time + datetime.timedelta(minutes=15)
try:
last = UserActivityInterval.objects.filter(user_profile=user_profile).order_by("-end")[0]
if log_time < last.end:
last.end = effective_end
last.save(update_fields=["end"])
return
except IndexError:
pass
UserActivityInterval.objects.create(user_profile=user_profile, start=log_time,
end=effective_end)
def subscribed_to_stream(user_profile, stream):
try:
if Subscription.objects.get(user_profile=user_profile,

View File

@ -0,0 +1,46 @@
from __future__ import absolute_import
from optparse import make_option
from django.core.management.base import BaseCommand
from zerver.models import get_user_profile_by_email, UserActivityInterval, \
UserProfile
import sys
import datetime
from django.utils.timezone import utc, is_naive
def analyze_activity(options):
day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
day_end = day_start + datetime.timedelta(days=1)
user_profile_query = UserProfile.objects.all()
if options["realm"]:
user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
total_duration = datetime.timedelta(0)
for user_profile in user_profile_query:
intervals = UserActivityInterval.objects.filter(user_profile=user_profile,
end__gte=day_start, start__lte=day_end)
if len(intervals) == 0:
continue
duration = datetime.timedelta(0)
for interval in intervals:
start = max(day_start, interval.start)
end = min(day_end, interval.end)
duration += end - start
total_duration += duration
print user_profile.email, duration
print "Total Duration: %s" % (total_duration,)
print "Total Duration in minutes: %s" % (total_duration.total_seconds() / 60.,)
print "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--realm', action='store'),
make_option('--date', action='store', default="2013-09-06"),
)
def handle(self, *args, **options):
analyze_activity(options)

View File

@ -3,7 +3,7 @@ from __future__ import absolute_import
from django.conf import settings
from django.core.management.base import BaseCommand
from zerver.lib.actions import process_user_activity_event, \
process_user_presence_event
process_user_presence_event, process_user_activity_interval_event
from zerver.lib.queue import SimpleQueueClient
import sys
import signal
@ -40,6 +40,8 @@ class Command(BaseCommand):
process_user_activity_event(event)
elif msg_type == 'user_presence':
process_user_presence_event(event)
elif msg_type == 'user_activity_interval':
process_user_activity_interval_event(event)
else:
print("[*] Unknown message type: %s" % (msg_type,))

View File

@ -0,0 +1,210 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'UserActivityInterval'
db.create_table(u'zerver_useractivityinterval', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user_profile', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['zerver.UserProfile'])),
('start', self.gf('django.db.models.fields.DateTimeField')(db_index=True)),
('end', self.gf('django.db.models.fields.DateTimeField')(db_index=True)),
))
db.send_create_signal(u'zerver', ['UserActivityInterval'])
def backwards(self, orm):
# Deleting model 'UserActivityInterval'
db.delete_table(u'zerver_useractivityinterval')
models = {
u'auth.group': {
'Meta': {'object_name': 'Group'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
u'auth.permission': {
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
u'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
u'zerver.client': {
'Meta': {'object_name': 'Client'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30', 'db_index': 'True'})
},
u'zerver.defaultstream': {
'Meta': {'unique_together': "(('realm', 'stream'),)", 'object_name': 'DefaultStream'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"}),
'stream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Stream']"})
},
u'zerver.huddle': {
'Meta': {'object_name': 'Huddle'},
'huddle_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
u'zerver.message': {
'Meta': {'object_name': 'Message'},
'content': ('django.db.models.fields.TextField', [], {}),
'edit_history': ('django.db.models.fields.TextField', [], {'null': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_edit_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
'pub_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Recipient']"}),
'rendered_content': ('django.db.models.fields.TextField', [], {'null': 'True'}),
'rendered_content_version': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
'sender': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"}),
'sending_client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '60', 'db_index': 'True'})
},
u'zerver.mituser': {
'Meta': {'object_name': 'MitUser'},
'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'status': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
u'zerver.preregistrationuser': {
'Meta': {'object_name': 'PreregistrationUser'},
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'invited_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']", 'null': 'True'}),
'referred_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']", 'null': 'True'}),
'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'streams': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['zerver.Stream']", 'null': 'True', 'symmetrical': 'False'})
},
u'zerver.realm': {
'Meta': {'object_name': 'Realm'},
'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'restricted_to_domain': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
},
u'zerver.realmemoji': {
'Meta': {'unique_together': "(('realm', 'name'),)", 'object_name': 'RealmEmoji'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'img_url': ('django.db.models.fields.TextField', [], {}),
'name': ('django.db.models.fields.TextField', [], {}),
'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"})
},
u'zerver.recipient': {
'Meta': {'unique_together': "(('type', 'type_id'),)", 'object_name': 'Recipient'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'type': ('django.db.models.fields.PositiveSmallIntegerField', [], {'db_index': 'True'}),
'type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'})
},
u'zerver.referral': {
'Meta': {'object_name': 'Referral'},
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'timestamp': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.stream': {
'Meta': {'unique_together': "(('name', 'realm'),)", 'object_name': 'Stream'},
'email_token': ('django.db.models.fields.CharField', [], {'default': "'04fa9e47ff96184d1830390a6b2013ff'", 'max_length': '32'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'invite_only': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}),
'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"})
},
u'zerver.streamcolor': {
'Meta': {'object_name': 'StreamColor'},
'color': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'subscription': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Subscription']"})
},
u'zerver.subscription': {
'Meta': {'unique_together': "(('user_profile', 'recipient'),)", 'object_name': 'Subscription'},
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'color': ('django.db.models.fields.CharField', [], {'default': "'#c2c2c2'", 'max_length': '10'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'in_home_view': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}),
'notifications': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Recipient']"}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.useractivity': {
'Meta': {'unique_together': "(('user_profile', 'client', 'query'),)", 'object_name': 'UserActivity'},
'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
'count': ('django.db.models.fields.IntegerField', [], {}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_visit': ('django.db.models.fields.DateTimeField', [], {}),
'query': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.useractivityinterval': {
'Meta': {'object_name': 'UserActivityInterval'},
'end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.usermessage': {
'Meta': {'unique_together': "(('user_profile', 'message'),)", 'object_name': 'UserMessage'},
'archived': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Message']"}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.userpresence': {
'Meta': {'unique_together': "(('user_profile', 'client'),)", 'object_name': 'UserPresence'},
'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '1'}),
'timestamp': ('django.db.models.fields.DateTimeField', [], {}),
'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
},
u'zerver.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'alert_words': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
'api_key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'avatar_source': ('django.db.models.fields.CharField', [], {'default': "'G'", 'max_length': '1'}),
'bot_owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']", 'null': 'True', 'on_delete': 'models.SET_NULL'}),
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'db_index': 'True'}),
'enable_desktop_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enable_offline_email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enable_sounds': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'enter_sends': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
'full_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'invites_granted': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'invites_used': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_bot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_pointer_updater': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
'last_reminder': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}),
'onboarding_steps': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'pointer': ('django.db.models.fields.IntegerField', [], {}),
'rate_limits': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '100'}),
'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"}),
'short_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'tutorial_status': ('django.db.models.fields.CharField', [], {'default': "'W'", 'max_length': '1'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
}
}
complete_apps = ['zerver']

View File

@ -681,6 +681,11 @@ class UserActivity(models.Model):
class Meta:
unique_together = ("user_profile", "client", "query")
class UserActivityInterval(models.Model):
user_profile = models.ForeignKey(UserProfile)
start = models.DateTimeField('start time', db_index=True)
end = models.DateTimeField('end time', db_index=True)
class UserPresence(models.Model):
user_profile = models.ForeignKey(UserProfile)
client = models.ForeignKey(Client)

View File

@ -36,7 +36,8 @@ from zerver.lib.actions import do_remove_subscription, bulk_remove_subscriptions
do_update_onboarding_steps, do_update_message, internal_prep_message, \
do_send_messages, do_add_subscription, get_default_subs, do_deactivate, \
user_email_is_unique, do_invite_users, do_refer_friend, compute_mit_user_fullname, \
do_add_alert_words, do_remove_alert_words, do_set_alert_words, get_subscribers
do_add_alert_words, do_remove_alert_words, do_set_alert_words, get_subscribers, \
update_user_activity_interval
from zerver.lib.create_user import random_api_key
from zerver.forms import RegistrationForm, HomepageForm, ToSForm, CreateBotForm, \
is_inactive, isnt_mit, not_mit_mailing_list
@ -671,7 +672,6 @@ def home(request):
logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer))
latest_read = None
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = simplejson.encoder.JSONEncoderForHTML().encode(dict(
@ -1814,12 +1814,14 @@ def get_status_list(requesting_user_profile):
@authenticated_json_post_view
@has_request_variables
def json_update_active_status(request, user_profile, status=REQ):
def json_update_active_status(request, user_profile, status=REQ,
new_user_input=REQ(default=False)):
status_val = UserPresence.status_from_string(status)
if status_val is None:
raise JsonableError("Invalid presence status: %s" % (status,))
else:
update_user_presence(user_profile, request.client, now(), status_val)
update_user_presence(user_profile, request.client, now(), status_val,
new_user_input)
ret = get_status_list(user_profile)
if user_profile.realm.domain == "mit.edu":