Add /stats page with basic stats graph.

Adds a new url route and a new json endpoint.

.eslintrc.json

@@ -87,7 +87,8 @@
         "current_msg_list": true,
         "home_msg_list": false,
         "pm_list": false,
-        "unread_ui": false
+        "unread_ui": false,
+        "Plotly": false
     },
     "rules": {
         "no-restricted-syntax": 1,

analytics/tests/test_views.py

@@ -0,0 +1,37 @@
+from django.utils.timezone import get_fixed_timezone
+from zerver.lib.test_classes import ZulipTestCase
+
+from analytics.lib.counts import CountStat
+from analytics.views import time_range
+
+from datetime import datetime, timedelta
+
+class TestTimeRange(ZulipTestCase):
+    def test_time_range(self):
+        # type: () -> None
+        HOUR = timedelta(hours=1)
+        DAY = timedelta(days=1)
+        TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC
+
+        # Using 22:59 so that converting to UTC and applying ceiling_to_{hour,day} do not commute
+        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)
+        # Round up to hour and day
+        ceiling_hour = datetime(2016, 3, 14, 23).replace(tzinfo=TZINFO)
+        ceiling_day = datetime(2016, 3, 15).replace(tzinfo=TZINFO)
+
+        # test start == end
+        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [ceiling_hour])
+        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [ceiling_day])
+        # test start == end == boundary, and min_length == 0
+        self.assertEqual(time_range(ceiling_hour, ceiling_hour, CountStat.HOUR, 0), [ceiling_hour])
+        self.assertEqual(time_range(ceiling_day, ceiling_day, CountStat.DAY, 0), [ceiling_day])
+        # test start and end on different boundaries
+        self.assertEqual(time_range(ceiling_hour, ceiling_hour+HOUR, CountStat.HOUR, None),
+                         [ceiling_hour, ceiling_hour+HOUR])
+        self.assertEqual(time_range(ceiling_day, ceiling_day+DAY, CountStat.DAY, None),
+                         [ceiling_day, ceiling_day+DAY])
+        # test min_length
+        self.assertEqual(time_range(ceiling_hour, ceiling_hour+HOUR, CountStat.HOUR, 4),
+                         [ceiling_hour-2*HOUR, ceiling_hour-HOUR, ceiling_hour, ceiling_hour+HOUR])
+        self.assertEqual(time_range(ceiling_day, ceiling_day+DAY, CountStat.DAY, 4),
+                         [ceiling_day-2*DAY, ceiling_day-DAY, ceiling_day, ceiling_day+DAY])

analytics/urls.py

@@ -1,4 +1,6 @@
-from django.conf.urls import url
+from django.conf.urls import url, include
+from zerver.lib.rest import rest_dispatch
+
 import analytics.views
 
 i18n_urlpatterns = [
@@ -8,6 +10,29 @@ i18n_urlpatterns = [
         name='analytics.views.get_realm_activity'),
     url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
         name='analytics.views.get_user_activity'),
+
+    # User-visible stats page
+    url(r'^stats$', analytics.views.stats,
+        name='analytics.views.stats'),
+]
+
+# These endpoints are a part of the API (V1), which uses:
+# * REST verbs
+# * Basic auth (username:password is email:apiKey)
+# * Takes and returns json-formatted data
+#
+# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
+#
+# All of these paths are accessed by either a /json or /api prefix
+v1_api_and_json_patterns = [
+    # get data for the graphs at /stats
+    url(r'^analytics/chart_data$', rest_dispatch,
+        {'GET': 'analytics.views.get_chart_data'}),
+]
+
+i18n_urlpatterns += [
+    url(r'^api/v1/', include(v1_api_and_json_patterns)),
+    url(r'^json/', include(v1_api_and_json_patterns)),
 ]
 
 urlpatterns = i18n_urlpatterns

analytics/views.py

@@ -5,16 +5,25 @@ from django.db import connection
 from django.db.models.query import QuerySet
 from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from django.template import RequestContext, loader
+from django.utils import timezone
+from django.utils.translation import ugettext as _
 from jinja2 import Markup as mark_safe
 
-from zerver.decorator import has_request_variables, REQ, zulip_internal
-from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import UserActivity, UserActivityInterval, Realm
+from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS
+from analytics.models import RealmCount, UserCount
+
+from zerver.decorator import has_request_variables, REQ, zulip_internal, \
+    zulip_login_required, to_non_negative_int, to_utc_datetime
+from zerver.lib.request import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, timestamp_to_datetime
+from zerver.models import Realm, UserProfile, UserActivity, UserActivityInterval
 from zproject.jinja2 import render_to_response
 
 from collections import defaultdict
 from datetime import datetime, timedelta
 import itertools
+import json
 import pytz
 import re
 import time
@@ -22,6 +31,75 @@ import time
 from six.moves import filter, map, range, zip
 from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union, Text
 
+@zulip_login_required
+def stats(request):
+    # type: (HttpRequest) -> HttpResponse
+    return render_to_response('analytics/stats.html')
+
+@has_request_variables
+def get_chart_data(request, user_profile, chart_name=REQ(),
+                   min_length=REQ(converter=to_non_negative_int, default=None),
+                   start=REQ(converter=to_utc_datetime, default=None),
+                   end=REQ(converter=to_utc_datetime, default=None)):
+    # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse
+    realm = user_profile.realm
+    if chart_name == 'messages_sent_to_realm':
+        data = get_messages_sent_to_realm(realm, min_length=min_length, start=start, end=end)
+    else:
+        raise JsonableError(_("Unknown chart name: %s") % (chart_name,))
+    return json_success(data=data)
+
+def get_messages_sent_to_realm(realm, min_length=None, start=None, end=None):
+    # type: (Realm, Optional[int], Optional[datetime], Optional[datetime]) -> Dict[str, Any]
+    # These are implicitly relying on realm.date_created and timezone.now being in UTC.
+    if start is None:
+        start = realm.date_created
+    if end is None:
+        end = timezone.now()
+    if start > end:
+        raise JsonableError(_("Start time is later than end time. Start: %s, End: %s") % (start, end))
+    interval = CountStat.DAY
+    end_times = time_range(start, end, interval, min_length)
+    indices = {}
+    for i, end_time in enumerate(end_times):
+        indices[end_time] = i
+
+    filter_set = RealmCount.objects.filter(
+        realm=realm, property='messages_sent:is_bot', interval=interval) \
+        .values_list('end_time', 'value')
+    humans = [0]*len(end_times)
+    for end_time, value in filter_set.filter(subgroup=False):
+        humans[indices[end_time]] = value
+    bots = [0]*len(end_times)
+    for end_time, value in filter_set.filter(subgroup=True):
+        bots[indices[end_time]] = value
+
+    return {'end_times': end_times, 'humans': humans, 'bots': bots, 'interval': interval}
+
+# If min_length is None, returns end_times from ceiling(start) to ceiling(end), inclusive.
+# If min_length is greater than 0, pads the list to the left.
+# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
+# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
+def time_range(start, end, interval, min_length):
+    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
+    if interval == CountStat.HOUR:
+        end = ceiling_to_hour(end)
+        step = timedelta(hours=1)
+    elif interval == CountStat.DAY:
+        end = ceiling_to_day(end)
+        step = timedelta(days=1)
+    else:
+        raise ValueError(_("Unknown interval."))
+
+    times = []
+    if min_length is not None:
+        start = min(start, end - (min_length-1)*step)
+    current = end
+    while current >= start:
+        times.append(current)
+        current -= step
+    return list(reversed(times))
+
 eastern_tz = pytz.timezone('US/Eastern')
 
 def make_table(title, cols, rows, has_row_class=False):

static/js/stats.js

@@ -0,0 +1,46 @@
+function populate_messages_sent_to_realm(data) {
+    var trace_humans = {
+        x: data.end_times.map(function (timestamp) {
+            return new Date(timestamp*1000);
+        }),
+        y: data.humans,
+        mode: 'lines',
+        name: 'Messages from humans',
+        hoverinfo: 'y'
+    };
+
+    var trace_bots = {
+        x: data.end_times.map(function (timestamp) {
+            return new Date(timestamp*1000);
+        }),
+        y: data.bots,
+        mode: 'lines',
+        name: 'Messages from bots',
+        hoverinfo: 'y'
+    };
+
+    var layout = {
+        title: 'Messages sent by humans and bots',
+        xaxis: {
+            type: 'date',
+        },
+        yaxis: {
+            fixedrange: true,
+            rangemode: 'tozero',
+        }
+    };
+
+    Plotly.newPlot('id_messages_sent_to_realm', [trace_humans, trace_bots], layout, {displayModeBar: false});
+}
+
+$.get({
+    url: '/json/analytics/chart_data',
+    data: {chart_name: 'messages_sent_to_realm', min_length: '10'},
+    idempotent: true,
+    success: function (data) {
+        populate_messages_sent_to_realm(data);
+    },
+    error: function (xhr) {
+        $('#id_stats_errors').text($.parseJSON(xhr.responseText).msg);
+    }
+});

templates/analytics/stats.html

@@ -0,0 +1,11 @@
+<html lang="en">
+<head>
+    {{ minified_js('stats')|safe }}
+</head>
+<body>
+  <div id="id_stats_errors" class="alert alert-error"></div>
+  <div id="id_messages_sent_to_realm" style="width: 800px; height: 400px;">
+    <!-- Plotly chart will be drawn inside this DIV -->
+  </div>
+</body>
+</html>

zerver/tests/test_templates.py

@@ -62,6 +62,7 @@ class TemplateTestCase(ZulipTestCase):
         ]
 
         logged_in = [
+            'analytics/stats.html',
             'zerver/home.html',
             'zerver/invite_user.html',
             'zerver/keyboard_shortcuts.html',

zproject/settings.py

@@ -848,6 +848,14 @@ JS_SPECS = {
         ),
         'output_filename': 'min/activity.js'
     },
+    'stats': {
+        'source_filenames': (
+            'node_modules/plotly.js/dist/plotly.js',
+            'node_modules/jquery/dist/jquery.js',
+            'js/stats.js'
+        ),
+        'output_filename': 'min/stats.js'
+    },
     # We also want to minify sockjs separately for the sockjs iframe transport
     'sockjs': {
         'source_filenames': ('node_modules/sockjs-client/sockjs.js',),