diff --git a/.eslintrc.json b/.eslintrc.json index a1c24b5158..28732e3e39 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -87,7 +87,8 @@ "current_msg_list": true, "home_msg_list": false, "pm_list": false, - "unread_ui": false + "unread_ui": false, + "Plotly": false }, "rules": { "no-restricted-syntax": 1, diff --git a/analytics/tests/test_views.py b/analytics/tests/test_views.py new file mode 100644 index 0000000000..d2658059fa --- /dev/null +++ b/analytics/tests/test_views.py @@ -0,0 +1,37 @@ +from django.utils.timezone import get_fixed_timezone +from zerver.lib.test_classes import ZulipTestCase + +from analytics.lib.counts import CountStat +from analytics.views import time_range + +from datetime import datetime, timedelta + +class TestTimeRange(ZulipTestCase): + def test_time_range(self): + # type: () -> None + HOUR = timedelta(hours=1) + DAY = timedelta(days=1) + TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC + + # Using 22:59 so that converting to UTC and applying ceiling_to_{hour,day} do not commute + a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO) + # Round up to hour and day + ceiling_hour = datetime(2016, 3, 14, 23).replace(tzinfo=TZINFO) + ceiling_day = datetime(2016, 3, 15).replace(tzinfo=TZINFO) + + # test start == end + self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [ceiling_hour]) + self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [ceiling_day]) + # test start == end == boundary, and min_length == 0 + self.assertEqual(time_range(ceiling_hour, ceiling_hour, CountStat.HOUR, 0), [ceiling_hour]) + self.assertEqual(time_range(ceiling_day, ceiling_day, CountStat.DAY, 0), [ceiling_day]) + # test start and end on different boundaries + self.assertEqual(time_range(ceiling_hour, ceiling_hour+HOUR, CountStat.HOUR, None), + [ceiling_hour, ceiling_hour+HOUR]) + self.assertEqual(time_range(ceiling_day, ceiling_day+DAY, CountStat.DAY, None), + [ceiling_day, ceiling_day+DAY]) + # test min_length + self.assertEqual(time_range(ceiling_hour, ceiling_hour+HOUR, CountStat.HOUR, 4), + [ceiling_hour-2*HOUR, ceiling_hour-HOUR, ceiling_hour, ceiling_hour+HOUR]) + self.assertEqual(time_range(ceiling_day, ceiling_day+DAY, CountStat.DAY, 4), + [ceiling_day-2*DAY, ceiling_day-DAY, ceiling_day, ceiling_day+DAY]) diff --git a/analytics/urls.py b/analytics/urls.py index 2dfbed0b44..dc5c845115 100644 --- a/analytics/urls.py +++ b/analytics/urls.py @@ -1,4 +1,6 @@ -from django.conf.urls import url +from django.conf.urls import url, include +from zerver.lib.rest import rest_dispatch + import analytics.views i18n_urlpatterns = [ @@ -8,6 +10,29 @@ i18n_urlpatterns = [ name='analytics.views.get_realm_activity'), url(r'^user_activity/(?P[\S]+)/$', analytics.views.get_user_activity, name='analytics.views.get_user_activity'), + + # User-visible stats page + url(r'^stats$', analytics.views.stats, + name='analytics.views.stats'), +] + +# These endpoints are a part of the API (V1), which uses: +# * REST verbs +# * Basic auth (username:password is email:apiKey) +# * Takes and returns json-formatted data +# +# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used +# +# All of these paths are accessed by either a /json or /api prefix +v1_api_and_json_patterns = [ + # get data for the graphs at /stats + url(r'^analytics/chart_data$', rest_dispatch, + {'GET': 'analytics.views.get_chart_data'}), +] + +i18n_urlpatterns += [ + url(r'^api/v1/', include(v1_api_and_json_patterns)), + url(r'^json/', include(v1_api_and_json_patterns)), ] urlpatterns = i18n_urlpatterns diff --git a/analytics/views.py b/analytics/views.py index 770da620f6..b0827aa92e 100644 --- a/analytics/views.py +++ b/analytics/views.py @@ -5,16 +5,25 @@ from django.db import connection from django.db.models.query import QuerySet from django.http import HttpResponseNotFound, HttpRequest, HttpResponse from django.template import RequestContext, loader +from django.utils import timezone +from django.utils.translation import ugettext as _ from jinja2 import Markup as mark_safe -from zerver.decorator import has_request_variables, REQ, zulip_internal -from zerver.lib.timestamp import timestamp_to_datetime -from zerver.models import UserActivity, UserActivityInterval, Realm +from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS +from analytics.models import RealmCount, UserCount + +from zerver.decorator import has_request_variables, REQ, zulip_internal, \ + zulip_login_required, to_non_negative_int, to_utc_datetime +from zerver.lib.request import JsonableError +from zerver.lib.response import json_success +from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, timestamp_to_datetime +from zerver.models import Realm, UserProfile, UserActivity, UserActivityInterval from zproject.jinja2 import render_to_response from collections import defaultdict from datetime import datetime, timedelta import itertools +import json import pytz import re import time @@ -22,6 +31,75 @@ import time from six.moves import filter, map, range, zip from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union, Text +@zulip_login_required +def stats(request): + # type: (HttpRequest) -> HttpResponse + return render_to_response('analytics/stats.html') + +@has_request_variables +def get_chart_data(request, user_profile, chart_name=REQ(), + min_length=REQ(converter=to_non_negative_int, default=None), + start=REQ(converter=to_utc_datetime, default=None), + end=REQ(converter=to_utc_datetime, default=None)): + # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse + realm = user_profile.realm + if chart_name == 'messages_sent_to_realm': + data = get_messages_sent_to_realm(realm, min_length=min_length, start=start, end=end) + else: + raise JsonableError(_("Unknown chart name: %s") % (chart_name,)) + return json_success(data=data) + +def get_messages_sent_to_realm(realm, min_length=None, start=None, end=None): + # type: (Realm, Optional[int], Optional[datetime], Optional[datetime]) -> Dict[str, Any] + # These are implicitly relying on realm.date_created and timezone.now being in UTC. + if start is None: + start = realm.date_created + if end is None: + end = timezone.now() + if start > end: + raise JsonableError(_("Start time is later than end time. Start: %s, End: %s") % (start, end)) + interval = CountStat.DAY + end_times = time_range(start, end, interval, min_length) + indices = {} + for i, end_time in enumerate(end_times): + indices[end_time] = i + + filter_set = RealmCount.objects.filter( + realm=realm, property='messages_sent:is_bot', interval=interval) \ + .values_list('end_time', 'value') + humans = [0]*len(end_times) + for end_time, value in filter_set.filter(subgroup=False): + humans[indices[end_time]] = value + bots = [0]*len(end_times) + for end_time, value in filter_set.filter(subgroup=True): + bots[indices[end_time]] = value + + return {'end_times': end_times, 'humans': humans, 'bots': bots, 'interval': interval} + +# If min_length is None, returns end_times from ceiling(start) to ceiling(end), inclusive. +# If min_length is greater than 0, pads the list to the left. +# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22], +# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22] +def time_range(start, end, interval, min_length): + # type: (datetime, datetime, str, Optional[int]) -> List[datetime] + if interval == CountStat.HOUR: + end = ceiling_to_hour(end) + step = timedelta(hours=1) + elif interval == CountStat.DAY: + end = ceiling_to_day(end) + step = timedelta(days=1) + else: + raise ValueError(_("Unknown interval.")) + + times = [] + if min_length is not None: + start = min(start, end - (min_length-1)*step) + current = end + while current >= start: + times.append(current) + current -= step + return list(reversed(times)) + eastern_tz = pytz.timezone('US/Eastern') def make_table(title, cols, rows, has_row_class=False): diff --git a/static/js/stats.js b/static/js/stats.js new file mode 100644 index 0000000000..ae401bb00a --- /dev/null +++ b/static/js/stats.js @@ -0,0 +1,46 @@ +function populate_messages_sent_to_realm(data) { + var trace_humans = { + x: data.end_times.map(function (timestamp) { + return new Date(timestamp*1000); + }), + y: data.humans, + mode: 'lines', + name: 'Messages from humans', + hoverinfo: 'y' + }; + + var trace_bots = { + x: data.end_times.map(function (timestamp) { + return new Date(timestamp*1000); + }), + y: data.bots, + mode: 'lines', + name: 'Messages from bots', + hoverinfo: 'y' + }; + + var layout = { + title: 'Messages sent by humans and bots', + xaxis: { + type: 'date', + }, + yaxis: { + fixedrange: true, + rangemode: 'tozero', + } + }; + + Plotly.newPlot('id_messages_sent_to_realm', [trace_humans, trace_bots], layout, {displayModeBar: false}); +} + +$.get({ + url: '/json/analytics/chart_data', + data: {chart_name: 'messages_sent_to_realm', min_length: '10'}, + idempotent: true, + success: function (data) { + populate_messages_sent_to_realm(data); + }, + error: function (xhr) { + $('#id_stats_errors').text($.parseJSON(xhr.responseText).msg); + } +}); diff --git a/templates/analytics/stats.html b/templates/analytics/stats.html new file mode 100644 index 0000000000..63a18bbc8b --- /dev/null +++ b/templates/analytics/stats.html @@ -0,0 +1,11 @@ + + + {{ minified_js('stats')|safe }} + + +
+
+ +
+ + diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index 61d6056675..ee4995fc6c 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -62,6 +62,7 @@ class TemplateTestCase(ZulipTestCase): ] logged_in = [ + 'analytics/stats.html', 'zerver/home.html', 'zerver/invite_user.html', 'zerver/keyboard_shortcuts.html', diff --git a/zproject/settings.py b/zproject/settings.py index 3b2dc1e8e7..d7079b499a 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -848,6 +848,14 @@ JS_SPECS = { ), 'output_filename': 'min/activity.js' }, + 'stats': { + 'source_filenames': ( + 'node_modules/plotly.js/dist/plotly.js', + 'node_modules/jquery/dist/jquery.js', + 'js/stats.js' + ), + 'output_filename': 'min/stats.js' + }, # We also want to minify sockjs separately for the sockjs iframe transport 'sockjs': { 'source_filenames': ('node_modules/sockjs-client/sockjs.js',),