Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remote-activity: Get user counts for all servers and realms. 

Instead of querying the database for every remote server and realm
in the remote activity chart, we now get the server and realm data
for the installation in two queries.
This commit is contained in:
Lauryn Menard 2024-01-19 17:06:10 +01:00 committed by Tim Abbott
parent c08e266956
commit 76b26612a0
3 changed files with 163 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
analytics/tests/test_activity_views.py
View File

@ -4,13 +4,16 @@ from unittest import mock
from django.utils.timezone import now as timezone_now
from corporate.lib.analytics import get_remote_server_audit_logs
from corporate.lib.stripe import add_months
from corporate.models import Customer, CustomerPlan, LicenseLedger
from zerver.lib.test_classes import ZulipTestCase
from zerver.models import Client, UserActivity, UserProfile
from zilencer.models import (
RemoteRealm,
RemoteRealmAuditLog,
RemoteZulipServer,
get_remote_customer_user_count,
get_remote_server_guest_and_non_guest_count,
)
@ -74,6 +77,40 @@ data_list = [
"event_time": event_time,
"extra_data": {},
},
{
"server_id": 1,
"realm_id": 3,
"event_type": RemoteRealmAuditLog.USER_CREATED,
"event_time": event_time,
"extra_data": {
RemoteRealmAuditLog.ROLE_COUNT: {
RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
UserProfile.ROLE_REALM_ADMINISTRATOR: 1,
UserProfile.ROLE_REALM_OWNER: 1,
UserProfile.ROLE_MODERATOR: 1,
UserProfile.ROLE_MEMBER: 1,
UserProfile.ROLE_GUEST: 1,
}
}
},
},
{
"server_id": 1,
"realm_id": 3,
"event_type": RemoteRealmAuditLog.USER_DEACTIVATED,
"event_time": event_time + timedelta(seconds=1),
"extra_data": {
RemoteRealmAuditLog.ROLE_COUNT: {
RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
UserProfile.ROLE_REALM_ADMINISTRATOR: 1,
UserProfile.ROLE_REALM_OWNER: 1,
UserProfile.ROLE_MODERATOR: 1,
UserProfile.ROLE_MEMBER: 0,
UserProfile.ROLE_GUEST: 1,
}
}
},
},
]
@ -107,9 +144,8 @@ class ActivityTest(ZulipTestCase):
self.assertEqual(result.status_code, 200)
# Add data for remote activity page
RemoteRealmAuditLog.objects.bulk_create([RemoteRealmAuditLog(**data) for data in data_list])
remote_server = RemoteZulipServer.objects.get(id=1)
customer = Customer.objects.create(remote_server=remote_server)
remote_realm = RemoteRealm.objects.get(name="Lear & Co.")
customer = Customer.objects.create(remote_realm=remote_realm)
plan = CustomerPlan.objects.create(
customer=customer,
billing_cycle_anchor=timezone_now(),
@ -125,13 +161,31 @@ class ActivityTest(ZulipTestCase):
is_renewal=True,
plan=plan,
)
RemoteZulipServer.objects.create(
server = RemoteZulipServer.objects.create(
uuid=str(uuid.uuid4()),
api_key="magic_secret_api_key",
hostname="demo.example.com",
contact_email="email@example.com",
)
with self.assert_database_query_count(15):
extra_data = {
RemoteRealmAuditLog.ROLE_COUNT: {
RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
UserProfile.ROLE_REALM_ADMINISTRATOR: 1,
UserProfile.ROLE_REALM_OWNER: 1,
UserProfile.ROLE_MODERATOR: 1,
UserProfile.ROLE_MEMBER: 1,
UserProfile.ROLE_GUEST: 1,
}
}
}
RemoteRealmAuditLog.objects.create(
server=server,
realm_id=10,
event_type=RemoteRealmAuditLog.USER_CREATED,
event_time=timezone_now() - timedelta(days=1),
extra_data=extra_data,
)
with self.assert_database_query_count(11):
result = self.client_get("/activity/remote")
self.assertEqual(result.status_code, 200)
@ -150,9 +204,17 @@ class ActivityTest(ZulipTestCase):
def test_get_remote_server_guest_and_non_guest_count(self) -> None:
RemoteRealmAuditLog.objects.bulk_create([RemoteRealmAuditLog(**data) for data in data_list])
server_id = 1
# Used in billing code
remote_server_counts = get_remote_server_guest_and_non_guest_count(
server_id=1, event_time=timezone_now()
server_id=server_id, event_time=timezone_now()
)
self.assertEqual(remote_server_counts.non_guest_user_count, 70)
self.assertEqual(remote_server_counts.guest_user_count, 15)
self.assertEqual(remote_server_counts.non_guest_user_count, 73)
self.assertEqual(remote_server_counts.guest_user_count, 16)
# Used in remote activity view code
server_logs = get_remote_server_audit_logs()
remote_activity_counts = get_remote_customer_user_count(server_logs[server_id])
self.assertEqual(remote_activity_counts.non_guest_user_count, 73)
self.assertEqual(remote_activity_counts.guest_user_count, 16)

58
analytics/views/remote_activity.py
View File

@ -11,15 +11,16 @@ from analytics.views.activity_common import (
remote_installation_stats_link,
remote_installation_support_link,
)
from corporate.lib.analytics import get_plan_data_by_remote_realm, get_plan_data_by_remote_server
from corporate.lib.analytics import (
get_plan_data_by_remote_realm,
get_plan_data_by_remote_server,
get_remote_realm_user_counts,
get_remote_server_audit_logs,
)
from corporate.lib.stripe import cents_to_dollar_string
from zerver.decorator import require_server_admin
from zerver.models.realms import get_org_type_display_name
from zilencer.models import (
RemoteRealm,
get_remote_realm_guest_and_non_guest_count,
get_remote_server_guest_and_non_guest_count,
)
from zilencer.models import get_remote_customer_user_count
@require_server_admin
@ -78,16 +79,16 @@ def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
)
cols = [
"ID",
"Hostname",
"Contact email",
"Zulip version",
"Last audit log update",
"Server ID",
"Server hostname",
"Server contact email",
"Server Zulip version",
"Server last audit log update",
"Server mobile users",
"Server mobile pushes",
"Realm ID",
"Realm name",
"Organization Type",
"Realm organization type",
"Plan name",
"Plan status",
"ARR",
@ -111,32 +112,37 @@ def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
rows = get_query_data(query)
plan_data_by_remote_server = get_plan_data_by_remote_server()
plan_data_by_remote_server_and_realm = get_plan_data_by_remote_realm()
audit_logs_by_remote_server = get_remote_server_audit_logs()
remote_realm_user_counts = get_remote_realm_user_counts()
total_row = []
server_mobile_data_counted = set()
remote_server_mobile_data_counted = set()
total_revenue = 0
total_mobile_users = 0
total_pushes = 0
for row in rows:
# Count mobile users and pushes forwarded, once per server
if row[SERVER_ID] not in server_mobile_data_counted:
if row[SERVER_ID] not in remote_server_mobile_data_counted:
if row[MOBILE_USER_COUNT] is not None:
total_mobile_users += row[MOBILE_USER_COUNT] # nocoverage
if row[MOBILE_PUSH_COUNT] is not None:
total_pushes += row[MOBILE_PUSH_COUNT] # nocoverage
server_mobile_data_counted.add(row[SERVER_ID])
remote_server_mobile_data_counted.add(row[SERVER_ID])
if row[REALM_ID] is None:
plan_data = plan_data_by_remote_server.get(row[SERVER_ID])
user_counts = get_remote_server_guest_and_non_guest_count(row[SERVER_ID])
audit_log_list = audit_logs_by_remote_server.get(row[SERVER_ID])
if audit_log_list is None:
user_counts = None # nocoverage
else:
user_counts = get_remote_customer_user_count(audit_log_list)
else:
server_remote_realms_data = plan_data_by_remote_server_and_realm.get(row[SERVER_ID])
if server_remote_realms_data is not None:
plan_data = server_remote_realms_data.get(row[REALM_ID]) # nocoverage
plan_data = server_remote_realms_data.get(row[REALM_ID])
else:
plan_data = None
remote_realm = RemoteRealm.objects.get(id=row[REALM_ID], server_id=row[SERVER_ID])
user_counts = get_remote_realm_guest_and_non_guest_count(remote_realm)
plan_data = None # nocoverage
user_counts = remote_realm_user_counts.get(row[REALM_ID])
# Format organization type for realm
org_type = row[ORG_TYPE]
row[ORG_TYPE] = get_org_type_display_name(org_type)
@ -145,16 +151,20 @@ def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
row.append("---")
row.append("---")
row.append("---")
else: # nocoverage
else:
total_revenue += plan_data.annual_revenue
revenue = cents_to_dollar_string(plan_data.annual_revenue)
row.append(plan_data.current_plan_name)
row.append(plan_data.current_status)
row.append(f"${revenue}")
# Add user counts
total_users = user_counts.non_guest_user_count + user_counts.guest_user_count
row.append(total_users)
row.append(user_counts.guest_user_count)
if user_counts is None:
row.append(0)
row.append(0)
else:
total_users = user_counts.non_guest_user_count + user_counts.guest_user_count
row.append(total_users)
row.append(user_counts.guest_user_count)
# Add server links
stats = remote_installation_stats_link(row[SERVER_ID])
support = remote_installation_support_link(row[SERVER_HOSTNAME])

60
corporate/lib/analytics.py
View File

@ -1,6 +1,8 @@
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime
from decimal import Decimal
from typing import Any, Dict
from typing import Any, Dict, List
from django.utils.timezone import now as timezone_now
@ -11,6 +13,11 @@ from corporate.lib.stripe import (
)
from corporate.models import Customer, CustomerPlan
from zerver.lib.utils import assert_is_not_none
from zilencer.models import (
RemoteCustomerUserCount,
RemoteRealmAuditLog,
get_remote_customer_user_count,
)
@dataclass
@ -147,3 +154,54 @@ def get_plan_data_by_remote_realm() -> Dict[int, Dict[int, RemoteActivityPlanDat
current_server_data[realm_id] = plan_data
return remote_server_plan_data_by_realm
def get_remote_realm_user_counts(
event_time: datetime = timezone_now(),
) -> Dict[int, RemoteCustomerUserCount]: # nocoverage
user_counts_by_realm: Dict[int, RemoteCustomerUserCount] = {}
for log in (
RemoteRealmAuditLog.objects.filter(
event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
event_time__lte=event_time,
remote_realm__isnull=False,
)
# Important: extra_data is empty for some pre-2020 audit logs
# prior to the introduction of realm_user_count_by_role
# logging. Meanwhile, modern Zulip servers using
# bulk_create_users to create the users in the system bot
# realm also generate such audit logs. Such audit logs should
# never be the latest in a normal realm.
.exclude(extra_data={})
.order_by("remote_realm", "-event_time")
.distinct("remote_realm")
):
assert log.remote_realm is not None
user_counts_by_realm[log.remote_realm.id] = get_remote_customer_user_count([log])
return user_counts_by_realm
def get_remote_server_audit_logs(
event_time: datetime = timezone_now(),
) -> Dict[int, List[RemoteRealmAuditLog]]:
logs_per_server: Dict[int, List[RemoteRealmAuditLog]] = defaultdict(list)
for log in (
RemoteRealmAuditLog.objects.filter(
event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
event_time__lte=event_time,
)
# Important: extra_data is empty for some pre-2020 audit logs
# prior to the introduction of realm_user_count_by_role
# logging. Meanwhile, modern Zulip servers using
# bulk_create_users to create the users in the system bot
# realm also generate such audit logs. Such audit logs should
# never be the latest in a normal realm.
.exclude(extra_data={})
.order_by("server_id", "realm_id", "-event_time")
.distinct("server_id", "realm_id")
.select_related("server")
):
logs_per_server[log.server.id].append(log)
return logs_per_server