mirror of https://github.com/zulip/zulip.git
Move existing queue processing workers to new framework
(imported from commit b448eeefcddb1840344bd337046be30714a7fdbc)
This commit is contained in:
Zev Benjamin
parent
6cf5ebc63d
commit
e450fc80c9
|
|
@ -55,7 +55,7 @@ stdout_logfile=/var/log/humbug/tornado.log ; stdout log path, NONE for n
|
|||
directory=/home/humbug/humbug-deployments/current/
|
||||
|
||||
[program:humbug-events-user-activity]
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_user_activity
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_queue user_activity
|
||||
priority=300 ; the relative start priority (default 999)
|
||||
autostart=true ; start at supervisord start (default: true)
|
||||
autorestart=true ; whether/when to restart (default: unexpected)
|
||||
|
|
@ -67,7 +67,7 @@ stdout_logfile=/var/log/humbug/events-user-activity.log ; stdout log pat
|
|||
directory=/home/humbug/humbug-deployments/current/
|
||||
|
||||
[program:humbug-events-signups]
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_signups
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_queue signups
|
||||
priority=400 ; the relative start priority (default 999)
|
||||
autostart=true ; start at supervisord start (default: true)
|
||||
autorestart=true ; whether/when to restart (default: unexpected)
|
||||
|
|
@ -79,7 +79,7 @@ stdout_logfile=/var/log/humbug/events-signups.log ; stdout log path, NON
|
|||
directory=/home/humbug/humbug-deployments/current/
|
||||
|
||||
[program:humbug-events-confirmation-emails]
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py send_confirmation_emails
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_queue invites
|
||||
priority=500 ; the relative start priority (default 999)
|
||||
autostart=true ; start at supervisord start (default: true)
|
||||
autorestart=true ; whether/when to restart (default: unexpected)
|
||||
|
|
@ -91,7 +91,7 @@ stdout_logfile=/var/log/humbug/events-confirmation-emails.log ; stdout l
|
|||
directory=/home/humbug/humbug-deployments/current/
|
||||
|
||||
[program:humbug-events-missedmessage_reminders]
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py send_missedmessage_reminders
|
||||
command=python /home/humbug/humbug-deployments/current/manage.py process_queue missedmessage_emails
|
||||
priority=600 ; the relative start priority (default 999)
|
||||
autostart=true ; start at supervisord start (default: true)
|
||||
autorestart=true ; whether/when to restart (default: unexpected)
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from postmonkey import PostMonkey, MailChimpException
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.conf import settings
|
||||
|
||||
import logging
|
||||
|
||||
from zerver.lib.queue import SimpleQueueClient
|
||||
|
||||
class Command(BaseCommand):
|
||||
pm = PostMonkey(settings.MAILCHIMP_API_KEY, timeout=10)
|
||||
|
||||
def subscribe(self, ch, method, properties, data):
|
||||
try:
|
||||
self.pm.listSubscribe(
|
||||
id=settings.ZULIP_FRIENDS_LIST_ID,
|
||||
email_address=data['EMAIL'],
|
||||
merge_vars=data['merge_vars'],
|
||||
double_optin=False,
|
||||
send_welcome=False)
|
||||
except MailChimpException, e:
|
||||
if e.code == 214:
|
||||
logging.warning("Attempted to sign up already existing email to list: %s" % (data['EMAIL'],))
|
||||
else:
|
||||
raise e
|
||||
|
||||
def handle(self, *args, **options):
|
||||
q = SimpleQueueClient()
|
||||
q.register_json_consumer("signups", self.subscribe)
|
||||
q.start_consuming()
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from zerver.lib.actions import process_user_activity_event, \
|
||||
process_user_presence_event, process_user_activity_interval_event
|
||||
from zerver.lib.queue import SimpleQueueClient
|
||||
import sys
|
||||
import signal
|
||||
import os
|
||||
import traceback
|
||||
import ujson
|
||||
|
||||
ERROR_LOG_FILE = os.path.join(settings.ERROR_LOG_DIR, "process_user_activity")
|
||||
|
||||
class Command(BaseCommand):
|
||||
option_list = BaseCommand.option_list
|
||||
help = "Process UserActivity log messages."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
activity_queue = SimpleQueueClient()
|
||||
|
||||
def callback_activity(ch, method, properties, event):
|
||||
print " [x] Received activity %r" % (event,)
|
||||
try:
|
||||
process_event(event)
|
||||
except Exception:
|
||||
if not os.path.exists(settings.ERROR_LOG_DIR):
|
||||
os.mkdir(settings.ERROR_LOG_DIR)
|
||||
# One can parse out just the JSON records from this log format using:
|
||||
#
|
||||
# grep "Error Processing" errors/process_user_activity | cut -f 2- -d:
|
||||
file(ERROR_LOG_FILE, "a").write(
|
||||
"Error Processing event: " + ujson.dumps(event) + "\n" +
|
||||
traceback.format_exc())
|
||||
|
||||
def process_event(event):
|
||||
msg_type = event['type']
|
||||
if msg_type == 'user_activity':
|
||||
process_user_activity_event(event)
|
||||
elif msg_type == 'user_presence':
|
||||
process_user_presence_event(event)
|
||||
elif msg_type == 'user_activity_interval':
|
||||
process_user_activity_interval_event(event)
|
||||
else:
|
||||
print("[*] Unknown message type: %s" % (msg_type,))
|
||||
|
||||
def signal_handler(signal, frame):
|
||||
print("[*] Closing and disconnecting from queues")
|
||||
activity_queue.stop_consuming()
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
print ' [*] Waiting for messages. To exit press CTRL+C'
|
||||
activity_queue.register_json_consumer('user_activity', callback_activity)
|
||||
activity_queue.start_consuming()
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from zerver.models import get_user_profile_by_email, get_prereg_user_by_email
|
||||
from zerver.lib.queue import SimpleQueueClient
|
||||
from zerver.lib.actions import do_send_confirmation_email
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
Send confirmation e-mails to invited users.
|
||||
|
||||
This command processes events from the `invites` queue.
|
||||
"""
|
||||
def subscribe(self, ch, method, properties, data):
|
||||
invitee = get_prereg_user_by_email(data["email"])
|
||||
referrer = get_user_profile_by_email(data["referrer_email"])
|
||||
do_send_confirmation_email(invitee, referrer)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
q = SimpleQueueClient()
|
||||
q.register_json_consumer("invites", self.subscribe)
|
||||
q.start_consuming()
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
import time
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from zerver.lib.queue import SimpleQueueClient
|
||||
from zerver.lib.actions import handle_missedmessage_emails
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
q = SimpleQueueClient()
|
||||
while True:
|
||||
missed_events = q.drain_queue("missedmessage_emails", json=True)
|
||||
by_recipient = defaultdict(list)
|
||||
|
||||
for event in missed_events:
|
||||
print "Received missed message event: %s" % (event,)
|
||||
by_recipient[event['user_profile_id']].append(event)
|
||||
|
||||
for user_profile_id, events in by_recipient.items():
|
||||
handle_missedmessage_emails(user_profile_id, events)
|
||||
|
||||
# Aggregate all messages received every 2 minutes to let someone finish sending a batch
|
||||
# of messages
|
||||
time.sleep(2 * 60)
|
||||
|
|
@ -1,6 +1,18 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from django.conf import settings
|
||||
from postmonkey import PostMonkey, MailChimpException
|
||||
from zerver.models import get_user_profile_by_email, get_prereg_user_by_email
|
||||
from zerver.lib.queue import SimpleQueueClient
|
||||
from zerver.lib.actions import handle_missedmessage_emails, process_user_activity_event, \
|
||||
process_user_presence_event, process_user_activity_interval_event, do_send_confirmation_email
|
||||
import os
|
||||
import ujson
|
||||
import traceback
|
||||
from collections import defaultdict
|
||||
import time
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
def assign_queue(queue_name):
|
||||
def decorate(clazz):
|
||||
|
|
@ -26,3 +38,76 @@ class QueueProcessingWorker(object):
|
|||
|
||||
def stop(self):
|
||||
self.q.stop_consuming()
|
||||
|
||||
@assign_queue('signups')
|
||||
class SignupWorker(QueueProcessingWorker):
|
||||
def __init__(self):
|
||||
super(SignupWorker, self).__init__()
|
||||
self.pm = PostMonkey(settings.MAILCHIMP_API_KEY, timeout=10)
|
||||
|
||||
def consume(self, ch, method, properties, data):
|
||||
try:
|
||||
self.pm.listSubscribe(
|
||||
id=settings.ZULIP_FRIENDS_LIST_ID,
|
||||
email_address=data['EMAIL'],
|
||||
merge_vars=data['merge_vars'],
|
||||
double_optin=False,
|
||||
send_welcome=False)
|
||||
except MailChimpException, e:
|
||||
if e.code == 214:
|
||||
logging.warning("Attempted to sign up already existing email to list: %s" % (data['EMAIL'],))
|
||||
else:
|
||||
raise e
|
||||
|
||||
@assign_queue('invites')
|
||||
class ConfirmationEmailWorker(QueueProcessingWorker):
|
||||
def consume(self, ch, method, properties, data):
|
||||
invitee = get_prereg_user_by_email(data["email"])
|
||||
referrer = get_user_profile_by_email(data["referrer_email"])
|
||||
do_send_confirmation_email(invitee, referrer)
|
||||
|
||||
@assign_queue('user_activity')
|
||||
class UserActivityWorker(QueueProcessingWorker):
|
||||
ERROR_LOG_FILE = os.path.join(settings.ERROR_LOG_DIR, "process_user_activity")
|
||||
def consume(self, ch, method, properties, event):
|
||||
print " [x] Received activity %r" % (event,)
|
||||
try:
|
||||
self.process_event(event)
|
||||
except Exception:
|
||||
if not os.path.exists(settings.ERROR_LOG_DIR):
|
||||
os.mkdir(settings.ERROR_LOG_DIR)
|
||||
# One can parse out just the JSON records from this log format using:
|
||||
#
|
||||
# grep "Error Processing" errors/process_user_activity | cut -f 2- -d:
|
||||
file(self.ERROR_LOG_FILE, "a").write(
|
||||
"Error Processing event: " + ujson.dumps(event) + "\n" +
|
||||
traceback.format_exc())
|
||||
|
||||
def process_event(self, event):
|
||||
msg_type = event['type']
|
||||
if msg_type == 'user_activity':
|
||||
process_user_activity_event(event)
|
||||
elif msg_type == 'user_presence':
|
||||
process_user_presence_event(event)
|
||||
elif msg_type == 'user_activity_interval':
|
||||
process_user_activity_interval_event(event)
|
||||
else:
|
||||
print("[*] Unknown message type: %s" % (msg_type,))
|
||||
|
||||
@assign_queue('missedmessage_emails')
|
||||
class MissedMessageWorker(QueueProcessingWorker):
|
||||
def start(self):
|
||||
while True:
|
||||
missed_events = self.q.drain_queue("missedmessage_emails", json=True)
|
||||
by_recipient = defaultdict(list)
|
||||
|
||||
for event in missed_events:
|
||||
logging.info("Received event: %s" % (event,))
|
||||
by_recipient[event['user_profile_id']].append(event)
|
||||
|
||||
for user_profile_id, events in by_recipient.items():
|
||||
handle_missedmessage_emails(user_profile_id, events)
|
||||
|
||||
# Aggregate all messages received every 2 minutes to let someone finish sending a batch
|
||||
# of messages
|
||||
time.sleep(2 * 60)
|
||||
|
|
|
|||
Loading…
Reference in New Issue