From ca8225cf4713bfc052bff62a0a53de364c1aef59 Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Tue, 19 Nov 2013 17:11:30 -0500 Subject: [PATCH] [manual] Add endpoint to cleanup a finished events queue. This requires a puppet apply on each of staging and prod0 to update the nginx configuration to support the new URL when it is deployed. (imported from commit a35a71a563fd1daca0d3ea4ec6874c5719a8564f) --- puppet/zulip/files/nginx/zulip-include/app | 2 +- tools/run-dev.py | 1 + zerver/lib/event_queue.py | 29 ++++++++++++++-------- zerver/management/commands/runtornado.py | 4 +++ zerver/tornadoviews.py | 11 ++++++++ zproject/urls.py | 3 ++- 6 files changed, 37 insertions(+), 13 deletions(-) diff --git a/puppet/zulip/files/nginx/zulip-include/app b/puppet/zulip/files/nginx/zulip-include/app index 7469a54651..25007ce0dd 100644 --- a/puppet/zulip/files/nginx/zulip-include/app +++ b/puppet/zulip/files/nginx/zulip-include/app @@ -14,7 +14,7 @@ location /static/ { } # Send longpoll requests to Tornado -location ~ /json/get_events|/api/v1/events { +location ~ /json/get_events|/json/events|/api/v1/events { proxy_pass http://localhost:9993; proxy_redirect off; diff --git a/tools/run-dev.py b/tools/run-dev.py index eeb1ac773b..e475db7c72 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -84,6 +84,7 @@ class Resource(resource.Resource): request.requestHeaders.setRawHeaders('X-Forwarded-Host', [proxy_host]) if (request.uri in ['/json/get_events'] or + request.uri.startswith('/json/events') or request.uri.startswith('/api/v1/events') or request.uri.startswith('/sockjs')): return proxy.ReverseProxyResource('localhost', tornado_port, '/'+name) diff --git a/zerver/lib/event_queue.py b/zerver/lib/event_queue.py index 4a8f50e95e..e271653746 100644 --- a/zerver/lib/event_queue.py +++ b/zerver/lib/event_queue.py @@ -141,6 +141,10 @@ class ClientDescriptor(object): ioloop.remove_timeout(self._timeout_handle) self._timeout_handle = None + def cleanup(self): + do_gc_event_queues([self.event_queue.id], [self.user_profile_id], + [self.realm_id]) + class EventQueue(object): def __init__(self, id): self.queue = deque() @@ -217,17 +221,7 @@ def allocate_client_descriptor(user_profile_id, realm_id, event_types, client_ty realm_clients_all_streams.setdefault(realm_id, []).append(client) return client -def gc_event_queues(): - start = time.time() - to_remove = set() - affected_users = set() - affected_realms = set() - for (id, client) in clients.iteritems(): - if client.idle(start): - to_remove.add(id) - affected_users.add(client.user_profile_id) - affected_realms.add(client.realm_id) - +def do_gc_event_queues(to_remove, affected_users, affected_realms): def filter_client_dict(client_dict, key): if key not in client_dict: return @@ -250,6 +244,19 @@ def gc_event_queues(): cb(clients[id].user_profile_id, clients[id], clients[id].user_profile_id not in user_clients) del clients[id] +def gc_event_queues(): + start = time.time() + to_remove = set() + affected_users = set() + affected_realms = set() + for (id, client) in clients.iteritems(): + if client.idle(start): + to_remove.add(id) + affected_users.add(client.user_profile_id) + affected_realms.add(client.realm_id) + + do_gc_event_queues(to_remove, affected_users, affected_realms) + logging.info(('Tornado removed %d idle event queues owned by %d users in %.3fs.' + ' Now %d active queues') % (len(to_remove), len(affected_users), time.time() - start, diff --git a/zerver/management/commands/runtornado.py b/zerver/management/commands/runtornado.py index d935db3979..9eabed1988 100644 --- a/zerver/management/commands/runtornado.py +++ b/zerver/management/commands/runtornado.py @@ -88,6 +88,7 @@ class Command(BaseCommand): try: urls = (r"/notify_tornado", r"/json/get_events", + r"/json/events", r"/api/v1/events", ) @@ -178,6 +179,9 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler): def post(self): self.get() + def delete(self): + self.get() + # Based on django.core.handlers.base: get_response def get_response(self, request): "Returns an HttpResponse object for the given HttpRequest" diff --git a/zerver/tornadoviews.py b/zerver/tornadoviews.py index dbd7e366b4..dbce608ffa 100644 --- a/zerver/tornadoviews.py +++ b/zerver/tornadoviews.py @@ -23,6 +23,17 @@ def notify(request): process_notification(ujson.loads(request.POST['data'])) return json_success() +@has_request_variables +def cleanup_event_queue(request, user_profile, queue_id=REQ()): + client = get_client_descriptor(queue_id) + if client is None: + return json_error("Bad event queue id: %s" % (queue_id,)) + if user_profile.id != client.user_profile_id: + return json_error("You are not authorized to access this queue") + request._log_data['extra'] = "[%s]" % (queue_id,) + client.cleanup() + return json_success() + @authenticated_json_post_view def json_get_events(request, user_profile): return get_events_backend(request, user_profile, apply_markdown=True) diff --git a/zproject/urls.py b/zproject/urls.py index 48c72e322f..4b3f10da1e 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -219,7 +219,8 @@ v1_api_and_json_patterns = patterns('zerver.views', ) + patterns('zerver.tornadoviews', url(r'^events$', 'rest_dispatch', - {'GET': 'get_events_backend'}), + {'GET': 'get_events_backend', + 'DELETE': 'cleanup_event_queue'}), ) if not settings.ENTERPRISE: v1_api_and_json_patterns += patterns('',