tornado: Fill the message memcached in a child process.

(imported from commit 3a7b4c0f4aad37319f4cfa7892709e99883f6a87)

zephyr/lib/message_cache.py

@@ -1,6 +1,8 @@
 from zephyr.models import Message
 from zephyr.lib.cache import cache_with_key, djcache
 
+MESSAGE_CACHE_SIZE = 25000
+
 def message_cache_key(message_id):
     return "message:%d" % (message_id,)
 
@@ -12,11 +14,12 @@ def cache_get_message(message_id):
     return Message.objects.select_related("client", "sender").get(id=message_id)
 
 # Called on Tornado startup to ensure our message cache isn't empty
-def populate_message_cache(count):
+def populate_message_cache():
     max_message_id = 0
     min_message_id = 0
     messages_for_memcached = {}
-    for m in Message.objects.select_related("sender", "client").all().order_by("-id")[0:count]:
+    for m in Message.objects.select_related("sender", "client").all().order_by(
+        "-id")[0:MESSAGE_CACHE_SIZE]:
         max_message_id = max(m.id, max_message_id)
         min_message_id = min(m.id, min_message_id)
         messages_for_memcached[message_cache_key(m.id)] = (m,)

zephyr/management/commands/fill_message_cache.py

@@ -0,0 +1,10 @@
+from optparse import make_option
+from django.core.management.base import BaseCommand
+from zephyr.lib.message_cache import populate_message_cache
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list
+    help = "Populate the memcached cache of messages."
+
+    def handle(self, *args, **options):
+        populate_message_cache()

zephyr/tornadoviews.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from zephyr.models import Message, UserProfile, UserMessage, UserActivity
 
 from zephyr.decorator import asynchronous, authenticated_api_view, \
@@ -6,6 +7,7 @@ from zephyr.decorator import asynchronous, authenticated_api_view, \
     JsonableError
 from zephyr.lib.response import json_success, json_error
 
+import os
 import datetime
 import simplejson
 import socket
@@ -13,10 +15,10 @@ import time
 import collections
 import sys
 import logging
+import subprocess
 from django.core.cache import cache
 from zephyr.lib.cache import cache_with_key
-from zephyr.lib.message_cache import cache_save_message, cache_get_message, \
-    populate_message_cache
+from zephyr.lib.message_cache import cache_save_message, cache_get_message
 
 SERVER_GENERATION = int(time.time())
 
@@ -74,6 +76,10 @@ def initialize_user_messages():
     for um in UserMessage.objects.filter(message_id__gte=cache_minimum_id).order_by("message"):
         add_user_message(um.user_profile_id, um.message_id)
 
+    # Filling the memcached cache is a little slow, so do it in a child process.
+    subprocess.Popen(["python", os.path.join(os.path.dirname(__file__), "..", "manage.py"),
+                      "fill_message_cache"])
+
 def add_user_message(user_profile_id, message_id):
     if cache_minimum_id == sys.maxint:
         initialize_user_messages()