From d1bab42da6e9a0379504377ce93dceba2c3f9c2f Mon Sep 17 00:00:00 2001 From: Mateusz Mandera Date: Fri, 12 Jul 2019 00:49:19 +0200 Subject: [PATCH] populate_db: Send generated messages in one do_send_messages call. Sending messages one-by-one is significantly slower compared to taking advantage of the batch-handling code in do_send_messages, so we pass all the messages to the function in one go. This is memory-hungry if there's a lot of messages, so we will allow splitting into smaller batches in the next commit. --- zilencer/management/commands/populate_db.py | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 66084ff534..202cde3437 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -590,6 +590,7 @@ def send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], num_messages = 0 random_max = 1000000 recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]] + messages = [] while num_messages < tot_messages: saved_data = {} # type: Dict[str, Any] message = Message() @@ -638,19 +639,21 @@ def send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], saved_data['subject'] = message.subject message.pub_date = choose_pub_date(num_messages, tot_messages, options['threads']) - - # We disable USING_RABBITMQ here, so that deferred work is - # executed in do_send_message_messages, rather than being - # queued. This is important, because otherwise, if run-dev.py - # wasn't running when populate_db was run, a developer can end - # up with queued events that reference objects from a previous - # life of the database, which naturally throws exceptions. - settings.USING_RABBITMQ = False - do_send_messages([{'message': message}]) - settings.USING_RABBITMQ = True + messages.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 + + # We disable USING_RABBITMQ here, so that deferred work is + # executed in do_send_message_messages, rather than being + # queued. This is important, because otherwise, if run-dev.py + # wasn't running when populate_db was run, a developer can end + # up with queued events that reference objects from a previous + # life of the database, which naturally throws exceptions. + settings.USING_RABBITMQ = False + do_send_messages([{'message': message} for message in messages]) + settings.USING_RABBITMQ = True + return tot_messages def choose_pub_date(num_messages: int, tot_messages: int, threads: int) -> datetime: