From d1aa68994a971f4dfb8a3351265e7b8fe27d2ce9 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 20 May 2021 12:13:53 -0700 Subject: [PATCH] render_messages: Use JSON Lines format. Signed-off-by: Anders Kaseorg --- zilencer/management/commands/compare_messages.py | 6 ++++-- zilencer/management/commands/render_messages.py | 7 ++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/zilencer/management/commands/compare_messages.py b/zilencer/management/commands/compare_messages.py index beb176ba39..93218b0978 100644 --- a/zilencer/management/commands/compare_messages.py +++ b/zilencer/management/commands/compare_messages.py @@ -1,6 +1,6 @@ from typing import Any -import ijson +import orjson from django.core.management.base import BaseCommand, CommandParser @@ -18,7 +18,9 @@ class Command(BaseCommand): total_count = 0 changed_count = 0 with open(options["dump1"]) as dump1, open(options["dump2"]) as dump2: - for m1, m2 in zip(ijson.items(dump1, "item"), ijson.items(dump2, "item")): + for line1, line2 in zip(dump1, dump2): + m1 = orjson.loads(line1) + m2 = orjson.loads(line2) total_count += 1 if m1["id"] != m2["id"]: self.stderr.write("Inconsistent messages dump") diff --git a/zilencer/management/commands/render_messages.py b/zilencer/management/commands/render_messages.py index b0c854002d..84001e8390 100644 --- a/zilencer/management/commands/render_messages.py +++ b/zilencer/management/commands/render_messages.py @@ -38,7 +38,6 @@ class Command(BaseCommand): os.makedirs(dest_dir) with open(options["destination"], "wb") as result: - result.write(b"[") messages = Message.objects.filter(id__gt=latest - amount, id__lte=latest).order_by("id") for message in queryset_iterator(messages): content = message.content @@ -59,9 +58,7 @@ class Command(BaseCommand): { "id": message.id, "content": render_markdown(message, content), - } + }, + option=orjson.OPT_APPEND_NEWLINE, ) ) - if message.id != latest: - result.write(b",") - result.write(b"]")