From ede4a88b495b5ab6ebf9ce8b111519cc825bde63 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Fri, 15 Apr 2022 09:45:47 -0700 Subject: [PATCH] populate_db: Remove unnecessary pre-population of URL cache. 76deb30312ac changed this to not just be the URL, but rather a prefixed hash of the URL, but failed to update this location which wrote to it. This meant that this pre-population step was writing to the wrong keys in the durable cache, and thus ineffective. Then, da33b72848cc switched the cache to be in-memory, making this write to the wrong keys in an in-process memory store. There is no way to pre-fill this sort of cache, except at server start-up. Finally, and most fundamentally, 8c0c9ca7a4e1 then disabled `inline_url_embed_preview` by default, making the code entirely moot. Remove the triply-unnecessary code. --- .../tests/fixtures/docs_url_preview_data.json | 26 ------------------- zilencer/management/commands/populate_db.py | 11 -------- 2 files changed, 37 deletions(-) delete mode 100644 zerver/tests/fixtures/docs_url_preview_data.json diff --git a/zerver/tests/fixtures/docs_url_preview_data.json b/zerver/tests/fixtures/docs_url_preview_data.json deleted file mode 100644 index 4c4ae68b66..0000000000 --- a/zerver/tests/fixtures/docs_url_preview_data.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "https://zulip.readthedocs.io/en/latest/subsystems/analytics.html": { - "description": "Zulip has a cool analytics system for tracking various useful statistics\nthat currently power the /stats page, and over time will power other\nfeatures, like showing usage statistics for the various streams. It is\ndesigned around the following goals:", - "title": "Analytics — Zulip 1.7.1+git documentation" - }, - "https://zulip.readthedocs.io/en/latest/subsystems/client.html": { - "description": "zerver.models.Client is Zulip analogue of the HTTP User-Agent\nheader (and is populated from User-Agent). It exists for use in\nanalytics and other places to provide human-readable summary data\nabout which Zulip client was used for an operation (e.g. was it the\nAndroid app, the desktop app, or a bot?).", - "title": "Clients in Zulip — Zulip 1.7.1+git documentation" - }, - "https://zulip.readthedocs.io/en/latest/subsystems/emoji.html": { - "description": "Emoji seem like a simple idea, but there actually a ton of\ncomplexity that goes into an effective emoji implementation. This\ndocument discusses a number of these issues.", - "title": "Emoji — Zulip 1.7.1+git documentation" - }, - "https://zulip.readthedocs.io/en/latest/subsystems/full-text-search.html": { - "description": "Zulip supports full-text search, which can be combined arbitrarily\nwith Zulip full suite of narrowing operators. By default, it only\nsupports English text, but there is an experimental\nPGroonga integration that provides\nfull-text search for all languages.", - "title": "Full-text search — Zulip 1.7.1+git documentation" - }, - "https://zulip.readthedocs.io/en/latest/subsystems/logging.html": { - "description": "Having a good system for logging error reporting is essential to\nmaking a large project like Zulip successful. Without reliable error\nreporting, one has to rely solely on bug reports from users in order\nto produce a working product.", - "title": "Logging and Error reporting — Zulip 1.7.1+git documentation" - }, - "https://zulip.readthedocs.io/en/latest/translating/translating.html": { - "description": "To make Zulip even better for users around the world, the Zulip UI is\nbeing translated into a number of major languages, including Spanish,\nGerman, Hindi, French, Chinese, Russian, and Japanese, with varying levels of\nprogress. If you speak a language other than English, your help with\ntranslating Zulip would be greatly appreciated!", - "title": "Translation guidelines — Zulip 1.7.1+git documentation" - } -} diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 1183a6002f..90eea15107 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -28,7 +28,6 @@ from zerver.actions.message_send import build_message_send_dict, do_send_message from zerver.actions.realm_emoji import check_add_realm_emoji from zerver.actions.users import do_change_user_role from zerver.lib.bulk_create import bulk_create_streams -from zerver.lib.cache import cache_set from zerver.lib.generate_test_data import create_test_data, generate_topics from zerver.lib.onboarding import create_if_missing_realm_internal_bots from zerver.lib.push_notifications import logger as push_notifications_logger @@ -36,7 +35,6 @@ from zerver.lib.server_initialization import create_internal_realm, create_users from zerver.lib.storage import static_path from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS from zerver.lib.types import ProfileFieldData -from zerver.lib.url_preview.preview import CACHE_NAME as PREVIEW_CACHE_NAME from zerver.lib.user_groups import create_user_group from zerver.lib.users import add_service from zerver.lib.utils import generate_api_key @@ -810,15 +808,6 @@ class Command(BaseCommand): # Generate a new set of test data. create_test_data() - # prepopulate the URL preview/embed data for the links present - # in the config.generate_data.json data set. This makes it - # possible for populate_db to run happily without Internet - # access. - with open("zerver/tests/fixtures/docs_url_preview_data.json", "rb") as f: - urls_with_preview_data = orjson.loads(f.read()) - for url in urls_with_preview_data: - cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) - if options["delete"]: if options["test_suite"]: # Create test users; the MIT ones are needed to test