From 0d7199b22ec3beaa9cd899bd4d64e12c54b80a5a Mon Sep 17 00:00:00 2001 From: PieterCK Date: Tue, 24 Sep 2024 17:21:20 +0700 Subject: [PATCH] data_import: Add migration status file to converted exports. This commit updates all third-party importer tools (Slack, Mattermost, and Rocket Chat) in the `zerver/data_import` directory to also output a migration_status.json file in their output tarball. This is required because all importable tarball will be checked for migration compatibility during import. Fixes #28443. --- zerver/data_import/mattermost.py | 3 +++ zerver/data_import/rocketchat.py | 3 +++ zerver/data_import/slack.py | 3 ++- zerver/tests/test_mattermost_importer.py | 3 +++ zerver/tests/test_rocketchat_importer.py | 2 ++ zerver/tests/test_slack_importer.py | 1 + 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py index bfc8e1b43f..0fc3d8d75f 100644 --- a/zerver/data_import/mattermost.py +++ b/zerver/data_import/mattermost.py @@ -40,6 +40,7 @@ from zerver.data_import.import_util import ( from zerver.data_import.sequencer import NEXT_ID, IdMapper from zerver.data_import.user_handler import UserHandler from zerver.lib.emoji import name_to_codepoint +from zerver.lib.export import do_common_export_processes from zerver.lib.markdown import IMAGE_EXTENSIONS from zerver.lib.upload import sanitize_name from zerver.lib.utils import process_list_in_batches @@ -1013,3 +1014,5 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: attachment: dict[str, list[Any]] = {"zerver_attachment": zerver_attachment} create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json") create_converted_data_files(attachment, realm_output_dir, "/attachment.json") + + do_common_export_processes(realm_output_dir) diff --git a/zerver/data_import/rocketchat.py b/zerver/data_import/rocketchat.py index 13e6205f2e..d2502951f7 100644 --- a/zerver/data_import/rocketchat.py +++ b/zerver/data_import/rocketchat.py @@ -31,6 +31,7 @@ from zerver.data_import.import_util import ( from zerver.data_import.sequencer import NEXT_ID, IdMapper from zerver.data_import.user_handler import UserHandler from zerver.lib.emoji import name_to_codepoint +from zerver.lib.export import do_common_export_processes from zerver.lib.markdown import IMAGE_EXTENSIONS from zerver.lib.upload import sanitize_name from zerver.lib.utils import process_list_in_batches @@ -1291,3 +1292,5 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: attachment: dict[str, list[Any]] = {"zerver_attachment": zerver_attachment} create_converted_data_files(attachment, output_dir, "/attachment.json") create_converted_data_files(uploads_list, output_dir, "/uploads/records.json") + + do_common_export_processes(output_dir) diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 5c880a18a6..27e0a56f21 100644 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -47,7 +47,7 @@ from zerver.data_import.slack_message_conversion import ( get_user_full_name, ) from zerver.lib.emoji import codepoint_to_name, get_emoji_file_name -from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE +from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processes from zerver.lib.mime_types import guess_type from zerver.lib.storage import static_path from zerver.lib.thumbnail import resize_realm_icon @@ -1508,6 +1508,7 @@ def do_convert_directory( create_converted_data_files(uploads_records, output_dir, "/uploads/records.json") create_converted_data_files(attachment, output_dir, "/attachment.json") create_converted_data_files(realm_icon_records, output_dir, "/realm_icons/records.json") + do_common_export_processes(output_dir) logging.info("######### DATA CONVERSION FINISHED #########\n") logging.info("Zulip data dump created at %s", output_dir) diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py index 36019b00e6..920b3c2b94 100644 --- a/zerver/tests/test_mattermost_importer.py +++ b/zerver/tests/test_mattermost_importer.py @@ -698,6 +698,9 @@ class MatterMostImporter(ZulipTestCase): self.assertEqual( os.path.exists(os.path.join(harry_team_output_dir, "attachment.json")), True ) + self.assertEqual( + os.path.exists(os.path.join(harry_team_output_dir, "migration_status.json")), True + ) realm = self.read_file(harry_team_output_dir, "realm.json") diff --git a/zerver/tests/test_rocketchat_importer.py b/zerver/tests/test_rocketchat_importer.py index 92e3a164e9..508e59fd50 100644 --- a/zerver/tests/test_rocketchat_importer.py +++ b/zerver/tests/test_rocketchat_importer.py @@ -912,6 +912,7 @@ class RocketChatImporter(ZulipTestCase): "INFO:root:Done processing emoji", "INFO:root:Direct message group channel found. UIDs: ['LdBZ7kPxtKESyHPEe', 'M2sXGqoQRJQwQoXY2', 'os6N2Xg2JkNMCSW9Z']", "INFO:root:skipping direct messages discussion mention: Discussion with Hermione", + "INFO:root:Exporting migration status", ], ) @@ -919,6 +920,7 @@ class RocketChatImporter(ZulipTestCase): self.assertEqual(os.path.exists(os.path.join(output_dir, "emoji")), True) self.assertEqual(os.path.exists(os.path.join(output_dir, "uploads")), True) self.assertEqual(os.path.exists(os.path.join(output_dir, "attachment.json")), True) + self.assertTrue(os.path.exists(output_dir + "/migration_status.json")) realm = self.read_file(output_dir, "realm.json") diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 434f4e1a8a..21c5c454f1 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -1337,6 +1337,7 @@ class SlackImporter(ZulipTestCase): self.assertTrue(os.path.exists(output_dir)) self.assertTrue(os.path.exists(output_dir + "/realm.json")) + self.assertTrue(os.path.exists(output_dir + "/migration_status.json")) realm_icons_path = os.path.join(output_dir, "realm_icons") realm_icon_records_path = os.path.join(realm_icons_path, "records.json")