data_import: Add migration status file to converted exports.

This commit updates all third-party importer tools (Slack, Mattermost,
and Rocket Chat) in the `zerver/data_import` directory to also output a
migration_status.json file in their output tarball.

This is required because all importable tarball will be checked for
migration compatibility during import.

Fixes #28443.

zerver/data_import/mattermost.py

@@ -40,6 +40,7 @@ from zerver.data_import.import_util import (
 from zerver.data_import.sequencer import NEXT_ID, IdMapper
 from zerver.data_import.user_handler import UserHandler
 from zerver.lib.emoji import name_to_codepoint
+from zerver.lib.export import do_common_export_processes
 from zerver.lib.markdown import IMAGE_EXTENSIONS
 from zerver.lib.upload import sanitize_name
 from zerver.lib.utils import process_list_in_batches
@@ -1013,3 +1014,5 @@ def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content:
         attachment: dict[str, list[Any]] = {"zerver_attachment": zerver_attachment}
         create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json")
         create_converted_data_files(attachment, realm_output_dir, "/attachment.json")
+
+        do_common_export_processes(realm_output_dir)

zerver/data_import/rocketchat.py

@@ -31,6 +31,7 @@ from zerver.data_import.import_util import (
 from zerver.data_import.sequencer import NEXT_ID, IdMapper
 from zerver.data_import.user_handler import UserHandler
 from zerver.lib.emoji import name_to_codepoint
+from zerver.lib.export import do_common_export_processes
 from zerver.lib.markdown import IMAGE_EXTENSIONS
 from zerver.lib.upload import sanitize_name
 from zerver.lib.utils import process_list_in_batches
@@ -1291,3 +1292,5 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
     attachment: dict[str, list[Any]] = {"zerver_attachment": zerver_attachment}
     create_converted_data_files(attachment, output_dir, "/attachment.json")
     create_converted_data_files(uploads_list, output_dir, "/uploads/records.json")
+
+    do_common_export_processes(output_dir)

zerver/data_import/slack.py

@@ -47,7 +47,7 @@ from zerver.data_import.slack_message_conversion import (
     get_user_full_name,
 )
 from zerver.lib.emoji import codepoint_to_name, get_emoji_file_name
-from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE
+from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processes
 from zerver.lib.mime_types import guess_type
 from zerver.lib.storage import static_path
 from zerver.lib.thumbnail import resize_realm_icon
@@ -1508,6 +1508,7 @@ def do_convert_directory(
     create_converted_data_files(uploads_records, output_dir, "/uploads/records.json")
     create_converted_data_files(attachment, output_dir, "/attachment.json")
     create_converted_data_files(realm_icon_records, output_dir, "/realm_icons/records.json")
+    do_common_export_processes(output_dir)
 
     logging.info("######### DATA CONVERSION FINISHED #########\n")
     logging.info("Zulip data dump created at %s", output_dir)

zerver/tests/test_mattermost_importer.py

@@ -698,6 +698,9 @@ class MatterMostImporter(ZulipTestCase):
         self.assertEqual(
             os.path.exists(os.path.join(harry_team_output_dir, "attachment.json")), True
         )
+        self.assertEqual(
+            os.path.exists(os.path.join(harry_team_output_dir, "migration_status.json")), True
+        )
 
         realm = self.read_file(harry_team_output_dir, "realm.json")
 

zerver/tests/test_rocketchat_importer.py

@@ -912,6 +912,7 @@ class RocketChatImporter(ZulipTestCase):
                 "INFO:root:Done processing emoji",
                 "INFO:root:Direct message group channel found. UIDs: ['LdBZ7kPxtKESyHPEe', 'M2sXGqoQRJQwQoXY2', 'os6N2Xg2JkNMCSW9Z']",
                 "INFO:root:skipping direct messages discussion mention: Discussion with Hermione",
+                "INFO:root:Exporting migration status",
             ],
         )
 
@@ -919,6 +920,7 @@ class RocketChatImporter(ZulipTestCase):
         self.assertEqual(os.path.exists(os.path.join(output_dir, "emoji")), True)
         self.assertEqual(os.path.exists(os.path.join(output_dir, "uploads")), True)
         self.assertEqual(os.path.exists(os.path.join(output_dir, "attachment.json")), True)
+        self.assertTrue(os.path.exists(output_dir + "/migration_status.json"))
 
         realm = self.read_file(output_dir, "realm.json")
 

zerver/tests/test_slack_importer.py

@@ -1337,6 +1337,7 @@ class SlackImporter(ZulipTestCase):
 
         self.assertTrue(os.path.exists(output_dir))
         self.assertTrue(os.path.exists(output_dir + "/realm.json"))
+        self.assertTrue(os.path.exists(output_dir + "/migration_status.json"))
 
         realm_icons_path = os.path.join(output_dir, "realm_icons")
         realm_icon_records_path = os.path.join(realm_icons_path, "records.json")