From a9838d808915221e3f15c9c4d4e8f1f14259beee Mon Sep 17 00:00:00 2001 From: PieterCK Date: Thu, 19 Sep 2024 22:54:50 +0700 Subject: [PATCH] import: Verify exported realm's migration compatibility. When transferring a realm to a server that has a different set of applied migrations (different Zulip versions), there is a chance that the imported data formats appear to be compatible but data invariants could still be violated. This commit adds an assertion during the import process to verify that both the exported realm and the importing server have matching Zulip versions and have a compatible set of migrations. --- zerver/lib/import_realm.py | 92 ++++++++- .../with_complete_migrations.json | 65 +++++++ .../with_missing_apps.json | 63 +++++++ .../with_unapplied_migrations.json | 61 ++++++ .../extra_migrations_error.txt | 17 ++ .../unapplied_migrations_error.txt | 17 ++ zerver/tests/test_import_export.py | 176 ++++++++++++++++++ 7 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json create mode 100644 zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json create mode 100644 zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json create mode 100644 zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt create mode 100644 zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index c3ef0a94ec..41e0f7782f 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -4,6 +4,7 @@ import os import shutil from concurrent.futures import ProcessPoolExecutor, as_completed from datetime import datetime, timezone +from difflib import unified_diff from typing import Any import bmemcached @@ -20,12 +21,22 @@ from psycopg2.extras import execute_values from psycopg2.sql import SQL, Identifier from analytics.models import RealmCount, StreamCount, UserCount +from version import ZULIP_VERSION from zerver.actions.create_realm import set_default_for_realm_permission_group_settings from zerver.actions.realm_settings import do_change_realm_plan_type from zerver.actions.user_settings import do_change_avatar_fields from zerver.lib.avatar_hash import user_avatar_base_path_from_ids from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields -from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName +from zerver.lib.export import ( + DATE_FIELDS, + Field, + MigrationStatusJson, + Path, + Record, + TableData, + TableName, + get_migrations_by_app, +) from zerver.lib.markdown import markdown_convert from zerver.lib.markdown import version as markdown_version from zerver.lib.message import get_last_message_id @@ -1124,6 +1135,16 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int = 1) -> Rea if not os.path.exists(import_dir): raise Exception("Missing import directory!") + migration_status_filename = os.path.join(import_dir, "migration_status.json") + if not os.path.exists(migration_status_filename): + raise Exception( + "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm." + ) + logging.info("Checking migration status of exported realm") + with open(migration_status_filename) as f: + migration_status: MigrationStatusJson = orjson.loads(f.read()) + check_migration_status(migration_status) + realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") @@ -2080,3 +2101,72 @@ def add_users_to_system_user_groups( ) for membership in usergroup_memberships ) + + +ZULIP_CLOUD_ONLY_APP_NAMES = ["zilencer", "corporate"] + + +def check_migration_status(exported_migration_status: MigrationStatusJson) -> None: + mismatched_migrations_log: dict[str, str] = {} + local_migration_status = MigrationStatusJson( + migrations_by_app=get_migrations_by_app(), zulip_version=ZULIP_VERSION + ) + + # Different major versions are the most common form of mismatch + # and should get a nice error message focused on that, not + # migrations. + # + # We could split on `-`, to get the maintenance release version, + # but unless migrations are different, it should generally be safe + # to import across minor release differences. + exported_primary_version = exported_migration_status["zulip_version"].split(".")[0] + local_primary_version = local_migration_status["zulip_version"].split(".")[0] + if exported_primary_version != local_primary_version: + raise Exception( + "Export was generated on a different Zulip major version.\n" + f"Export={exported_migration_status['zulip_version']}\n" + f"Server={local_migration_status['zulip_version']}" + ) + exported_migrations_by_app = exported_migration_status["migrations_by_app"] + local_migrations_by_app = local_migration_status["migrations_by_app"] + all_apps = set(exported_migrations_by_app.keys()).union(set(local_migrations_by_app.keys())) + + for app in all_apps: + exported_app_migrations = exported_migrations_by_app.get(app) + local_app_migrations = local_migrations_by_app.get(app) + + if app in ZULIP_CLOUD_ONLY_APP_NAMES and ( + local_app_migrations is None or exported_app_migrations is None + ): + # This applications are expected to be present only on + # Zulip Cloud, so don't warn about them. + continue + + if not exported_app_migrations: + logging.warning("This server has '%s' app installed, but exported realm does not.", app) + elif not local_app_migrations: + logging.warning("Exported realm has '%s' app installed, but this server does not.", app) + elif local_app_migrations != exported_app_migrations: + diff = list( + unified_diff(exported_app_migrations, local_app_migrations, lineterm="", n=1) + ) + mismatched_migrations_log[f"\n'{app}' app:\n"] = "\n".join(diff[3:]) + + if mismatched_migrations_log: + # The order of the list output by the diff is nondeterministic, so the + # error logs needs to be sorted first. + sorted_error_log: list[str] = [ + f"{key}{value}" for key, value in sorted(mismatched_migrations_log.items()) + ] + + error_message = ( + "Export was generated on a different Zulip version.\n" + f"Export={exported_migration_status['zulip_version']}\n" + f"Server={local_migration_status['zulip_version']}\n" + "\n" + "Database formats differ between the exported realm and this server.\n" + "Printing migrations that differ between the versions:\n" + "--- exported realm\n" + "+++ this server" + ) + "\n".join(sorted_error_log) + raise Exception(error_message) diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json new file mode 100644 index 0000000000..966beee297 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json @@ -0,0 +1,65 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length", + "0003_alter_user_email_max_length", + "0004_alter_user_username_opts" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain", + "0030_realm_org_type", + "0031_remove_system_avatar_source" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "phonenumber": ["0001_initial", "0001_squashed_0001_initial"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "sessions": ["0001_initial"], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json new file mode 100644 index 0000000000..9f28d3effa --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json @@ -0,0 +1,63 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length", + "0003_alter_user_email_max_length", + "0004_alter_user_username_opts" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain", + "0030_realm_org_type", + "0031_remove_system_avatar_source" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json new file mode 100644 index 0000000000..25753b4bf2 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json @@ -0,0 +1,61 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "phonenumber": ["0001_initial", "0001_squashed_0001_initial"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "sessions": ["0001_initial"], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt new file mode 100644 index 0000000000..f3d9fb7481 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt @@ -0,0 +1,17 @@ +Export was generated on a different Zulip version. +Export=10.0-dev+git +Server=10.0-dev+git + +Database formats differ between the exported realm and this server. +Printing migrations that differ between the versions: +--- exported realm ++++ this server +'auth' app: + 0002_alter_permission_name_max_length +-0003_alter_user_email_max_length +-0004_alter_user_username_opts + +'zerver' app: + 0029_realm_subdomain +-0030_realm_org_type +-0031_remove_system_avatar_source diff --git a/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt new file mode 100644 index 0000000000..c00cec3229 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt @@ -0,0 +1,17 @@ +Export was generated on a different Zulip version. +Export=10.0-dev+git +Server=10.0-dev+git + +Database formats differ between the exported realm and this server. +Printing migrations that differ between the versions: +--- exported realm ++++ this server +'auth' app: + 0002_alter_permission_name_max_length ++0003_alter_user_email_max_length ++0004_alter_user_username_opts + +'zerver' app: + 0029_realm_subdomain ++0030_realm_org_type ++0031_remove_system_avatar_source diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 3b71d30e9d..d0910a3dcd 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -16,6 +16,7 @@ from django.utils.timezone import now as timezone_now from typing_extensions import override from analytics.models import UserCount +from version import ZULIP_VERSION from zerver.actions.alert_words import do_add_alert_words from zerver.actions.create_user import do_create_user from zerver.actions.custom_profile_fields import ( @@ -43,6 +44,7 @@ from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.bot_config import set_bot_config from zerver.lib.bot_lib import StateHandler from zerver.lib.export import ( + AppMigrations, MigrationStatusJson, Record, do_export_realm, @@ -340,6 +342,12 @@ class ExportFile(ZulipTestCase): db_paths = {user_avatar_path(user) + ".original"} self.assertEqual(exported_paths, db_paths) + def get_applied_migrations_fixture(self, fixture_name: str) -> AppMigrations: + fixture = orjson.loads( + self.fixture_data(fixture_name, "import_fixtures/applied_migrations_fixtures") + ) + return fixture + def verify_migration_status_json(self) -> None: # This function asserts that the generated migration_status.json # is structurally familiar for it to be used for assertion at @@ -2032,6 +2040,174 @@ class RealmImportExportTest(ExportFile): expected_group_names.add(SystemGroups.FULL_MEMBERS) self.assertSetEqual(logged_membership_by_user_id[user.id], expected_group_names) + def test_import_realm_with_unapplied_migrations(self) -> None: + realm = get_realm("zulip") + with ( + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_unapplied_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self.export_realm( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + + expected_error_message = self.fixture_data( + "unapplied_migrations_error.txt", "import_fixtures/check_migrations_errors" + ).strip() + error_message = str(e.exception).strip() + self.assertEqual(expected_error_message, error_message) + + def test_import_realm_with_extra_migrations(self) -> None: + realm = get_realm("zulip") + with ( + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture( + "with_unapplied_migrations.json" + ) + self.export_realm( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + expected_error_message = self.fixture_data( + "extra_migrations_error.txt", "import_fixtures/check_migrations_errors" + ).strip() + error_message = str(e.exception).strip() + self.assertEqual(expected_error_message, error_message) + + def test_import_realm_with_extra_exported_apps(self) -> None: + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="WARNING") as mock_log, + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture("with_missing_apps.json") + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + missing_apps_log = [ + "WARNING:root:Exported realm has 'phonenumber' app installed, but this server does not.", + "WARNING:root:Exported realm has 'sessions' app installed, but this server does not.", + ] + # The log output is sorted because it's order is nondeterministic. + self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log)) + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_import_realm_with_missing_apps(self) -> None: + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="WARNING") as mock_log, + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture("with_missing_apps.json") + mock_import.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + missing_apps_log = [ + "WARNING:root:This server has 'phonenumber' app installed, but exported realm does not.", + "WARNING:root:This server has 'sessions' app installed, but exported realm does not.", + ] + self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log)) + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_check_migration_for_zulip_cloud_realm(self) -> None: + # This test ensures that `check_migrations_status` correctly handles + # checking the migrations of a Zulip Cloud-like realm (with zilencer/ + # corporate apps installed) when importing into a self-hosted realm + # (where these apps are not installed). + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self_hosted_migrations = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + for key in ["zilencer", "corporate"]: + self_hosted_migrations.pop(key, None) + mock_import.return_value = self_hosted_migrations + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_import_realm_without_migration_status_file(self) -> None: + realm = get_realm("zulip") + with patch("zerver.lib.export.export_migration_status"): + self.export_realm_and_create_auditlog(realm) + + with self.assertRaises(Exception) as e, self.assertLogs(level="INFO"): + do_import_realm( + get_output_dir(), + "test-zulip", + ) + expected_error_message = "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm." + self.assertEqual(expected_error_message, str(e.exception)) + + def test_import_realm_with_different_stated_zulip_version(self) -> None: + realm = get_realm("zulip") + self.export_realm_and_create_auditlog(realm) + + with ( + patch("zerver.lib.import_realm.ZULIP_VERSION", "8.0"), + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + ): + do_import_realm( + get_output_dir(), + "test-zulip", + ) + expected_error_message = ( + "Export was generated on a different Zulip major version.\n" + f"Export={ZULIP_VERSION}\n" + "Server=8.0" + ) + self.assertEqual(expected_error_message, str(e.exception)) + class SingleUserExportTest(ExportFile): def do_files_test(self, is_s3: bool) -> None: