diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index c3ef0a94ec..41e0f7782f 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -4,6 +4,7 @@ import os import shutil from concurrent.futures import ProcessPoolExecutor, as_completed from datetime import datetime, timezone +from difflib import unified_diff from typing import Any import bmemcached @@ -20,12 +21,22 @@ from psycopg2.extras import execute_values from psycopg2.sql import SQL, Identifier from analytics.models import RealmCount, StreamCount, UserCount +from version import ZULIP_VERSION from zerver.actions.create_realm import set_default_for_realm_permission_group_settings from zerver.actions.realm_settings import do_change_realm_plan_type from zerver.actions.user_settings import do_change_avatar_fields from zerver.lib.avatar_hash import user_avatar_base_path_from_ids from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields -from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName +from zerver.lib.export import ( + DATE_FIELDS, + Field, + MigrationStatusJson, + Path, + Record, + TableData, + TableName, + get_migrations_by_app, +) from zerver.lib.markdown import markdown_convert from zerver.lib.markdown import version as markdown_version from zerver.lib.message import get_last_message_id @@ -1124,6 +1135,16 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int = 1) -> Rea if not os.path.exists(import_dir): raise Exception("Missing import directory!") + migration_status_filename = os.path.join(import_dir, "migration_status.json") + if not os.path.exists(migration_status_filename): + raise Exception( + "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm." + ) + logging.info("Checking migration status of exported realm") + with open(migration_status_filename) as f: + migration_status: MigrationStatusJson = orjson.loads(f.read()) + check_migration_status(migration_status) + realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") @@ -2080,3 +2101,72 @@ def add_users_to_system_user_groups( ) for membership in usergroup_memberships ) + + +ZULIP_CLOUD_ONLY_APP_NAMES = ["zilencer", "corporate"] + + +def check_migration_status(exported_migration_status: MigrationStatusJson) -> None: + mismatched_migrations_log: dict[str, str] = {} + local_migration_status = MigrationStatusJson( + migrations_by_app=get_migrations_by_app(), zulip_version=ZULIP_VERSION + ) + + # Different major versions are the most common form of mismatch + # and should get a nice error message focused on that, not + # migrations. + # + # We could split on `-`, to get the maintenance release version, + # but unless migrations are different, it should generally be safe + # to import across minor release differences. + exported_primary_version = exported_migration_status["zulip_version"].split(".")[0] + local_primary_version = local_migration_status["zulip_version"].split(".")[0] + if exported_primary_version != local_primary_version: + raise Exception( + "Export was generated on a different Zulip major version.\n" + f"Export={exported_migration_status['zulip_version']}\n" + f"Server={local_migration_status['zulip_version']}" + ) + exported_migrations_by_app = exported_migration_status["migrations_by_app"] + local_migrations_by_app = local_migration_status["migrations_by_app"] + all_apps = set(exported_migrations_by_app.keys()).union(set(local_migrations_by_app.keys())) + + for app in all_apps: + exported_app_migrations = exported_migrations_by_app.get(app) + local_app_migrations = local_migrations_by_app.get(app) + + if app in ZULIP_CLOUD_ONLY_APP_NAMES and ( + local_app_migrations is None or exported_app_migrations is None + ): + # This applications are expected to be present only on + # Zulip Cloud, so don't warn about them. + continue + + if not exported_app_migrations: + logging.warning("This server has '%s' app installed, but exported realm does not.", app) + elif not local_app_migrations: + logging.warning("Exported realm has '%s' app installed, but this server does not.", app) + elif local_app_migrations != exported_app_migrations: + diff = list( + unified_diff(exported_app_migrations, local_app_migrations, lineterm="", n=1) + ) + mismatched_migrations_log[f"\n'{app}' app:\n"] = "\n".join(diff[3:]) + + if mismatched_migrations_log: + # The order of the list output by the diff is nondeterministic, so the + # error logs needs to be sorted first. + sorted_error_log: list[str] = [ + f"{key}{value}" for key, value in sorted(mismatched_migrations_log.items()) + ] + + error_message = ( + "Export was generated on a different Zulip version.\n" + f"Export={exported_migration_status['zulip_version']}\n" + f"Server={local_migration_status['zulip_version']}\n" + "\n" + "Database formats differ between the exported realm and this server.\n" + "Printing migrations that differ between the versions:\n" + "--- exported realm\n" + "+++ this server" + ) + "\n".join(sorted_error_log) + raise Exception(error_message) diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json new file mode 100644 index 0000000000..966beee297 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json @@ -0,0 +1,65 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length", + "0003_alter_user_email_max_length", + "0004_alter_user_username_opts" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain", + "0030_realm_org_type", + "0031_remove_system_avatar_source" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "phonenumber": ["0001_initial", "0001_squashed_0001_initial"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "sessions": ["0001_initial"], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json new file mode 100644 index 0000000000..9f28d3effa --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json @@ -0,0 +1,63 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length", + "0003_alter_user_email_max_length", + "0004_alter_user_username_opts" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain", + "0030_realm_org_type", + "0031_remove_system_avatar_source" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json new file mode 100644 index 0000000000..25753b4bf2 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json @@ -0,0 +1,61 @@ +{ + "contenttypes": ["0001_initial", "0002_remove_content_type_name"], + "auth": [ + "0001_initial", + "0002_alter_permission_name_max_length" + ], + "zerver": [ + "0001_initial", + "0029_realm_subdomain" + ], + "analytics": [ + "0001_initial", + "0002_remove_huddlecount", + "0003_fillstate" + ], + "confirmation": [ + "0001_initial", + "0002_realmcreationkey", + "0003_emailchangeconfirmation" + ], + "zilencer": [ + "0001_initial", + "0002_remote_zulip_server", + "0003_add_default_for_remotezulipserver_last_updated_field" + ], + "corporate": [ + "0001_initial", + "0002_customer_default_discount", + "0003_customerplan" + ], + "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"], + "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"], + "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"], + "phonenumber": ["0001_initial", "0001_squashed_0001_initial"], + "two_factor": [ + "0001_initial", + "0002_auto_20150110_0810", + "0003_auto_20150817_1733", + "0004_auto_20160205_1827" + ], + "sessions": ["0001_initial"], + "default": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400" + ], + "social_auth": [ + "0001_initial", + "0002_add_related_name", + "0003_alter_email_max_length", + "0004_auto_20160423_0400", + "0005_auto_20160727_2333" + ], + "social_django": [ + "0006_partial", + "0007_code_timestamp", + "0008_partial_timestamp", + "0009_auto_20191118_0520" + ] +} diff --git a/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt new file mode 100644 index 0000000000..f3d9fb7481 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt @@ -0,0 +1,17 @@ +Export was generated on a different Zulip version. +Export=10.0-dev+git +Server=10.0-dev+git + +Database formats differ between the exported realm and this server. +Printing migrations that differ between the versions: +--- exported realm ++++ this server +'auth' app: + 0002_alter_permission_name_max_length +-0003_alter_user_email_max_length +-0004_alter_user_username_opts + +'zerver' app: + 0029_realm_subdomain +-0030_realm_org_type +-0031_remove_system_avatar_source diff --git a/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt new file mode 100644 index 0000000000..c00cec3229 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt @@ -0,0 +1,17 @@ +Export was generated on a different Zulip version. +Export=10.0-dev+git +Server=10.0-dev+git + +Database formats differ between the exported realm and this server. +Printing migrations that differ between the versions: +--- exported realm ++++ this server +'auth' app: + 0002_alter_permission_name_max_length ++0003_alter_user_email_max_length ++0004_alter_user_username_opts + +'zerver' app: + 0029_realm_subdomain ++0030_realm_org_type ++0031_remove_system_avatar_source diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 3b71d30e9d..d0910a3dcd 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -16,6 +16,7 @@ from django.utils.timezone import now as timezone_now from typing_extensions import override from analytics.models import UserCount +from version import ZULIP_VERSION from zerver.actions.alert_words import do_add_alert_words from zerver.actions.create_user import do_create_user from zerver.actions.custom_profile_fields import ( @@ -43,6 +44,7 @@ from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.bot_config import set_bot_config from zerver.lib.bot_lib import StateHandler from zerver.lib.export import ( + AppMigrations, MigrationStatusJson, Record, do_export_realm, @@ -340,6 +342,12 @@ class ExportFile(ZulipTestCase): db_paths = {user_avatar_path(user) + ".original"} self.assertEqual(exported_paths, db_paths) + def get_applied_migrations_fixture(self, fixture_name: str) -> AppMigrations: + fixture = orjson.loads( + self.fixture_data(fixture_name, "import_fixtures/applied_migrations_fixtures") + ) + return fixture + def verify_migration_status_json(self) -> None: # This function asserts that the generated migration_status.json # is structurally familiar for it to be used for assertion at @@ -2032,6 +2040,174 @@ class RealmImportExportTest(ExportFile): expected_group_names.add(SystemGroups.FULL_MEMBERS) self.assertSetEqual(logged_membership_by_user_id[user.id], expected_group_names) + def test_import_realm_with_unapplied_migrations(self) -> None: + realm = get_realm("zulip") + with ( + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_unapplied_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self.export_realm( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + + expected_error_message = self.fixture_data( + "unapplied_migrations_error.txt", "import_fixtures/check_migrations_errors" + ).strip() + error_message = str(e.exception).strip() + self.assertEqual(expected_error_message, error_message) + + def test_import_realm_with_extra_migrations(self) -> None: + realm = get_realm("zulip") + with ( + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture( + "with_unapplied_migrations.json" + ) + self.export_realm( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + expected_error_message = self.fixture_data( + "extra_migrations_error.txt", "import_fixtures/check_migrations_errors" + ).strip() + error_message = str(e.exception).strip() + self.assertEqual(expected_error_message, error_message) + + def test_import_realm_with_extra_exported_apps(self) -> None: + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="WARNING") as mock_log, + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + mock_import.return_value = self.get_applied_migrations_fixture("with_missing_apps.json") + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + missing_apps_log = [ + "WARNING:root:Exported realm has 'phonenumber' app installed, but this server does not.", + "WARNING:root:Exported realm has 'sessions' app installed, but this server does not.", + ] + # The log output is sorted because it's order is nondeterministic. + self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log)) + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_import_realm_with_missing_apps(self) -> None: + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="WARNING") as mock_log, + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture("with_missing_apps.json") + mock_import.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + missing_apps_log = [ + "WARNING:root:This server has 'phonenumber' app installed, but exported realm does not.", + "WARNING:root:This server has 'sessions' app installed, but exported realm does not.", + ] + self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log)) + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_check_migration_for_zulip_cloud_realm(self) -> None: + # This test ensures that `check_migrations_status` correctly handles + # checking the migrations of a Zulip Cloud-like realm (with zilencer/ + # corporate apps installed) when importing into a self-hosted realm + # (where these apps are not installed). + realm = get_realm("zulip") + with ( + self.settings(BILLING_ENABLED=False), + self.assertLogs(level="INFO"), + patch("zerver.lib.export.get_migrations_by_app") as mock_export, + patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import, + ): + mock_export.return_value = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + self_hosted_migrations = self.get_applied_migrations_fixture( + "with_complete_migrations.json" + ) + for key in ["zilencer", "corporate"]: + self_hosted_migrations.pop(key, None) + mock_import.return_value = self_hosted_migrations + self.export_realm_and_create_auditlog( + realm, + export_type=RealmExport.EXPORT_FULL_WITH_CONSENT, + ) + do_import_realm(get_output_dir(), "test-zulip") + + self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists()) + imported_realm = Realm.objects.get(string_id="test-zulip") + self.assertNotEqual(imported_realm.id, realm.id) + + def test_import_realm_without_migration_status_file(self) -> None: + realm = get_realm("zulip") + with patch("zerver.lib.export.export_migration_status"): + self.export_realm_and_create_auditlog(realm) + + with self.assertRaises(Exception) as e, self.assertLogs(level="INFO"): + do_import_realm( + get_output_dir(), + "test-zulip", + ) + expected_error_message = "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm." + self.assertEqual(expected_error_message, str(e.exception)) + + def test_import_realm_with_different_stated_zulip_version(self) -> None: + realm = get_realm("zulip") + self.export_realm_and_create_auditlog(realm) + + with ( + patch("zerver.lib.import_realm.ZULIP_VERSION", "8.0"), + self.assertRaises(Exception) as e, + self.assertLogs(level="INFO"), + ): + do_import_realm( + get_output_dir(), + "test-zulip", + ) + expected_error_message = ( + "Export was generated on a different Zulip major version.\n" + f"Export={ZULIP_VERSION}\n" + "Server=8.0" + ) + self.assertEqual(expected_error_message, str(e.exception)) + class SingleUserExportTest(ExportFile): def do_files_test(self, is_s3: bool) -> None: