import: Verify exported realm's migration compatibility.

When transferring a realm to a server that has a different set of
applied migrations (different Zulip versions), there is a chance that
the imported data formats appear to be compatible but data invariants
could still be violated.

This commit adds an assertion during the import process to verify
that both the exported realm and the importing server have matching
Zulip versions and have a compatible set of migrations.

zerver/lib/import_realm.py

@@ -4,6 +4,7 @@ import os
 import shutil
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from datetime import datetime, timezone
+from difflib import unified_diff
 from typing import Any
 
 import bmemcached
@@ -20,12 +21,22 @@ from psycopg2.extras import execute_values
 from psycopg2.sql import SQL, Identifier
 
 from analytics.models import RealmCount, StreamCount, UserCount
+from version import ZULIP_VERSION
 from zerver.actions.create_realm import set_default_for_realm_permission_group_settings
 from zerver.actions.realm_settings import do_change_realm_plan_type
 from zerver.actions.user_settings import do_change_avatar_fields
 from zerver.lib.avatar_hash import user_avatar_base_path_from_ids
 from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields
-from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName
+from zerver.lib.export import (
+    DATE_FIELDS,
+    Field,
+    MigrationStatusJson,
+    Path,
+    Record,
+    TableData,
+    TableName,
+    get_migrations_by_app,
+)
 from zerver.lib.markdown import markdown_convert
 from zerver.lib.markdown import version as markdown_version
 from zerver.lib.message import get_last_message_id
@@ -1124,6 +1135,16 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int = 1) -> Rea
     if not os.path.exists(import_dir):
         raise Exception("Missing import directory!")
 
+    migration_status_filename = os.path.join(import_dir, "migration_status.json")
+    if not os.path.exists(migration_status_filename):
+        raise Exception(
+            "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm."
+        )
+    logging.info("Checking migration status of exported realm")
+    with open(migration_status_filename) as f:
+        migration_status: MigrationStatusJson = orjson.loads(f.read())
+    check_migration_status(migration_status)
+
     realm_data_filename = os.path.join(import_dir, "realm.json")
     if not os.path.exists(realm_data_filename):
         raise Exception("Missing realm.json file!")
@@ -2080,3 +2101,72 @@ def add_users_to_system_user_groups(
         )
         for membership in usergroup_memberships
     )
+
+
+ZULIP_CLOUD_ONLY_APP_NAMES = ["zilencer", "corporate"]
+
+
+def check_migration_status(exported_migration_status: MigrationStatusJson) -> None:
+    mismatched_migrations_log: dict[str, str] = {}
+    local_migration_status = MigrationStatusJson(
+        migrations_by_app=get_migrations_by_app(), zulip_version=ZULIP_VERSION
+    )
+
+    # Different major versions are the most common form of mismatch
+    # and should get a nice error message focused on that, not
+    # migrations.
+    #
+    # We could split on `-`, to get the maintenance release version,
+    # but unless migrations are different, it should generally be safe
+    # to import across minor release differences.
+    exported_primary_version = exported_migration_status["zulip_version"].split(".")[0]
+    local_primary_version = local_migration_status["zulip_version"].split(".")[0]
+    if exported_primary_version != local_primary_version:
+        raise Exception(
+            "Export was generated on a different Zulip major version.\n"
+            f"Export={exported_migration_status['zulip_version']}\n"
+            f"Server={local_migration_status['zulip_version']}"
+        )
+    exported_migrations_by_app = exported_migration_status["migrations_by_app"]
+    local_migrations_by_app = local_migration_status["migrations_by_app"]
+    all_apps = set(exported_migrations_by_app.keys()).union(set(local_migrations_by_app.keys()))
+
+    for app in all_apps:
+        exported_app_migrations = exported_migrations_by_app.get(app)
+        local_app_migrations = local_migrations_by_app.get(app)
+
+        if app in ZULIP_CLOUD_ONLY_APP_NAMES and (
+            local_app_migrations is None or exported_app_migrations is None
+        ):
+            # This applications are expected to be present only on
+            # Zulip Cloud, so don't warn about them.
+            continue
+
+        if not exported_app_migrations:
+            logging.warning("This server has '%s' app installed, but exported realm does not.", app)
+        elif not local_app_migrations:
+            logging.warning("Exported realm has '%s' app installed, but this server does not.", app)
+        elif local_app_migrations != exported_app_migrations:
+            diff = list(
+                unified_diff(exported_app_migrations, local_app_migrations, lineterm="", n=1)
+            )
+            mismatched_migrations_log[f"\n'{app}' app:\n"] = "\n".join(diff[3:])
+
+    if mismatched_migrations_log:
+        # The order of the list output by the diff is nondeterministic, so the
+        # error logs needs to be sorted first.
+        sorted_error_log: list[str] = [
+            f"{key}{value}" for key, value in sorted(mismatched_migrations_log.items())
+        ]
+
+        error_message = (
+            "Export was generated on a different Zulip version.\n"
+            f"Export={exported_migration_status['zulip_version']}\n"
+            f"Server={local_migration_status['zulip_version']}\n"
+            "\n"
+            "Database formats differ between the exported realm and this server.\n"
+            "Printing migrations that differ between the versions:\n"
+            "--- exported realm\n"
+            "+++ this server"
+        ) + "\n".join(sorted_error_log)
+        raise Exception(error_message)

zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json

@@ -0,0 +1,65 @@
+{
+    "contenttypes": ["0001_initial", "0002_remove_content_type_name"],
+    "auth": [
+        "0001_initial",
+        "0002_alter_permission_name_max_length",
+        "0003_alter_user_email_max_length",
+        "0004_alter_user_username_opts"
+    ],
+    "zerver": [
+        "0001_initial",
+        "0029_realm_subdomain",
+        "0030_realm_org_type",
+        "0031_remove_system_avatar_source"
+    ],
+    "analytics": [
+        "0001_initial",
+        "0002_remove_huddlecount",
+        "0003_fillstate"
+    ],
+    "confirmation": [
+        "0001_initial",
+        "0002_realmcreationkey",
+        "0003_emailchangeconfirmation"
+    ],
+    "zilencer": [
+        "0001_initial",
+        "0002_remote_zulip_server",
+        "0003_add_default_for_remotezulipserver_last_updated_field"
+    ],
+    "corporate": [
+        "0001_initial",
+        "0002_customer_default_discount",
+        "0003_customerplan"
+    ],
+    "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
+    "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
+    "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
+    "phonenumber": ["0001_initial", "0001_squashed_0001_initial"],
+    "two_factor": [
+        "0001_initial",
+        "0002_auto_20150110_0810",
+        "0003_auto_20150817_1733",
+        "0004_auto_20160205_1827"
+    ],
+    "sessions": ["0001_initial"],
+    "default": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400"
+    ],
+    "social_auth": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400",
+        "0005_auto_20160727_2333"
+    ],
+    "social_django": [
+        "0006_partial",
+        "0007_code_timestamp",
+        "0008_partial_timestamp",
+        "0009_auto_20191118_0520"
+    ]
+}

zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json

@@ -0,0 +1,63 @@
+{
+    "contenttypes": ["0001_initial", "0002_remove_content_type_name"],
+    "auth": [
+        "0001_initial",
+        "0002_alter_permission_name_max_length",
+        "0003_alter_user_email_max_length",
+        "0004_alter_user_username_opts"
+    ],
+    "zerver": [
+        "0001_initial",
+        "0029_realm_subdomain",
+        "0030_realm_org_type",
+        "0031_remove_system_avatar_source"
+    ],
+    "analytics": [
+        "0001_initial",
+        "0002_remove_huddlecount",
+        "0003_fillstate"
+    ],
+    "confirmation": [
+        "0001_initial",
+        "0002_realmcreationkey",
+        "0003_emailchangeconfirmation"
+    ],
+    "zilencer": [
+        "0001_initial",
+        "0002_remote_zulip_server",
+        "0003_add_default_for_remotezulipserver_last_updated_field"
+    ],
+    "corporate": [
+        "0001_initial",
+        "0002_customer_default_discount",
+        "0003_customerplan"
+    ],
+    "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
+    "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
+    "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
+    "two_factor": [
+        "0001_initial",
+        "0002_auto_20150110_0810",
+        "0003_auto_20150817_1733",
+        "0004_auto_20160205_1827"
+    ],
+    "default": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400"
+    ],
+    "social_auth": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400",
+        "0005_auto_20160727_2333"
+    ],
+    "social_django": [
+        "0006_partial",
+        "0007_code_timestamp",
+        "0008_partial_timestamp",
+        "0009_auto_20191118_0520"
+    ]
+}

zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json

@@ -0,0 +1,61 @@
+{
+    "contenttypes": ["0001_initial", "0002_remove_content_type_name"],
+    "auth": [
+        "0001_initial",
+        "0002_alter_permission_name_max_length"
+    ],
+    "zerver": [
+        "0001_initial",
+        "0029_realm_subdomain"
+    ],
+    "analytics": [
+        "0001_initial",
+        "0002_remove_huddlecount",
+        "0003_fillstate"
+    ],
+    "confirmation": [
+        "0001_initial",
+        "0002_realmcreationkey",
+        "0003_emailchangeconfirmation"
+    ],
+    "zilencer": [
+        "0001_initial",
+        "0002_remote_zulip_server",
+        "0003_add_default_for_remotezulipserver_last_updated_field"
+    ],
+    "corporate": [
+        "0001_initial",
+        "0002_customer_default_discount",
+        "0003_customerplan"
+    ],
+    "otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
+    "otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
+    "pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
+    "phonenumber": ["0001_initial", "0001_squashed_0001_initial"],
+    "two_factor": [
+        "0001_initial",
+        "0002_auto_20150110_0810",
+        "0003_auto_20150817_1733",
+        "0004_auto_20160205_1827"
+    ],
+    "sessions": ["0001_initial"],
+    "default": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400"
+    ],
+    "social_auth": [
+        "0001_initial",
+        "0002_add_related_name",
+        "0003_alter_email_max_length",
+        "0004_auto_20160423_0400",
+        "0005_auto_20160727_2333"
+    ],
+    "social_django": [
+        "0006_partial",
+        "0007_code_timestamp",
+        "0008_partial_timestamp",
+        "0009_auto_20191118_0520"
+    ]
+}

zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt

@@ -0,0 +1,17 @@
+Export was generated on a different Zulip version.
+Export=10.0-dev+git
+Server=10.0-dev+git
+
+Database formats differ between the exported realm and this server.
+Printing migrations that differ between the versions:
+--- exported realm
++++ this server
+'auth' app:
+ 0002_alter_permission_name_max_length
+-0003_alter_user_email_max_length
+-0004_alter_user_username_opts
+
+'zerver' app:
+ 0029_realm_subdomain
+-0030_realm_org_type
+-0031_remove_system_avatar_source

zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt

@@ -0,0 +1,17 @@
+Export was generated on a different Zulip version.
+Export=10.0-dev+git
+Server=10.0-dev+git
+
+Database formats differ between the exported realm and this server.
+Printing migrations that differ between the versions:
+--- exported realm
++++ this server
+'auth' app:
+ 0002_alter_permission_name_max_length
++0003_alter_user_email_max_length
++0004_alter_user_username_opts
+
+'zerver' app:
+ 0029_realm_subdomain
++0030_realm_org_type
++0031_remove_system_avatar_source

zerver/tests/test_import_export.py

@@ -16,6 +16,7 @@ from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
 from analytics.models import UserCount
+from version import ZULIP_VERSION
 from zerver.actions.alert_words import do_add_alert_words
 from zerver.actions.create_user import do_create_user
 from zerver.actions.custom_profile_fields import (
@@ -43,6 +44,7 @@ from zerver.lib.avatar_hash import user_avatar_path
 from zerver.lib.bot_config import set_bot_config
 from zerver.lib.bot_lib import StateHandler
 from zerver.lib.export import (
+    AppMigrations,
     MigrationStatusJson,
     Record,
     do_export_realm,
@@ -340,6 +342,12 @@ class ExportFile(ZulipTestCase):
         db_paths = {user_avatar_path(user) + ".original"}
         self.assertEqual(exported_paths, db_paths)
 
+    def get_applied_migrations_fixture(self, fixture_name: str) -> AppMigrations:
+        fixture = orjson.loads(
+            self.fixture_data(fixture_name, "import_fixtures/applied_migrations_fixtures")
+        )
+        return fixture
+
     def verify_migration_status_json(self) -> None:
         # This function asserts that the generated migration_status.json
         # is structurally familiar for it to be used for assertion at
@@ -2032,6 +2040,174 @@ class RealmImportExportTest(ExportFile):
                 expected_group_names.add(SystemGroups.FULL_MEMBERS)
             self.assertSetEqual(logged_membership_by_user_id[user.id], expected_group_names)
 
+    def test_import_realm_with_unapplied_migrations(self) -> None:
+        realm = get_realm("zulip")
+        with (
+            self.assertRaises(Exception) as e,
+            self.assertLogs(level="INFO"),
+            patch("zerver.lib.export.get_migrations_by_app") as mock_export,
+            patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
+        ):
+            mock_export.return_value = self.get_applied_migrations_fixture(
+                "with_unapplied_migrations.json"
+            )
+            mock_import.return_value = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            self.export_realm(
+                realm,
+                export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
+            )
+            do_import_realm(get_output_dir(), "test-zulip")
+
+        expected_error_message = self.fixture_data(
+            "unapplied_migrations_error.txt", "import_fixtures/check_migrations_errors"
+        ).strip()
+        error_message = str(e.exception).strip()
+        self.assertEqual(expected_error_message, error_message)
+
+    def test_import_realm_with_extra_migrations(self) -> None:
+        realm = get_realm("zulip")
+        with (
+            self.assertRaises(Exception) as e,
+            self.assertLogs(level="INFO"),
+            patch("zerver.lib.export.get_migrations_by_app") as mock_export,
+            patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
+        ):
+            mock_export.return_value = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            mock_import.return_value = self.get_applied_migrations_fixture(
+                "with_unapplied_migrations.json"
+            )
+            self.export_realm(
+                realm,
+                export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
+            )
+            do_import_realm(get_output_dir(), "test-zulip")
+        expected_error_message = self.fixture_data(
+            "extra_migrations_error.txt", "import_fixtures/check_migrations_errors"
+        ).strip()
+        error_message = str(e.exception).strip()
+        self.assertEqual(expected_error_message, error_message)
+
+    def test_import_realm_with_extra_exported_apps(self) -> None:
+        realm = get_realm("zulip")
+        with (
+            self.settings(BILLING_ENABLED=False),
+            self.assertLogs(level="WARNING") as mock_log,
+            patch("zerver.lib.export.get_migrations_by_app") as mock_export,
+            patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
+        ):
+            mock_export.return_value = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            mock_import.return_value = self.get_applied_migrations_fixture("with_missing_apps.json")
+            self.export_realm_and_create_auditlog(
+                realm,
+                export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
+            )
+            do_import_realm(get_output_dir(), "test-zulip")
+        missing_apps_log = [
+            "WARNING:root:Exported realm has 'phonenumber' app installed, but this server does not.",
+            "WARNING:root:Exported realm has 'sessions' app installed, but this server does not.",
+        ]
+        # The log output is sorted because it's order is nondeterministic.
+        self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log))
+        self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
+        imported_realm = Realm.objects.get(string_id="test-zulip")
+        self.assertNotEqual(imported_realm.id, realm.id)
+
+    def test_import_realm_with_missing_apps(self) -> None:
+        realm = get_realm("zulip")
+        with (
+            self.settings(BILLING_ENABLED=False),
+            self.assertLogs(level="WARNING") as mock_log,
+            patch("zerver.lib.export.get_migrations_by_app") as mock_export,
+            patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
+        ):
+            mock_export.return_value = self.get_applied_migrations_fixture("with_missing_apps.json")
+            mock_import.return_value = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            self.export_realm_and_create_auditlog(
+                realm,
+                export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
+            )
+            do_import_realm(get_output_dir(), "test-zulip")
+        missing_apps_log = [
+            "WARNING:root:This server has 'phonenumber' app installed, but exported realm does not.",
+            "WARNING:root:This server has 'sessions' app installed, but exported realm does not.",
+        ]
+        self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log))
+        self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
+        imported_realm = Realm.objects.get(string_id="test-zulip")
+        self.assertNotEqual(imported_realm.id, realm.id)
+
+    def test_check_migration_for_zulip_cloud_realm(self) -> None:
+        # This test ensures that `check_migrations_status` correctly handles
+        # checking the migrations of a Zulip Cloud-like realm (with zilencer/
+        # corporate apps installed) when importing into a self-hosted realm
+        # (where these apps are not installed).
+        realm = get_realm("zulip")
+        with (
+            self.settings(BILLING_ENABLED=False),
+            self.assertLogs(level="INFO"),
+            patch("zerver.lib.export.get_migrations_by_app") as mock_export,
+            patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
+        ):
+            mock_export.return_value = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            self_hosted_migrations = self.get_applied_migrations_fixture(
+                "with_complete_migrations.json"
+            )
+            for key in ["zilencer", "corporate"]:
+                self_hosted_migrations.pop(key, None)
+            mock_import.return_value = self_hosted_migrations
+            self.export_realm_and_create_auditlog(
+                realm,
+                export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
+            )
+            do_import_realm(get_output_dir(), "test-zulip")
+
+        self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
+        imported_realm = Realm.objects.get(string_id="test-zulip")
+        self.assertNotEqual(imported_realm.id, realm.id)
+
+    def test_import_realm_without_migration_status_file(self) -> None:
+        realm = get_realm("zulip")
+        with patch("zerver.lib.export.export_migration_status"):
+            self.export_realm_and_create_auditlog(realm)
+
+        with self.assertRaises(Exception) as e, self.assertLogs(level="INFO"):
+            do_import_realm(
+                get_output_dir(),
+                "test-zulip",
+            )
+        expected_error_message = "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm."
+        self.assertEqual(expected_error_message, str(e.exception))
+
+    def test_import_realm_with_different_stated_zulip_version(self) -> None:
+        realm = get_realm("zulip")
+        self.export_realm_and_create_auditlog(realm)
+
+        with (
+            patch("zerver.lib.import_realm.ZULIP_VERSION", "8.0"),
+            self.assertRaises(Exception) as e,
+            self.assertLogs(level="INFO"),
+        ):
+            do_import_realm(
+                get_output_dir(),
+                "test-zulip",
+            )
+        expected_error_message = (
+            "Export was generated on a different Zulip major version.\n"
+            f"Export={ZULIP_VERSION}\n"
+            "Server=8.0"
+        )
+        self.assertEqual(expected_error_message, str(e.exception))
+
 
 class SingleUserExportTest(ExportFile):
     def do_files_test(self, is_s3: bool) -> None: