Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

import: Verify exported realm's migration compatibility. 

When transferring a realm to a server that has a different set of
applied migrations (different Zulip versions), there is a chance that
the imported data formats appear to be compatible but data invariants
could still be violated.

This commit adds an assertion during the import process to verify
that both the exported realm and the importing server have matching
Zulip versions and have a compatible set of migrations.
This commit is contained in:
PieterCK 2024-09-19 22:54:50 +07:00 committed by Tim Abbott
parent 40bcb4b42b
commit a9838d8089
7 changed files with 490 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
zerver/lib/import_realm.py
View File

@ -4,6 +4,7 @@ import os
import shutil
from concurrent.futures import ProcessPoolExecutor, as_completed
from datetime import datetime, timezone
from difflib import unified_diff
from typing import Any
import bmemcached
@ -20,12 +21,22 @@ from psycopg2.extras import execute_values
from psycopg2.sql import SQL, Identifier
from analytics.models import RealmCount, StreamCount, UserCount
from version import ZULIP_VERSION
from zerver.actions.create_realm import set_default_for_realm_permission_group_settings
from zerver.actions.realm_settings import do_change_realm_plan_type
from zerver.actions.user_settings import do_change_avatar_fields
from zerver.lib.avatar_hash import user_avatar_base_path_from_ids
from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields
from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName
from zerver.lib.export import (
DATE_FIELDS,
Field,
MigrationStatusJson,
Path,
Record,
TableData,
TableName,
get_migrations_by_app,
)
from zerver.lib.markdown import markdown_convert
from zerver.lib.markdown import version as markdown_version
from zerver.lib.message import get_last_message_id
@ -1124,6 +1135,16 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int = 1) -> Rea
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
migration_status_filename = os.path.join(import_dir, "migration_status.json")
if not os.path.exists(migration_status_filename):
raise Exception(
"Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm."
)
logging.info("Checking migration status of exported realm")
with open(migration_status_filename) as f:
migration_status: MigrationStatusJson = orjson.loads(f.read())
check_migration_status(migration_status)
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
@ -2080,3 +2101,72 @@ def add_users_to_system_user_groups(
)
for membership in usergroup_memberships
)
ZULIP_CLOUD_ONLY_APP_NAMES = ["zilencer", "corporate"]
def check_migration_status(exported_migration_status: MigrationStatusJson) -> None:
mismatched_migrations_log: dict[str, str] = {}
local_migration_status = MigrationStatusJson(
migrations_by_app=get_migrations_by_app(), zulip_version=ZULIP_VERSION
)
# Different major versions are the most common form of mismatch
# and should get a nice error message focused on that, not
# migrations.
#
# We could split on `-`, to get the maintenance release version,
# but unless migrations are different, it should generally be safe
# to import across minor release differences.
exported_primary_version = exported_migration_status["zulip_version"].split(".")[0]
local_primary_version = local_migration_status["zulip_version"].split(".")[0]
if exported_primary_version != local_primary_version:
raise Exception(
"Export was generated on a different Zulip major version.\n"
f"Export={exported_migration_status['zulip_version']}\n"
f"Server={local_migration_status['zulip_version']}"
)
exported_migrations_by_app = exported_migration_status["migrations_by_app"]
local_migrations_by_app = local_migration_status["migrations_by_app"]
all_apps = set(exported_migrations_by_app.keys()).union(set(local_migrations_by_app.keys()))
for app in all_apps:
exported_app_migrations = exported_migrations_by_app.get(app)
local_app_migrations = local_migrations_by_app.get(app)
if app in ZULIP_CLOUD_ONLY_APP_NAMES and (
local_app_migrations is None or exported_app_migrations is None
):
# This applications are expected to be present only on
# Zulip Cloud, so don't warn about them.
continue
if not exported_app_migrations:
logging.warning("This server has '%s' app installed, but exported realm does not.", app)
elif not local_app_migrations:
logging.warning("Exported realm has '%s' app installed, but this server does not.", app)
elif local_app_migrations != exported_app_migrations:
diff = list(
unified_diff(exported_app_migrations, local_app_migrations, lineterm="", n=1)
)
mismatched_migrations_log[f"\n'{app}' app:\n"] = "\n".join(diff[3:])
if mismatched_migrations_log:
# The order of the list output by the diff is nondeterministic, so the
# error logs needs to be sorted first.
sorted_error_log: list[str] = [
f"{key}{value}" for key, value in sorted(mismatched_migrations_log.items())
]
error_message = (
"Export was generated on a different Zulip version.\n"
f"Export={exported_migration_status['zulip_version']}\n"
f"Server={local_migration_status['zulip_version']}\n"
"\n"
"Database formats differ between the exported realm and this server.\n"
"Printing migrations that differ between the versions:\n"
"--- exported realm\n"
"+++ this server"
) + "\n".join(sorted_error_log)
raise Exception(error_message)

65
zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_complete_migrations.json vendored Normal file
View File

@ -0,0 +1,65 @@
{
"contenttypes": ["0001_initial", "0002_remove_content_type_name"],
"auth": [
"0001_initial",
"0002_alter_permission_name_max_length",
"0003_alter_user_email_max_length",
"0004_alter_user_username_opts"
],
"zerver": [
"0001_initial",
"0029_realm_subdomain",
"0030_realm_org_type",
"0031_remove_system_avatar_source"
],
"analytics": [
"0001_initial",
"0002_remove_huddlecount",
"0003_fillstate"
],
"confirmation": [
"0001_initial",
"0002_realmcreationkey",
"0003_emailchangeconfirmation"
],
"zilencer": [
"0001_initial",
"0002_remote_zulip_server",
"0003_add_default_for_remotezulipserver_last_updated_field"
],
"corporate": [
"0001_initial",
"0002_customer_default_discount",
"0003_customerplan"
],
"otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
"otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
"pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
"phonenumber": ["0001_initial", "0001_squashed_0001_initial"],
"two_factor": [
"0001_initial",
"0002_auto_20150110_0810",
"0003_auto_20150817_1733",
"0004_auto_20160205_1827"
],
"sessions": ["0001_initial"],
"default": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400"
],
"social_auth": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400",
"0005_auto_20160727_2333"
],
"social_django": [
"0006_partial",
"0007_code_timestamp",
"0008_partial_timestamp",
"0009_auto_20191118_0520"
]
}

63
zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_missing_apps.json vendored Normal file
View File

@ -0,0 +1,63 @@
{
"contenttypes": ["0001_initial", "0002_remove_content_type_name"],
"auth": [
"0001_initial",
"0002_alter_permission_name_max_length",
"0003_alter_user_email_max_length",
"0004_alter_user_username_opts"
],
"zerver": [
"0001_initial",
"0029_realm_subdomain",
"0030_realm_org_type",
"0031_remove_system_avatar_source"
],
"analytics": [
"0001_initial",
"0002_remove_huddlecount",
"0003_fillstate"
],
"confirmation": [
"0001_initial",
"0002_realmcreationkey",
"0003_emailchangeconfirmation"
],
"zilencer": [
"0001_initial",
"0002_remote_zulip_server",
"0003_add_default_for_remotezulipserver_last_updated_field"
],
"corporate": [
"0001_initial",
"0002_customer_default_discount",
"0003_customerplan"
],
"otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
"otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
"pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
"two_factor": [
"0001_initial",
"0002_auto_20150110_0810",
"0003_auto_20150817_1733",
"0004_auto_20160205_1827"
],
"default": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400"
],
"social_auth": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400",
"0005_auto_20160727_2333"
],
"social_django": [
"0006_partial",
"0007_code_timestamp",
"0008_partial_timestamp",
"0009_auto_20191118_0520"
]
}

61
zerver/tests/fixtures/import_fixtures/applied_migrations_fixtures/with_unapplied_migrations.json vendored Normal file
View File

@ -0,0 +1,61 @@
{
"contenttypes": ["0001_initial", "0002_remove_content_type_name"],
"auth": [
"0001_initial",
"0002_alter_permission_name_max_length"
],
"zerver": [
"0001_initial",
"0029_realm_subdomain"
],
"analytics": [
"0001_initial",
"0002_remove_huddlecount",
"0003_fillstate"
],
"confirmation": [
"0001_initial",
"0002_realmcreationkey",
"0003_emailchangeconfirmation"
],
"zilencer": [
"0001_initial",
"0002_remote_zulip_server",
"0003_add_default_for_remotezulipserver_last_updated_field"
],
"corporate": [
"0001_initial",
"0002_customer_default_discount",
"0003_customerplan"
],
"otp_static": ["0001_initial", "0002_throttling", "0003_add_timestamps"],
"otp_totp": ["0001_initial", "0002_auto_20190420_0723", "0003_add_timestamps"],
"pgroonga": ["0001_enable", "0002_html_escape_subject", "0003_v2_api_upgrade"],
"phonenumber": ["0001_initial", "0001_squashed_0001_initial"],
"two_factor": [
"0001_initial",
"0002_auto_20150110_0810",
"0003_auto_20150817_1733",
"0004_auto_20160205_1827"
],
"sessions": ["0001_initial"],
"default": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400"
],
"social_auth": [
"0001_initial",
"0002_add_related_name",
"0003_alter_email_max_length",
"0004_auto_20160423_0400",
"0005_auto_20160727_2333"
],
"social_django": [
"0006_partial",
"0007_code_timestamp",
"0008_partial_timestamp",
"0009_auto_20191118_0520"
]
}

17
zerver/tests/fixtures/import_fixtures/check_migrations_errors/extra_migrations_error.txt vendored Normal file
View File

@ -0,0 +1,17 @@
Export was generated on a different Zulip version.
Export=10.0-dev+git
Server=10.0-dev+git
Database formats differ between the exported realm and this server.
Printing migrations that differ between the versions:
--- exported realm
+++ this server
'auth' app:
0002_alter_permission_name_max_length
-0003_alter_user_email_max_length
-0004_alter_user_username_opts
'zerver' app:
0029_realm_subdomain
-0030_realm_org_type
-0031_remove_system_avatar_source

17
zerver/tests/fixtures/import_fixtures/check_migrations_errors/unapplied_migrations_error.txt vendored Normal file
View File

@ -0,0 +1,17 @@
Export was generated on a different Zulip version.
Export=10.0-dev+git
Server=10.0-dev+git
Database formats differ between the exported realm and this server.
Printing migrations that differ between the versions:
--- exported realm
+++ this server
'auth' app:
0002_alter_permission_name_max_length
+0003_alter_user_email_max_length
+0004_alter_user_username_opts
'zerver' app:
0029_realm_subdomain
+0030_realm_org_type
+0031_remove_system_avatar_source

176
zerver/tests/test_import_export.py
View File

@ -16,6 +16,7 @@ from django.utils.timezone import now as timezone_now
from typing_extensions import override
from analytics.models import UserCount
from version import ZULIP_VERSION
from zerver.actions.alert_words import do_add_alert_words
from zerver.actions.create_user import do_create_user
from zerver.actions.custom_profile_fields import (
@ -43,6 +44,7 @@ from zerver.lib.avatar_hash import user_avatar_path
from zerver.lib.bot_config import set_bot_config
from zerver.lib.bot_lib import StateHandler
from zerver.lib.export import (
AppMigrations,
MigrationStatusJson,
Record,
do_export_realm,
@ -340,6 +342,12 @@ class ExportFile(ZulipTestCase):
db_paths = {user_avatar_path(user) + ".original"}
self.assertEqual(exported_paths, db_paths)
def get_applied_migrations_fixture(self, fixture_name: str) -> AppMigrations:
fixture = orjson.loads(
self.fixture_data(fixture_name, "import_fixtures/applied_migrations_fixtures")
)
return fixture
def verify_migration_status_json(self) -> None:
# This function asserts that the generated migration_status.json
# is structurally familiar for it to be used for assertion at
@ -2032,6 +2040,174 @@ class RealmImportExportTest(ExportFile):
expected_group_names.add(SystemGroups.FULL_MEMBERS)
self.assertSetEqual(logged_membership_by_user_id[user.id], expected_group_names)
def test_import_realm_with_unapplied_migrations(self) -> None:
realm = get_realm("zulip")
with (
self.assertRaises(Exception) as e,
self.assertLogs(level="INFO"),
patch("zerver.lib.export.get_migrations_by_app") as mock_export,
patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
):
mock_export.return_value = self.get_applied_migrations_fixture(
"with_unapplied_migrations.json"
)
mock_import.return_value = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
self.export_realm(
realm,
export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
)
do_import_realm(get_output_dir(), "test-zulip")
expected_error_message = self.fixture_data(
"unapplied_migrations_error.txt", "import_fixtures/check_migrations_errors"
).strip()
error_message = str(e.exception).strip()
self.assertEqual(expected_error_message, error_message)
def test_import_realm_with_extra_migrations(self) -> None:
realm = get_realm("zulip")
with (
self.assertRaises(Exception) as e,
self.assertLogs(level="INFO"),
patch("zerver.lib.export.get_migrations_by_app") as mock_export,
patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
):
mock_export.return_value = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
mock_import.return_value = self.get_applied_migrations_fixture(
"with_unapplied_migrations.json"
)
self.export_realm(
realm,
export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
)
do_import_realm(get_output_dir(), "test-zulip")
expected_error_message = self.fixture_data(
"extra_migrations_error.txt", "import_fixtures/check_migrations_errors"
).strip()
error_message = str(e.exception).strip()
self.assertEqual(expected_error_message, error_message)
def test_import_realm_with_extra_exported_apps(self) -> None:
realm = get_realm("zulip")
with (
self.settings(BILLING_ENABLED=False),
self.assertLogs(level="WARNING") as mock_log,
patch("zerver.lib.export.get_migrations_by_app") as mock_export,
patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
):
mock_export.return_value = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
mock_import.return_value = self.get_applied_migrations_fixture("with_missing_apps.json")
self.export_realm_and_create_auditlog(
realm,
export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
)
do_import_realm(get_output_dir(), "test-zulip")
missing_apps_log = [
"WARNING:root:Exported realm has 'phonenumber' app installed, but this server does not.",
"WARNING:root:Exported realm has 'sessions' app installed, but this server does not.",
]
# The log output is sorted because it's order is nondeterministic.
self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log))
self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
imported_realm = Realm.objects.get(string_id="test-zulip")
self.assertNotEqual(imported_realm.id, realm.id)
def test_import_realm_with_missing_apps(self) -> None:
realm = get_realm("zulip")
with (
self.settings(BILLING_ENABLED=False),
self.assertLogs(level="WARNING") as mock_log,
patch("zerver.lib.export.get_migrations_by_app") as mock_export,
patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
):
mock_export.return_value = self.get_applied_migrations_fixture("with_missing_apps.json")
mock_import.return_value = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
self.export_realm_and_create_auditlog(
realm,
export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
)
do_import_realm(get_output_dir(), "test-zulip")
missing_apps_log = [
"WARNING:root:This server has 'phonenumber' app installed, but exported realm does not.",
"WARNING:root:This server has 'sessions' app installed, but exported realm does not.",
]
self.assertEqual(sorted(mock_log.output), sorted(missing_apps_log))
self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
imported_realm = Realm.objects.get(string_id="test-zulip")
self.assertNotEqual(imported_realm.id, realm.id)
def test_check_migration_for_zulip_cloud_realm(self) -> None:
# This test ensures that `check_migrations_status` correctly handles
# checking the migrations of a Zulip Cloud-like realm (with zilencer/
# corporate apps installed) when importing into a self-hosted realm
# (where these apps are not installed).
realm = get_realm("zulip")
with (
self.settings(BILLING_ENABLED=False),
self.assertLogs(level="INFO"),
patch("zerver.lib.export.get_migrations_by_app") as mock_export,
patch("zerver.lib.import_realm.get_migrations_by_app") as mock_import,
):
mock_export.return_value = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
self_hosted_migrations = self.get_applied_migrations_fixture(
"with_complete_migrations.json"
)
for key in ["zilencer", "corporate"]:
self_hosted_migrations.pop(key, None)
mock_import.return_value = self_hosted_migrations
self.export_realm_and_create_auditlog(
realm,
export_type=RealmExport.EXPORT_FULL_WITH_CONSENT,
)
do_import_realm(get_output_dir(), "test-zulip")
self.assertTrue(Realm.objects.filter(string_id="test-zulip").exists())
imported_realm = Realm.objects.get(string_id="test-zulip")
self.assertNotEqual(imported_realm.id, realm.id)
def test_import_realm_without_migration_status_file(self) -> None:
realm = get_realm("zulip")
with patch("zerver.lib.export.export_migration_status"):
self.export_realm_and_create_auditlog(realm)
with self.assertRaises(Exception) as e, self.assertLogs(level="INFO"):
do_import_realm(
get_output_dir(),
"test-zulip",
)
expected_error_message = "Missing migration_status.json file! Make sure you're using the same Zulip version as the exported realm."
self.assertEqual(expected_error_message, str(e.exception))
def test_import_realm_with_different_stated_zulip_version(self) -> None:
realm = get_realm("zulip")
self.export_realm_and_create_auditlog(realm)
with (
patch("zerver.lib.import_realm.ZULIP_VERSION", "8.0"),
self.assertRaises(Exception) as e,
self.assertLogs(level="INFO"),
):
do_import_realm(
get_output_dir(),
"test-zulip",
)
expected_error_message = (
"Export was generated on a different Zulip major version.\n"
f"Export={ZULIP_VERSION}\n"
"Server=8.0"
)
self.assertEqual(expected_error_message, str(e.exception))
class SingleUserExportTest(ExportFile):
def do_files_test(self, is_s3: bool) -> None: