diff --git a/docs/overview/changelog.md b/docs/overview/changelog.md index cfda864a8c..b8babd0579 100644 --- a/docs/overview/changelog.md +++ b/docs/overview/changelog.md @@ -1431,9 +1431,8 @@ Zulip installations; it has minimal changes for existing servers. disruption by running this migration first, before beginning the user-facing downtime. However, if you'd like to watch the downtime phase of the upgrade closely, we recommend - [running them first manually](../production/expensive-migrations.md) - and as well as the usual trick of - doing an apt upgrade first. + [running them first manually](https://zulip.readthedocs.io/en/1.9.0/production/expensive-migrations.html) + as well as the usual trick of doing an apt upgrade first. #### Full feature changelog @@ -1821,9 +1820,8 @@ running a version from before 1.7 should upgrade directly to 1.7.1. minimizes disruption by running these first, before beginning the user-facing downtime. However, if you'd like to watch the downtime phase of the upgrade closely, we recommend - [running them first manually](../production/expensive-migrations.md) and as well - as the usual trick of - doing an apt upgrade first. + [running them first manually](https://zulip.readthedocs.io/en/1.9.0/production/expensive-migrations.html) + as well as the usual trick of doing an apt upgrade first. - We've removed support for an uncommon legacy deployment model where a Zulip server served multiple organizations on the same domain. diff --git a/docs/production/expensive-migrations.md b/docs/production/expensive-migrations.md deleted file mode 100644 index f8cfe83c06..0000000000 --- a/docs/production/expensive-migrations.md +++ /dev/null @@ -1,72 +0,0 @@ -```{eval-rst} -:orphan: -``` - -# Running expensive migrations early - -Zulip 1.7 and 1.9 each contain some significant database migrations -that can take several minutes to run. - -The upgrade process automatically minimizes disruption by running -these first, before beginning the user-facing downtime. However, if -you'd like to watch the downtime phase of the upgrade closely, you -can run them manually before starting the upgrade: - -1. Log in to your Zulip server as the `zulip` user (or as `root` and - then run `su zulip` to drop privileges), and - `cd /home/zulip/deployments/current` -2. Run `./manage.py dbshell`. This will open a shell connected to the - PostgreSQL database. -3. In the PostgreSQL shell, run the following commands: - - ```postgresql - CREATE INDEX CONCURRENTLY - zerver_usermessage_is_private_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 2048) != 0; - - CREATE INDEX CONCURRENTLY - zerver_usermessage_active_mobile_push_notification_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 4096) != 0; - ``` - - (These first migrations are the only new ones in Zulip 1.9). - - ```postgresql - CREATE INDEX CONCURRENTLY - zerver_usermessage_mentioned_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 8) != 0; - - CREATE INDEX CONCURRENTLY - zerver_usermessage_starred_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 2) != 0; - - CREATE INDEX CONCURRENTLY - zerver_usermessage_has_alert_word_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 512) != 0; - - CREATE INDEX CONCURRENTLY - zerver_usermessage_wildcard_mentioned_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 8) != 0 OR (flags & 16) != 0; - - CREATE INDEX CONCURRENTLY - zerver_usermessage_unread_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 1) = 0; - ``` - -These will take some time to run, during which the server will -continue to serve user traffic as usual with no disruption. Once they -finish, you can proceed with installing Zulip 1.7. - -To help you estimate how long these will take on your server: count -the number of UserMessage rows, with `select COUNT(*) from zerver_usermessage;` -at the `./manage.py dbshell` prompt. At the time these migrations -were run on chat.zulip.org, it had 75M UserMessage rows; the first 5 -indexes took about 1 minute each to create, and the final, -"unread_message" index took more like 10 minutes. diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index a563a7ce1c..5c3870feb0 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -293,15 +293,6 @@ else: preexec_fn=su_to_zulip, ) -usermessage_index_migrations = [ - "[ ] 0082_index_starred_user_messages", - "[ ] 0083_index_mentioned_user_messages", - "[ ] 0095_index_unread_user_messages", - "[ ] 0098_index_has_alert_word_user_messages", - "[ ] 0099_index_wildcard_mentioned_user_messages", - "[ ] 0177_user_message_add_and_index_is_private_flag", - "[ ] 0180_usermessage_add_active_mobile_push_notification", -] # Our next optimization is to check whether any migrations are needed # before we start the critical section of the restart. This saves # about 1s of downtime in a no-op upgrade. @@ -311,16 +302,10 @@ if not args.skip_migrations: migrations_output = subprocess.check_output( ["./manage.py", "showmigrations"], preexec_fn=su_to_zulip, text=True ) - need_create_large_indexes = False for ln in migrations_output.split("\n"): line_str = ln.strip() if line_str.startswith("[ ]"): migrations_needed = True - if line_str in usermessage_index_migrations: - need_create_large_indexes = True - if need_create_large_indexes: - logging.info("Creating some expensive indexes before starting downtime.") - subprocess.check_call(["./manage.py", "create_large_indexes"], preexec_fn=su_to_zulip) if (not args.skip_puppet or migrations_needed) and IS_SERVER_UP: # By default, we shut down the service to apply migrations and diff --git a/zerver/management/commands/create_large_indexes.py b/zerver/management/commands/create_large_indexes.py deleted file mode 100644 index d7e0dace56..0000000000 --- a/zerver/management/commands/create_large_indexes.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import Any - -from django.db import connection - -from zerver.lib.management import ZulipBaseCommand - - -def create_indexes() -> None: - # Creating concurrent indexes is kind of a pain with current versions - # of Django/PostgreSQL, because you will get this error with seemingly - # reasonable code: - # - # CREATE INDEX CONCURRENTLY cannot be executed from a function or multi-command string - # - # For a lot more detail on this process, refer to the commit message - # that added this file to the repo. - - with connection.cursor() as cursor: - # copied from 0082 - print("Creating index zerver_usermessage_starred_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_starred_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 2) != 0; - """ - ) - - # copied from 0083 - print("Creating index zerver_usermessage_mentioned_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_mentioned_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 8) != 0; - """ - ) - - # copied from 0095 - print("Creating index zerver_usermessage_unread_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_unread_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 1) = 0; - """ - ) - - # copied from 0098 - print("Creating index zerver_usermessage_has_alert_word_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_has_alert_word_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 512) != 0; - """ - ) - - # copied from 0099 - print("Creating index zerver_usermessage_wildcard_mentioned_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_wildcard_mentioned_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 8) != 0 OR (flags & 16) != 0; - """ - ) - - # copied from 0177 - print("Creating index zerver_usermessage_is_private_message_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_is_private_message_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 2048) != 0; - """ - ) - - # copied from 0180 - print("Creating index zerver_usermessage_active_mobile_push_notification_id.") - cursor.execute( - """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_active_mobile_push_notification_id - ON zerver_usermessage (user_profile_id, message_id) - WHERE (flags & 4096) != 0; - """ - ) - - print("Finished.") - - -class Command(ZulipBaseCommand): - help = """Create concurrent indexes for large tables.""" - - def handle(self, *args: Any, **options: str) -> None: - create_indexes() diff --git a/zerver/migrations/0082_index_starred_user_messages.py b/zerver/migrations/0082_index_starred_user_messages.py index 06d7b90efb..52c6775a30 100644 --- a/zerver/migrations/0082_index_starred_user_messages.py +++ b/zerver/migrations/0082_index_starred_user_messages.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_starred_message_id + CREATE INDEX zerver_usermessage_starred_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 2) != 0; """, diff --git a/zerver/migrations/0083_index_mentioned_user_messages.py b/zerver/migrations/0083_index_mentioned_user_messages.py index 25446a91b3..3370dae828 100644 --- a/zerver/migrations/0083_index_mentioned_user_messages.py +++ b/zerver/migrations/0083_index_mentioned_user_messages.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_mentioned_message_id + CREATE INDEX zerver_usermessage_mentioned_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 8) != 0; """, diff --git a/zerver/migrations/0095_index_unread_user_messages.py b/zerver/migrations/0095_index_unread_user_messages.py index b3483ee2c7..93dda5964c 100644 --- a/zerver/migrations/0095_index_unread_user_messages.py +++ b/zerver/migrations/0095_index_unread_user_messages.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_unread_message_id + CREATE INDEX zerver_usermessage_unread_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 1) = 0; """, diff --git a/zerver/migrations/0098_index_has_alert_word_user_messages.py b/zerver/migrations/0098_index_has_alert_word_user_messages.py index 54c859073a..efcfbec481 100644 --- a/zerver/migrations/0098_index_has_alert_word_user_messages.py +++ b/zerver/migrations/0098_index_has_alert_word_user_messages.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_has_alert_word_message_id + CREATE INDEX zerver_usermessage_has_alert_word_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 512) != 0; """, diff --git a/zerver/migrations/0099_index_wildcard_mentioned_user_messages.py b/zerver/migrations/0099_index_wildcard_mentioned_user_messages.py index 6fb7248139..a5379b81d9 100644 --- a/zerver/migrations/0099_index_wildcard_mentioned_user_messages.py +++ b/zerver/migrations/0099_index_wildcard_mentioned_user_messages.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_wildcard_mentioned_message_id + CREATE INDEX zerver_usermessage_wildcard_mentioned_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 8) != 0 OR (flags & 16) != 0; """, diff --git a/zerver/migrations/0177_user_message_add_and_index_is_private_flag.py b/zerver/migrations/0177_user_message_add_and_index_is_private_flag.py index 40c68783fa..6907dd2080 100644 --- a/zerver/migrations/0177_user_message_add_and_index_is_private_flag.py +++ b/zerver/migrations/0177_user_message_add_and_index_is_private_flag.py @@ -97,7 +97,7 @@ class Migration(migrations.Migration): ), migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_is_private_message_id + CREATE INDEX zerver_usermessage_is_private_message_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 2048) != 0; """, diff --git a/zerver/migrations/0180_usermessage_add_active_mobile_push_notification.py b/zerver/migrations/0180_usermessage_add_active_mobile_push_notification.py index d3bae14ffa..7998f926e0 100644 --- a/zerver/migrations/0180_usermessage_add_active_mobile_push_notification.py +++ b/zerver/migrations/0180_usermessage_add_active_mobile_push_notification.py @@ -57,7 +57,7 @@ class Migration(migrations.Migration): ), migrations.RunSQL( """ - CREATE INDEX IF NOT EXISTS zerver_usermessage_active_mobile_push_notification_id + CREATE INDEX zerver_usermessage_active_mobile_push_notification_id ON zerver_usermessage (user_profile_id, message_id) WHERE (flags & 4096) != 0; """,