From 840884ec89c49e66edfd37d745a62243703c5762 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 30 Jan 2023 20:30:34 +0000 Subject: [PATCH] upgrade-zulip: Provide directories to run hooks before/after upgrade. These hooks are run immediately around the critical section of the upgrade. If the upgrade fails for preparatory reasons, the pre-deploy hook may not be run; if it fails during the upgrade, the post-deploy hook will not be run. Hooks are called from the CWD of the new deploy, with arguments of the old version and the new version. If they exit with non-0 exit code, the deploy aborts. --- docs/production/deployment.md | 6 +++++ docs/production/upgrade.md | 12 ++++++++++ puppet/zulip/manifests/app_frontend_base.pp | 1 + puppet/zulip/manifests/hooks/base.pp | 15 ++++++++++++ scripts/lib/upgrade-zulip-stage-2 | 26 ++++++++++++++++++++- 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 puppet/zulip/manifests/hooks/base.pp diff --git a/docs/production/deployment.md b/docs/production/deployment.md index 4294cecb1b..885199b471 100644 --- a/docs/production/deployment.md +++ b/docs/production/deployment.md @@ -85,6 +85,12 @@ running on the server; though installing alongside other applications is not recommended, we do have [some notes on the process](install-existing-server.md). +## Deployment hooks + +Zulip's upgrades have a hook system which allows for arbitrary +user-configured actions to run before and after an upgrade; see the +[upgrading documentation](upgrade.md#deployment-hooks) for details. + ## Running Zulip's service dependencies on different machines Zulip has full support for each top-level service living on its own diff --git a/docs/production/upgrade.md b/docs/production/upgrade.md index 8680794eb2..1ff44e7e15 100644 --- a/docs/production/upgrade.md +++ b/docs/production/upgrade.md @@ -211,6 +211,18 @@ earlier previous version by running `restart-server` script stops any running Zulip server, and starts the version corresponding to the `restart-server` path you call. +## Deployment hooks + +Zulip's upgrades have a hook system which allows for arbitrary +user-configured actions to run before and after an upgrade. + +Files in the `/etc/zulip/pre-deploy.d` and `/etc/zulip/post-deploy.d` +directories are inspected for files ending with `.hook`, just before +and after the critical period when the server is restarted. Each file +is called, sorted in alphabetical order, from the working directory of +the new version, with arguments of the old and new Zulip versions. If +they exit with non-0 exit code, the upgrade will abort. + ## Preserving local changes to service configuration files :::{warning} diff --git a/puppet/zulip/manifests/app_frontend_base.pp b/puppet/zulip/manifests/app_frontend_base.pp index acc0a274cc..ea7a0838e2 100644 --- a/puppet/zulip/manifests/app_frontend_base.pp +++ b/puppet/zulip/manifests/app_frontend_base.pp @@ -5,6 +5,7 @@ class zulip::app_frontend_base { include zulip::sasl_modules include zulip::supervisor include zulip::tornado_sharding + include zulip::hooks::base if $::os['family'] == 'Debian' { # Upgrade and other tooling wants to be able to get a database diff --git a/puppet/zulip/manifests/hooks/base.pp b/puppet/zulip/manifests/hooks/base.pp new file mode 100644 index 0000000000..99adf87ef8 --- /dev/null +++ b/puppet/zulip/manifests/hooks/base.pp @@ -0,0 +1,15 @@ +# @summary Install sentry-cli binary and pre/post deploy hooks +# +class zulip::hooks::base { + file { [ + '/etc/zulip/hooks', + '/etc/zulip/hooks/pre-deploy.d', + '/etc/zulip/hooks/post-deploy.d', + ]: + ensure => directory, + owner => 'zulip', + group => 'zulip', + mode => '0755', + tag => ['hooks'], + } +} diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index ddbf74ce50..72f6316a43 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -14,7 +14,7 @@ import shutil import subprocess import sys import time -from typing import NoReturn, Optional +from typing import Literal, NoReturn, Optional os.environ["PYTHONUNBUFFERED"] = "y" @@ -407,6 +407,13 @@ if not migrations_needed: ["./manage.py", "fill_memcached_caches", "--skip-checks"], preexec_fn=su_to_zulip ) +# Install hooks before we check for puppet changes, so we skip an +# unnecessary stop/start cycle if hook changes are the only ones. +logging.info("Installing hooks") +subprocess.check_call( + ["./scripts/zulip-puppet-apply", "--tags", "hooks", "--force"], +) + # If we are planning on running puppet, we can pre-run it in --noop # mode and see if it will actually make any changes; if not, we can # skip it during the upgrade. We omit this check if the server is @@ -432,6 +439,19 @@ if not args.skip_puppet and IS_SERVER_UP: sys.exit(1) +def run_hooks(kind: Literal["pre-deploy", "post-deploy"]) -> None: + # Updates to the above literal to add a new hook type should + # adjust puppet's `app_frontend_base.pp` as well. + path = f"/etc/zulip/hooks/{kind}.d" + if not os.path.exists(path): + return + for script_name in sorted(f for f in os.listdir(path) if f.endswith(".hook")): + subprocess.check_call( + [os.path.join(path, script_name), old_version, NEW_ZULIP_VERSION], + cwd=deploy_path, + ) + + if args.skip_restart: logging.info("Successfully configured in %s!", deploy_path) else: @@ -440,6 +460,8 @@ else: # steps that happen between here and the "Restarting Zulip" line # below. + run_hooks("pre-deploy") + if rabbitmq_dist_listen: shutdown_server() logging.info("Shutting down rabbitmq to adjust its ports...") @@ -491,6 +513,8 @@ else: logging.info("Upgrade complete!") + run_hooks("post-deploy") + if args.audit_fts_indexes: logging.info("Correcting full-text search indexes for updated dictionary files") logging.info("This may take a while but the server should work while it runs.")