upgrade-zulip: Provide directories to run hooks before/after upgrade.

These hooks are run immediately around the critical section of the
upgrade.  If the upgrade fails for preparatory reasons, the pre-deploy
hook may not be run; if it fails during the upgrade, the post-deploy
hook will not be run.  Hooks are called from the CWD of the new
deploy, with arguments of the old version and the new version.  If
they exit with non-0 exit code, the deploy aborts.

docs/production/deployment.md

@@ -85,6 +85,12 @@ running on the server; though installing alongside other applications
 is not recommended, we do have [some notes on the
 process](install-existing-server.md).
 
+## Deployment hooks
+
+Zulip's upgrades have a hook system which allows for arbitrary
+user-configured actions to run before and after an upgrade; see the
+[upgrading documentation](upgrade.md#deployment-hooks) for details.
+
 ## Running Zulip's service dependencies on different machines
 
 Zulip has full support for each top-level service living on its own

docs/production/upgrade.md

@@ -211,6 +211,18 @@ earlier previous version by running
 `restart-server` script stops any running Zulip server, and starts
 the version corresponding to the `restart-server` path you call.
 
+## Deployment hooks
+
+Zulip's upgrades have a hook system which allows for arbitrary
+user-configured actions to run before and after an upgrade.
+
+Files in the `/etc/zulip/pre-deploy.d` and `/etc/zulip/post-deploy.d`
+directories are inspected for files ending with `.hook`, just before
+and after the critical period when the server is restarted. Each file
+is called, sorted in alphabetical order, from the working directory of
+the new version, with arguments of the old and new Zulip versions. If
+they exit with non-0 exit code, the upgrade will abort.
+
 ## Preserving local changes to service configuration files
 
 :::{warning}

puppet/zulip/manifests/app_frontend_base.pp

@@ -5,6 +5,7 @@ class zulip::app_frontend_base {
   include zulip::sasl_modules
   include zulip::supervisor
   include zulip::tornado_sharding
+  include zulip::hooks::base
 
   if $::os['family'] == 'Debian' {
     # Upgrade and other tooling wants to be able to get a database

puppet/zulip/manifests/hooks/base.pp

@@ -0,0 +1,15 @@
+# @summary Install sentry-cli binary and pre/post deploy hooks
+#
+class zulip::hooks::base {
+  file { [
+    '/etc/zulip/hooks',
+    '/etc/zulip/hooks/pre-deploy.d',
+    '/etc/zulip/hooks/post-deploy.d',
+  ]:
+    ensure => directory,
+    owner  => 'zulip',
+    group  => 'zulip',
+    mode   => '0755',
+    tag    => ['hooks'],
+  }
+}

scripts/lib/upgrade-zulip-stage-2

@@ -14,7 +14,7 @@ import shutil
 import subprocess
 import sys
 import time
-from typing import NoReturn, Optional
+from typing import Literal, NoReturn, Optional
 
 os.environ["PYTHONUNBUFFERED"] = "y"
 
@@ -407,6 +407,13 @@ if not migrations_needed:
         ["./manage.py", "fill_memcached_caches", "--skip-checks"], preexec_fn=su_to_zulip
     )
 
+# Install hooks before we check for puppet changes, so we skip an
+# unnecessary stop/start cycle if hook changes are the only ones.
+logging.info("Installing hooks")
+subprocess.check_call(
+    ["./scripts/zulip-puppet-apply", "--tags", "hooks", "--force"],
+)
+
 # If we are planning on running puppet, we can pre-run it in --noop
 # mode and see if it will actually make any changes; if not, we can
 # skip it during the upgrade.  We omit this check if the server is
@@ -432,6 +439,19 @@ if not args.skip_puppet and IS_SERVER_UP:
         sys.exit(1)
 
 
+def run_hooks(kind: Literal["pre-deploy", "post-deploy"]) -> None:
+    # Updates to the above literal to add a new hook type should
+    # adjust puppet's `app_frontend_base.pp` as well.
+    path = f"/etc/zulip/hooks/{kind}.d"
+    if not os.path.exists(path):
+        return
+    for script_name in sorted(f for f in os.listdir(path) if f.endswith(".hook")):
+        subprocess.check_call(
+            [os.path.join(path, script_name), old_version, NEW_ZULIP_VERSION],
+            cwd=deploy_path,
+        )
+
+
 if args.skip_restart:
     logging.info("Successfully configured in %s!", deploy_path)
 else:
@@ -440,6 +460,8 @@ else:
     # steps that happen between here and the "Restarting Zulip" line
     # below.
 
+    run_hooks("pre-deploy")
+
     if rabbitmq_dist_listen:
         shutdown_server()
         logging.info("Shutting down rabbitmq to adjust its ports...")
@@ -491,6 +513,8 @@ else:
 
     logging.info("Upgrade complete!")
 
+    run_hooks("post-deploy")
+
 if args.audit_fts_indexes:
     logging.info("Correcting full-text search indexes for updated dictionary files")
     logging.info("This may take a while but the server should work while it runs.")