upgrade: Check PostgreSQL versions after venv setup.

The claim in the comment from c8ec3dfcf6, that we can and should use
the current deploy's venv, misses one key case -- when upgrading the
operating system, the current deploy's venv is unworkable, since it
was configured for a previous version of Python.  As such, any attempt
to load Django to verify the version of PostgreSQL it is talking to
must happen after the venv is configured.

Move the database version check into
`scripts/lib/check-database-compatibility`, which also moves it after
the new venv is configured.

Because we no longer reliably know, at `apt-get upgrade` time, what
version of PostgreSQL is installed, we hold all versions of the
pgroonga packages.

scripts/lib/check-database-compatibility

@@ -1,12 +1,20 @@
 #!/usr/bin/env python3
 import logging
 import os
+import subprocess
 import sys
 
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.insert(0, ZULIP_PATH)
+
 from scripts.lib.setup_path import setup_path
-from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, assert_not_running_as_root, parse_version_from
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_not_running_as_root,
+    get_config,
+    get_config_file,
+    parse_version_from,
+)
 from version import ZULIP_VERSION as NEW_VERSION
 
 assert_not_running_as_root()
@@ -18,6 +26,51 @@ from django.db import connection
 from django.db.migrations.loader import MigrationLoader
 
 django.setup()
+
+django_pg_version = connection.cursor().connection.server_version // 10000
+if os.path.exists("/etc/init.d/postgresql") and os.path.exists("/etc/zulip/zulip.conf"):
+    postgresql_version = int(get_config(get_config_file(), "postgresql", "version", "0"))
+    if postgresql_version == 0:
+        postgresql_version = django_pg_version
+        subprocess.check_call(
+            [
+                "crudini",
+                "--set",
+                "/etc/zulip/zulip.conf",
+                "postgresql",
+                "version",
+                str(postgresql_version),
+            ]
+        )
+    elif postgresql_version != django_pg_version:
+        logging.critical(
+            "PostgreSQL version mismatch: %d (running) vs %d (configured)",
+            django_pg_version,
+            postgresql_version,
+        )
+        logging.info(
+            "/etc/zulip/zulip.conf claims that Zulip is running PostgreSQL\n"
+            "%d, but the server is connected to a PostgreSQL running\n"
+            "version %d.  Check the output from pg_lsclusters to verify\n"
+            "which clusters are running, and update /etc/zulip/zulip.conf to match.\n"
+            "\n"
+            "In general, this results from manually upgrading PostgreSQL; you\n"
+            "should follow our instructions for using our tool to do so:\n"
+            "https://zulip.readthedocs.io/en/latest/production/upgrade.html#upgrading-postgresql",
+            postgresql_version,
+            django_pg_version,
+        )
+        sys.exit(1)
+
+if django_pg_version < 12:
+    logging.critical("Unsupported PostgreSQL version: %d", postgresql_version)
+    logging.info(
+        "Please upgrade to PostgreSQL 12 or newer first.\n"
+        "See https://zulip.readthedocs.io/en/latest/production/"
+        "upgrade-or-modify.html#upgrading-postgresql"
+    )
+    sys.exit(1)
+
 loader = MigrationLoader(connection)
 missing = set(loader.applied_migrations)
 

scripts/lib/upgrade-zulip-stage-2

@@ -204,64 +204,6 @@ def shutdown_server() -> None:
         IS_SERVER_UP = False
 
 
-# postgresql.version is required for database servers, but wasn't
-# previously; fill it in based on what the OS provides.
-postgresql_version = None
-if os.path.exists("/etc/init.d/postgresql"):
-    postgresql_version = get_config(config_file, "postgresql", "version")
-    django_pg_version = subprocess.check_output(
-        # We use the _current_ deploy's manage.py, since ours has not
-        # yet had its virtualenv configured yet.
-        [
-            "../current/manage.py",
-            "shell",
-            "-c",
-            "from django.db import connection; print(int(connection.cursor().connection.server_version/10000))",
-        ],
-        preexec_fn=su_to_zulip,
-        text=True,
-    ).strip()
-    if not postgresql_version:
-        postgresql_version = django_pg_version
-        subprocess.check_call(
-            [
-                "crudini",
-                "--set",
-                "/etc/zulip/zulip.conf",
-                "postgresql",
-                "version",
-                postgresql_version,
-            ]
-        )
-    elif postgresql_version != django_pg_version:
-        logging.critical(
-            "PostgreSQL version mismatch: %s (running) vs %s (configured)",
-            django_pg_version,
-            postgresql_version,
-        )
-        logging.info(
-            "/etc/zulip/zulip.conf claims that Zulip is running PostgreSQL\n"
-            "%s, but the server is connected to a PostgreSQL running\n"
-            "version %s.  Check the output from pg_lsclusters to verify\n"
-            "which clusters are running, and update /etc/zulip/zulip.conf to match.\n"
-            "\n"
-            "In general, this results from manually upgrading PostgreSQL; you\n"
-            "should follow our instructions for using our tool to do so:\n"
-            "https://zulip.readthedocs.io/en/latest/production/upgrade.html#upgrading-postgresql",
-            postgresql_version,
-            django_pg_version,
-        )
-        sys.exit(1)
-
-    if int(postgresql_version) < 12:
-        logging.critical("Unsupported PostgreSQL version: %s", postgresql_version)
-        logging.info(
-            "Please upgrade to PostgreSQL 12 or newer first.\n"
-            "See https://zulip.readthedocs.io/en/latest/production/"
-            "upgrade-or-modify.html#upgrading-postgresql"
-        )
-        sys.exit(1)
-
 if glob.glob("/usr/share/postgresql/*/extension/tsearch_extras.control"):
     # Remove legacy tsearch_extras package references
     run_psql_as_postgres(
@@ -274,17 +216,13 @@ if not (minimal_change or args.skip_puppet):
     # We need to temporarily hold pgroonga, if installed -- upgrading
     # it without running the appropriate upgrade SQL can cause
     # PostgreSQL to crash
-    if postgresql_version is not None and get_config_bool(config_file, "machine", "pgroonga"):
+    if get_config_bool(config_file, "machine", "pgroonga"):
-        subprocess.check_call(
+        subprocess.check_call(["apt-mark", "hold", "postgresql-*-pgdg-pgroonga"])
-            ["apt-mark", "hold", f"postgresql-{postgresql_version}-pgdg-pgroonga"]
-        )
     logging.info("Upgrading system packages...")
     subprocess.check_call(["apt-get", "update"])
     subprocess.check_call(["apt-get", "-y", "--with-new-pkgs", "upgrade"])
-    if postgresql_version is not None and get_config_bool(config_file, "machine", "pgroonga"):
+    if get_config_bool(config_file, "machine", "pgroonga"):
-        subprocess.check_call(
+        subprocess.check_call(["apt-mark", "unhold", "postgresql-*-pgdg-pgroonga"])
-            ["apt-mark", "unhold", f"postgresql-{postgresql_version}-pgdg-pgroonga"]
-        )
 
 # To bootstrap zulip-puppet-apply, we need to install the system yaml
 # package; new installs get this, but old installs may not have it.