zulip/scripts/lib/upgrade-zulip-from-git

#!/usr/bin/env python3
import argparse
import logging
import os
import subprocess
import sys
import time

LOCAL_GIT_CACHE_DIR = "/srv/zulip.git"
os.environ["PYTHONUNBUFFERED"] = "y"

sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
from scripts.lib.zulip_tools import (
    DEPLOYMENTS_DIR,
    assert_running_as_root,
    get_config,
    get_config_file,
    get_deploy_options,
    get_deployment_lock,
    make_deploy_path,
    overwrite_symlink,
    release_deployment_lock,
    su_to_zulip,
)

config_file = get_config_file()
deploy_options = get_deploy_options(config_file)
upstream_url = "https://github.com/zulip/zulip.git"
remote_url = get_config(config_file, "deployment", "git_repo_url", upstream_url)

assert_running_as_root(strip_lib_from_paths=True)

# make sure we have appropriate file permissions
os.umask(0o22)

logging.Formatter.converter = time.gmtime
logging.basicConfig(format="%(asctime)s upgrade-zulip-from-git: %(message)s", level=logging.INFO)

parser = argparse.ArgumentParser()
parser.add_argument("refname", help="Git reference, e.g. a branch, tag, or commit ID.")
parser.add_argument(
    "--remote-url", help="Override the Git remote URL configured in /etc/zulip/zulip.conf."
)
args, extra_options = parser.parse_known_args()

refname = args.refname
# Command line remote URL will be given preference above the one
# in /etc/zulip/zulip.conf.
if args.remote_url:
    remote_url = args.remote_url

os.makedirs(DEPLOYMENTS_DIR, exist_ok=True)
os.makedirs("/home/zulip/logs", exist_ok=True)

error_rerun_script = f"{DEPLOYMENTS_DIR}/current/scripts/upgrade-zulip-from-git {refname}"
get_deployment_lock(error_rerun_script)

try:
    deploy_path = make_deploy_path()

    # Populate LOCAL_GIT_CACHE_DIR with both the requested remote and zulip/zulip.
    if not os.path.exists(LOCAL_GIT_CACHE_DIR):
        logging.info("Making local repository cache")
        subprocess.check_call(
            ["git", "init", "--bare", "-q", LOCAL_GIT_CACHE_DIR],
            stdout=subprocess.DEVNULL,
        )
        subprocess.check_call(
            ["git", "remote", "add", "origin", remote_url],
            cwd=LOCAL_GIT_CACHE_DIR,
        )

    if os.stat(LOCAL_GIT_CACHE_DIR).st_uid == 0:
        subprocess.check_call(["chown", "-R", "zulip:zulip", LOCAL_GIT_CACHE_DIR])

    os.chdir(LOCAL_GIT_CACHE_DIR)
    subprocess.check_call(
        ["git", "remote", "set-url", "origin", remote_url], preexec_fn=su_to_zulip
    )

    fetch_spec = subprocess.check_output(
        ["git", "config", "remote.origin.fetch"],
        preexec_fn=su_to_zulip,
        text=True,
    ).strip()
    if fetch_spec in ("+refs/*:refs/*", "+refs/heads/*:refs/heads/*"):
        # The refspec impinges on refs/heads/ -- this is an old mirror
        # configuration.
        logging.info("Cleaning up mirrored repository")
        # remotes.origin.mirror may not be set -- we do not use
        # check_call to ignore errors if it's already missing
        subprocess.call(
            ["git", "config", "--unset", "remote.origin.mirror"],
            preexec_fn=su_to_zulip,
        )
        subprocess.check_call(
            ["git", "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"],
            preexec_fn=su_to_zulip,
        )
        matching_refs = subprocess.check_output(
            ["git", "for-each-ref", "--format=%(refname)", "refs/pull/", "refs/heads/"],
            preexec_fn=su_to_zulip,
            text=True,
        ).splitlines()

        # We can't use `git worktree list --porcelain -z` here because
        # Ubuntu 20.04 Focal only has git 2.25.1, and -z was
        # introduced in 2.36
        worktree_data = subprocess.check_output(
            ["git", "worktree", "list", "--porcelain"],
            preexec_fn=su_to_zulip,
            text=True,
        ).splitlines()
        keep_refs = set()
        for worktree_line in worktree_data:
            if worktree_line.startswith("branch "):
                keep_refs.add(worktree_line[len("branch ") :])

        delete_input = "".join(
            f"delete {refname}\n" for refname in matching_refs if refname not in keep_refs
        )
        subprocess.run(
            ["git", "update-ref", "--stdin"],
            check=True,
            preexec_fn=su_to_zulip,
            input=delete_input,
            text=True,
        )

        logging.info("Repacking repository after pruning unnecessary refs...")
        subprocess.check_call(
            ["git", "gc", "--prune=now"],
            preexec_fn=su_to_zulip,
        )

    # Ensure upstream remote is configured; we need this to make `git describe` accurate.
    remotes = subprocess.check_output(["git", "remote"], preexec_fn=su_to_zulip).split(b"\n")
    if b"upstream" not in remotes:
        subprocess.check_call(
            ["git", "remote", "add", "upstream", upstream_url], preexec_fn=su_to_zulip
        )
    else:
        subprocess.check_call(
            ["git", "remote", "set-url", "upstream", upstream_url], preexec_fn=su_to_zulip
        )

    logging.info("Fetching the latest commits")
    subprocess.check_call(
        ["git", "fetch", "--prune", "--quiet", "--tags", "--all"], preexec_fn=su_to_zulip
    )

    # Generate the deployment directory via git worktree from our local repository.
    try:
        fullref = f"refs/tags/{refname}"
        commit_hash = subprocess.check_output(
            ["git", "rev-parse", "--verify", fullref],
            preexec_fn=su_to_zulip,
            text=True,
            stderr=subprocess.DEVNULL,
        ).strip()
    except subprocess.CalledProcessError as e:
        if e.returncode == 128:
            # Try in the origin namespace
            fullref = f"refs/remotes/origin/{refname}"
            commit_hash = subprocess.check_output(
                ["git", "rev-parse", "--verify", fullref],
                preexec_fn=su_to_zulip,
                text=True,
                stderr=subprocess.DEVNULL,
            ).strip()
    refname = fullref
    logging.info("Upgrading to %s, in %s", commit_hash, deploy_path)
    subprocess.check_call(
        ["git", "worktree", "add", "--detach", deploy_path, refname],
        stdout=subprocess.DEVNULL,
        preexec_fn=su_to_zulip,
    )
    os.chdir(deploy_path)
    extra_flags = []
    if not refname.startswith("refs/tags/"):
        extra_flags = ["-t"]
    subprocess.check_call(
        [
            "git",
            "checkout",
            "-q",
            *extra_flags,
            "-b",
            "deployment-" + os.path.basename(deploy_path),
            refname,
        ],
        preexec_fn=su_to_zulip,
    )

    overwrite_symlink("/etc/zulip/settings.py", "zproject/prod_settings.py")

    overwrite_symlink(deploy_path, os.path.join(DEPLOYMENTS_DIR, "next"))

    try:
        subprocess.check_call(
            [
                os.path.join(deploy_path, "scripts", "lib", "upgrade-zulip-stage-2"),
                deploy_path,
                "--from-git",
                *deploy_options,
                *extra_options,
            ]
        )
    except subprocess.CalledProcessError:
        # There's no use in showing a stacktrace here; it just hides
        # the error from stage 2.
        sys.exit(1)
finally:
    release_deployment_lock()