zulip/scripts/lib/install

#!/usr/bin/env bash
set -e

usage() {
    cat <<EOF
Usage:
  install --hostname=zulip.example.com --email=zulip-admin@example.com [options...]
  install --help

Other options:
  --certbot
  --self-signed-cert
  --no-init-db
  --cacert
  --no-dist-upgrade
  --postgres-missing-dictionaries
  --remote-postgres

The --hostname and --email options are required,
unless --no-init-db is set and --certbot is not.

EOF
};

# Shell option parsing.  Over time, we'll want to move some of the
# environment variables below into this self-documenting system.
args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,no-overwrite-settings,self-signed-cert,certbot,postgres-missing-dictionaries,remote-postgres,hostname:,email:,cacert: -n "$0" -- "$@")"
eval "set -- $args"
while true; do
    case "$1" in
        --help) usage; exit 0;;
        --self-signed-cert) SELF_SIGNED_CERT=1; shift;;
        --cacert) export CUSTOM_CA_CERTIFICATES="$2"; shift; shift;;
        --certbot) USE_CERTBOT=1; shift;;
        --hostname) EXTERNAL_HOST="$2"; shift; shift;;
        --email) ZULIP_ADMINISTRATOR="$2"; shift; shift;;
        --no-overwrite-settings) NO_OVERWRITE_SETTINGS=1; shift;;
        --no-init-db) NO_INIT_DB=1; shift;;
        --no-dist-upgrade) NO_DIST_UPGRADE=1; shift;;
        --postgres-missing-dictionaries) POSTGRES_MISSING_DICTIONARIES=1; shift;;
        --remote-postgres) REMOTE_POSTGRES=1; shift;;
        --) shift; break;;
    esac
done

if [ "$#" -gt 0 ]; then
    usage >&2
    exit 1
fi

## Options from environment variables.
#
# Specify options for apt.
read -r -a APT_OPTIONS <<< "${APT_OPTIONS:-}"
# Install additional packages.
read -r -a ADDITIONAL_PACKAGES <<< "${ADDITIONAL_PACKAGES:-}"
# Deployment type is almost always voyager.
DEPLOYMENT_TYPE="${DEPLOYMENT_TYPE:-voyager}"
# Comma-separated list of puppet manifests to install.  default is
# zulip::voyager for an all-in-one system or zulip::dockervoyager for
# Docker.  Use e.g. zulip::app_frontend for a Zulip frontend server.
PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::voyager}"
VIRTUALENV_NEEDED="${VIRTUALENV_NEEDED:-yes}"

if [ -n "$SELF_SIGNED_CERT" ] && [ -n "$USE_CERTBOT" ]; then
    set +x
    echo "error: --self-signed-cert and --certbot are incompatible" >&2
    echo >&2
    usage >&2
    exit 1
fi

if [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; then
    if [ -n "$USE_CERTBOT" ] || [ -z "$NO_INIT_DB" ]; then
        usage >&2
        exit 1
    fi
fi

if [ "$EXTERNAL_HOST" = zulip.example.com ] ||
   [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then
    # These example values are specifically checked for and would fail
    # later; see check_config in zerver/lib/management.py.
    echo 'error: The example hostname and email must be replaced with real values.' >&2
    echo >&2
    usage >&2
    exit 1
fi

# Do set -x after option parsing is complete
set -x

ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"

# Force a known locale.  Some packages on PyPI fail to install in some locales.
localedef -i en_US -f UTF-8 en_US.UTF-8
export LC_ALL="en_US.UTF-8"
export LANG="en_US.UTF-8"
export LANGUAGE="en_US.UTF-8"

# Check for a supported OS release.
if [ -f /etc/os-release ]; then
    os_info="$(. /etc/os-release; printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME")"
    { read -r os_id; read -r os_id_like; read -r os_version_id; read -r os_version_codename || true; } <<< "$os_info"
fi

case "$os_id$os_version_id" in
    ubuntu16.04|ubuntu18.04|debian9|debian10) ;;
    *)
        set +x
        cat <<EOF

Unsupported OS release: $os_id $os_version_id

Zulip in production is supported only on:
 - Debian 9 "stretch"
 - Debian 10 "buster"
 - Ubuntu 16.04 LTS "xenial"
 - Ubuntu 18.04 LTS "bionic"

For more information, see:
  https://zulip.readthedocs.io/en/latest/production/requirements.html
EOF
        exit 1
esac

if [ "$os_id" = ubuntu ] && ! apt-cache policy |
           grep -q "^     release v=$os_version_id,o=Ubuntu,a=$os_version_codename,n=$os_version_codename,l=Ubuntu,c=universe"; then
    set +x
    cat <<'EOF'

You must enable the Ubuntu Universe repository before installing
Zulip.  You can do this with:

    sudo add-apt-repository universe
    sudo apt update

For more information, see:
  https://zulip.readthedocs.io/en/latest/production/requirements.html
EOF
    exit 1
fi

# Check for at least ~1.86GB of RAM before starting installation;
# otherwise users will find out about insufficient RAM via weird
# errors like a segfault running `pip install`.
# Additionally, some AWS images that are advertised to be 2 GB
# are actually 1880000B in size.
mem_kb=$(head -n1 /proc/meminfo | awk '{print $2}')
if [ "$mem_kb" -lt 1860000 ]; then
    set +x
    echo -e '\033[0;31m' >&2
    echo "Insufficient RAM.  Zulip requires at least 2GB of RAM." >&2
    echo >&2
    echo -e '\033[0m' >&2
    exit 1
fi

# Do package update, e.g. do `apt-get update` on Debian
case " $os_id $os_id_like " in
    *' debian '*)
        # setup-apt-repo does an `apt-get update`
        "$ZULIP_PATH"/scripts/lib/setup-apt-repo
        ;;
    *' rhel '*)
        "$ZULIP_PATH"/scripts/lib/setup-yum-repo
        ;;
esac

# Handle issues around upstart on Ubuntu Xenial
"$ZULIP_PATH"/scripts/lib/check-upstart

# Check early for missing SSL certificates
if [ "$PUPPET_CLASSES" = "zulip::voyager" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then
    set +x
    cat <<EOF

No SSL certificate found.  One or both required files is missing:
    /etc/ssl/private/zulip.key
    /etc/ssl/certs/zulip.combined-chain.crt

Suggested solutions:
 * For most sites, the --certbot option is recommended.
 * If you have your own key and cert, see docs linked below
   for how to install them.
 * For non-production testing, try the --self-signed-cert option.

For help and more details, see our SSL documentation:
  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html

Once fixed, just rerun scripts/setup/install; it'll pick up from here!

EOF
   exit 1
fi

# don't run dist-upgrade in one click apps to make the
# installation process more seamless.
if [ -z "$NO_DIST_UPGRADE" ]; then
    case " $os_id $os_id_like " in
        *' debian '*)
            apt-get -y dist-upgrade "${APT_OPTIONS[@]}"
            ;;
        # On CentOS, there is no need to do `yum -y upgrade` because `yum -y
        # update` already does the same thing.
    esac
fi

case " $os_id $os_id_like " in
    *' debian '*)
        if ! apt-get install -y \
            puppet git curl wget jq \
            python python3 python-six python3-six crudini \
            "${ADDITIONAL_PACKAGES[@]}"; then
            set +x
            echo -e '\033[0;31m' >&2
            echo "Installing packages failed; is network working and (on Ubuntu) the universe repository enabled?" >&2
            echo >&2
            echo -e '\033[0m' >&2
            exit 1
        fi
        ;;
    *' rhel '*)
        if ! yum install -y \
            puppet git curl wget jq \
            python python3 python-six python3-six crudini \
            "${ADDITIONAL_PACKAGES[@]}"; then
            set +x
            echo -e '\033[0;31m' >&2
            echo "Installing packages failed; is network working?" >&2
            echo >&2
            echo -e '\033[0m' >&2
            exit 1
        fi
        ;;
esac

if [ -n "$USE_CERTBOT" ]; then
    "$ZULIP_PATH"/scripts/setup/setup-certbot \
        --no-zulip-conf --method=standalone \
        "$EXTERNAL_HOST" --email "$ZULIP_ADMINISTRATOR"
elif [ -n "$SELF_SIGNED_CERT" ]; then
    "$ZULIP_PATH"/scripts/setup/generate-self-signed-cert \
        --exists-ok "${EXTERNAL_HOST:-$(hostname)}"
fi

# Create and activate a virtualenv
if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
    "$ZULIP_PATH"/scripts/lib/create-production-venv "$ZULIP_PATH"
    "$ZULIP_PATH"/scripts/lib/create-thumbor-venv "$ZULIP_PATH"
fi

"$ZULIP_PATH"/scripts/lib/install-node

# Generate /etc/zulip/zulip.conf .
mkdir -p /etc/zulip
(
    cat <<EOF
[machine]
puppet_classes = $PUPPET_CLASSES
deploy_type = $DEPLOYMENT_TYPE
EOF

    # Note: there are four dpkg-query outputs to consider:
    #
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # root@host# apt install rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # install ok installed
    # root@host# apt remove rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # deinstall ok config-files
    # root@host# apt purge rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # unknown ok not-installed
    #
    # (There are more possibilities in the case of dpkg errors.)  Here
    # we are checking for either empty or not-installed.
    if [ -n "$TRAVIS" ] || ! dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null | grep -vq ' not-installed$'; then
        cat <<EOF

[rabbitmq]
nodename = zulip@localhost
EOF
    fi

    if [ -n "$USE_CERTBOT" ]; then
        cat <<EOF

[certbot]
auto_renew = yes
EOF
    fi
) > /etc/zulip/zulip.conf

case ",$PUPPET_CLASSES," in
    *,zulip::voyager,* | *,zulip::dockervoyager,* | *,zulip::app_frontend,*)
        if [ -z "$NO_OVERWRITE_SETTINGS" ] || ! [ -e "/etc/zulip/settings.py" ]; then
            cp -a "$ZULIP_PATH"/zproject/prod_settings_template.py /etc/zulip/settings.py
            if [ -n "$EXTERNAL_HOST" ]; then
                sed -i "s/^EXTERNAL_HOST =.*/EXTERNAL_HOST = '$EXTERNAL_HOST'/" /etc/zulip/settings.py
            fi
            if [ -n "$ZULIP_ADMINISTRATOR" ]; then
                sed -i "s/^ZULIP_ADMINISTRATOR =.*/ZULIP_ADMINISTRATOR = '$ZULIP_ADMINISTRATOR'/" /etc/zulip/settings.py
            fi
        fi
        ln -nsf /etc/zulip/settings.py "$ZULIP_PATH"/zproject/prod_settings.py
        "$ZULIP_PATH"/scripts/setup/generate_secrets.py --production
        ;;
esac

"$ZULIP_PATH"/scripts/zulip-puppet-apply -f

case " $os_id $os_id_like " in
    *' debian '*)
        SUPERVISOR_CONF_DIR="/etc/supervisor/conf.d"
        ;;
    *' rhel '*)
        SUPERVISOR_CONF_DIR="/etc/supervisord.d/conf.d"
        ;;
esac
# Detect which features were selected for the below
set +e
[ -e "/etc/init.d/nginx" ]; has_nginx=$?
[ -e "$SUPERVISOR_CONF_DIR/zulip.conf" ]; has_appserver=$?
[ -e "/etc/cron.d/rabbitmq-numconsumers" ]; has_rabbit=$?
[ -e "/etc/init.d/postgresql" ]; has_postgres=$?
set -e

# Docker service setup is done in the docker config, not here
if [ "$DEPLOYMENT_TYPE" = "dockervoyager" ]; then
    has_nginx=1
    has_appserver=0
    has_rabbit=1
    has_postgres=1
fi

if [ -n "$POSTGRES_MISSING_DICTIONARIES" ]; then
    crudini --set /etc/zulip/zulip.conf postgresql missing_dictionaries true
fi

if [ -n "$REMOTE_POSTGRES" ]; then
  has_postgres=1
fi

# These server restarting bits should be moveable into puppet-land, ideally
case " $os_id $os_id_like " in
    *' debian '*)
        apt-get -y upgrade
        ;;
    *' rhel '*)
        # No action is required because `yum update` already does upgrade.
        ;;
esac

if [ "$has_nginx" = 0 ]; then
    # Check nginx was configured properly now that we've installed it.
    # Most common failure mode is certs not having been installed.
    nginx -t || (
        set +x
        cat <<EOF

Verifying the Zulip nginx configuration failed!

This is almost always a problem with your SSL certificates.  See:
  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html

Once fixed, just rerun scripts/setup/install; it'll pick up from here!

EOF
        exit 1
    )
    service nginx restart
fi

if [ "$has_rabbit" = 0 ]; then
    if ! rabbitmqctl status >/dev/null; then
        set +x
        cat <<EOF

RabbitMQ seems to not have started properly after the installation process.
Often this is caused by misconfigured /etc/hosts in virtualized environments.
For more information, see:
  https://github.com/zulip/zulip/issues/53#issuecomment-143805121

EOF
        exit 1
    fi
    "$ZULIP_PATH"/scripts/setup/configure-rabbitmq
fi

if [ "$has_postgres" = 0 ]; then
    "$ZULIP_PATH"/scripts/setup/postgres-init-db
fi

if [ "$has_appserver" = 0 ]; then
    deploy_path=$("$ZULIP_PATH"/scripts/lib/zulip_tools.py make_deploy_path)
    mv "$ZULIP_PATH" "$deploy_path"
    ln -nsf /home/zulip/deployments/next "$ZULIP_PATH"
    ln -nsf "$deploy_path" /home/zulip/deployments/next
    ln -nsf "$deploy_path" /home/zulip/deployments/current
    ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
    mkdir -p "$deploy_path"/prod-static/serve
    cp -rT "$deploy_path"/prod-static/serve /home/zulip/prod-static
    chown -R zulip:zulip /home/zulip /var/log/zulip /etc/zulip/settings.py

    if ! [ -e "/home/zulip/prod-static/generated" ]; then
        # If we're installing from a git checkout, we need to run
        # `tools/update-prod-static` in order to build the static
        # assets.
        su zulip -c '/home/zulip/deployments/current/tools/update-prod-static --authors-not-required'
    fi
fi

if [ -e "/var/run/supervisor.sock" ]; then
    chown zulip:zulip /var/run/supervisor.sock
fi

if [ -n "$NO_INIT_DB" ] || [ -n "$REMOTE_POSTGRES" ]; then
    set +x
    cat <<EOF

 Success!

 Stopping because --no-init-db or --remote-postgres was passed.
 To complete the installation, configure postgres and then run:

   su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database'
   su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
EOF
    exit 0
fi

su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database --quiet'

su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'