zulip/scripts/lib/install

#!/usr/bin/env bash
set -e

usage() {
    cat <<EOF
Usage:
  install [--hostname=zulip.example.com] [--email=admin@example.com] [options...]
  install --help

Other options:
  --certbot (requires --hostname and --email)
  --self-signed-cert
EOF
    exit 0
};

# Shell option parsing.  Over time, we'll want to move some of the
# environment variables below into this self-documenting system.
args="$(getopt -o '' --long help,self-signed-cert,certbot,hostname:,email:,express -n "$0" -- "$@")"
eval "set -- $args"
while true; do
    case "$1" in
        --help)
            show_help=1
            shift
            ;;
        --self-signed-cert)
            SELF_SIGNED_CERT=1
            shift
            ;;
        --certbot)
            USE_CERTBOT=1
            shift
            ;;
        --hostname)
            EXTERNAL_HOST="$2"
            shift
            shift
            ;;
        --email)
            ZULIP_ADMINISTRATOR="$2"
            shift
            shift
            ;;
        --express)  # experimental, not documented
            EXPRESS_SETUP=1
            shift
            ;;
        --)
            shift
            break
            ;;
    esac
done

if [ "$#" -gt 0 ]; then
    usage
fi

if [ -n "$show_help" ]; then
    usage
fi

if [ -n "$SELF_SIGNED_CERT" ] && [ -n "$USE_CERTBOT" ]; then
    echo "error: --self-signed-cert and --certbot are incompatible" >&2
    echo >&2
    usage
fi

if [ -n "$USE_CERTBOT" ] \
       && { [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; }; then
    usage
fi

# Do set -x after option parsing is complete
set -x

# Force a known locale.  Some packages on PyPI fail to install in some locales.
export LC_ALL="en_US.UTF-8"

# Specify options for apt.
APT_OPTIONS="${APT_OPTIONS:-}"
# Install additional packages using apt.
ADDITIONAL_PACKAGES=${ADDITIONAL_PACKAGES:-}
# Deployment type is almost always voyager.
DEPLOYMENT_TYPE="${DEPLOYMENT_TYPE:-voyager}"
# Comma-separated list of puppet manifests to install.  default is
# zulip::voyager for an all-in-one system or zulip::dockervoyager for
# Docker.  Use e.g. zulip::app_frontend for a Zulip frontend server.
PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::voyager}"
VIRTUALENV_NEEDED="${VIRTUALENV_NEEDED:-yes}"

# Check for at least ~1.9GB of RAM before starting installation;
# otherwise users will find out about insufficient RAM via weird
# errors like a segfault running `pip install`.
mem_kb=$(cat /proc/meminfo | head -n1 | awk '{print $2}')
if [ "$mem_kb" -lt 1900000 ]; then
    echo "Insufficient RAM.  Zulip requires at least 2GB of RAM."
    exit 1
fi

if ! [ -e /usr/bin/realpath ]; then
    # realpath is in coreutils on Xenial, but not in Trusty
    apt-get install -y realpath
fi

ZULIP_PATH="$(realpath $(dirname $0)/../..)"

# setup-apt-repo does an `apt-get update`
"$ZULIP_PATH"/scripts/lib/setup-apt-repo

# Handle issues around upstart on Ubuntu Xenial
"$ZULIP_PATH"/scripts/lib/check-upstart

# Check early for missing SSL certificates
if [ "$PUPPET_CLASSES" = "zulip::voyager" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then
   set +x
   echo
   echo "Could not find SSL certificates!"
   for f in "/etc/ssl/private/zulip.key" "/etc/ssl/certs/zulip.combined-chain.crt"; do
       [ -e "$f" ] || echo " - $f is missing!"
   done
   echo "See https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html for help."
   echo "For non-production testing, try the --self-signed-cert option."
   echo
   echo "Once fixed, just rerun scripts/setup/install; it'll pick up from here!"
   echo
   exit 1
fi

apt-get -y dist-upgrade $APT_OPTIONS
apt-get install -y \
    puppet git curl \
    python python3 python-six python3-six crudini \
    $ADDITIONAL_PACKAGES

if [ -n "$USE_CERTBOT" ]; then
    "$ZULIP_PATH"/scripts/setup/setup-certbot \
        --no-zulip-conf --method=standalone \
        --hostname "$EXTERNAL_HOST" --email "$ZULIP_ADMINISTRATOR"
fi

# Create and activate a virtualenv
if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
    "$ZULIP_PATH"/scripts/lib/create-production-venv "$ZULIP_PATH"
fi

"$ZULIP_PATH"/scripts/lib/install-node

# puppet apply
mkdir -p /etc/zulip
(
    echo -e "[machine]\npuppet_classes = $PUPPET_CLASSES\ndeploy_type = $DEPLOYMENT_TYPE";

    # Note: there are four dpkg-query outputs to consider:
    #
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # root@host# apt install rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # install ok installed
    # root@host# apt remove rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # deinstall ok config-files
    # root@host# apt purge rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # unknown ok not-installed
    #
    # (There are more possibilities in the case of dpkg errors.)  Here
    # we are checking for either empty or not-installed.
    if [ -n "$TRAVIS" ] || ! dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null | grep -vq ' not-installed$'; then
        echo -e "\n[rabbitmq]\nnodename = zulip@localhost"
    fi

    if [ -n "$USE_CERTBOT" ]; then
        echo -e "\n[certbot]\nauto_renew = yes"
    fi
) > /etc/zulip/zulip.conf
"$ZULIP_PATH"/scripts/zulip-puppet-apply -f

# Detect which features were selected for the below
set +e
[ -e "/etc/init.d/camo" ]; has_camo=$?
[ -e "/etc/init.d/nginx" ]; has_nginx=$?
[ -e "/etc/supervisor/conf.d/zulip.conf" ]; has_appserver=$?
[ -e "/etc/cron.d/rabbitmq-numconsumers" ]; has_rabbit=$?
[ -e "/etc/init.d/postgresql" ]; has_postgres=$?
set -e

# Docker service setup is done in the docker config, not here
if [ "$DEPLOYMENT_TYPE" = "dockervoyager" ]; then
    has_camo=1
    has_nginx=1
    has_appserver=1
    has_rabbit=1
    has_postgres=1
fi

# These server restarting bits should be moveable into puppet-land, ideally
apt-get -y upgrade

if [ "$has_nginx" = 0 ]; then
    if [ -n "$SELF_SIGNED_CERT" ]; then
        "$ZULIP_PATH"/scripts/setup/generate-self-signed-cert --exists-ok "${EXTERNAL_HOST:-$(hostname)}"
    fi

    # Check nginx was configured properly now that we've installed it.
    # Most common failure mode is certs not having been installed.
    nginx -t || (
        set +x
        echo
        echo "Verifying the Zulip nginx configuration failed!"
        echo
        echo "This is almost always a problem with your SSL certificates."
        echo "See https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html for help"
        echo
        echo "Once fixed, just rerun scripts/setup/install; it'll pick up from here!"
        echo
        exit 1
    )
    service nginx restart
fi

if [ "$has_appserver" = 0 ]; then
    "$ZULIP_PATH"/scripts/setup/generate_secrets.py --production
    cp -a "$ZULIP_PATH"/zproject/prod_settings_template.py /etc/zulip/settings.py
    if [ -n "$EXTERNAL_HOST" ]; then
        sed -i "s/^EXTERNAL_HOST =.*/EXTERNAL_HOST = '$EXTERNAL_HOST'/" /etc/zulip/settings.py
    fi
    if [ -n "ZULIP_ADMINISTRATOR" ]; then
        sed -i "s/^ZULIP_ADMINISTRATOR =.*/ZULIP_ADMINISTRATOR = '$ZULIP_ADMINISTRATOR'/" /etc/zulip/settings.py
    fi
    ln -nsf /etc/zulip/settings.py "$ZULIP_PATH"/zproject/prod_settings.py
fi

# Restart camo since generate_secrets.py likely replaced its secret key
if [ "$has_camo" = 0 ]; then
    # Cut off stdin because a bug in the Debian packaging for camo
    # causes our stdin to leak to the daemon, which can cause tools
    # invoking the installer to hang.
    # TODO: fix in Debian too.
    service camo restart </dev/null
fi

if [ "$has_rabbit" = 0 ]; then
    if ! rabbitmqctl status >/dev/null; then
        set +x
        echo; echo "RabbitMQ seems to not have started properly after the installation process."
        echo "Often, this can be caused by misconfigured /etc/hosts in virtualized environments"
        echo "See https://github.com/zulip/zulip/issues/53#issuecomment-143805121"
        echo "for more information"
        echo
        set -x
        exit 1
    fi
    "$ZULIP_PATH"/scripts/setup/configure-rabbitmq
fi

if [ "$has_postgres" = 0 ]; then
    "$ZULIP_PATH"/scripts/setup/postgres-init-db
fi

if [ "$has_appserver" = 0 ]; then
    deploy_path=$("$ZULIP_PATH"/scripts/lib/zulip_tools.py make_deploy_path)
    mv "$ZULIP_PATH" "$deploy_path"
    ln -nsf /home/zulip/deployments/next "$ZULIP_PATH"
    ln -nsf "$deploy_path" /home/zulip/deployments/next
    ln -nsf "$deploy_path" /home/zulip/deployments/current
    ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
    mkdir -p "$deploy_path"/prod-static/serve
    cp -rT "$deploy_path"/prod-static/serve /home/zulip/prod-static
    chown -R zulip:zulip /home/zulip /var/log/zulip /etc/zulip/settings.py

    if ! [ -e "/home/zulip/prod-static/generated" ]; then
        # If we're installing from a git checkout, we need to run
        # `tools/update-prod-static` in order to build the static
        # assets.
        su zulip -c "/home/zulip/deployments/current/tools/update-prod-static --authors-not-required"
    fi
fi

if [ -e "/var/run/supervisor.sock" ]; then
    # If supervisor isn't running, no need to chown its socket
    chown zulip:zulip /var/run/supervisor.sock
fi

if [ -z "$EXPRESS_SETUP" ]; then
    set +x
    cat <<EOF

 Installation complete!

 Now edit /etc/zulip/settings.py and fill in the mandatory values.

 Once you've done that, please run:

 su zulip -c /home/zulip/deployments/current/scripts/setup/initialize-database

 To configure the initial database.
EOF
    exit 0
fi

# TODO suppress instructions on success
sudo -u zulip /home/zulip/deployments/current/scripts/setup/initialize-database

sudo -u zulip /home/zulip/deployments/current/manage.py generate_realm_creation_link