zulip/tools/travis/production-helper

#!/bin/bash
set -e
set -x

apt-get install -y openssl ssl-cert
ln -s /etc/ssl/certs/ssl-cert-snakeoil.pem /etc/ssl/certs/zulip.combined-chain.crt
ln -s /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/private/zulip.key

tar -xf zulip-server-travis.tar.gz
mv zulip-server-travis /root/zulip

# Do an apt upgrade to start with an up-to-date machine
export APT_OPTIONS="-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold"
apt-get update
# Hold upgrades to packages which are expensive to upgrade due to size
# or computational cost (e.g. initramfs rebuilds) and aren't really
# used by Zulip in production.
apt-mark hold initramfs-tools initramfs-tools-bin oracle-java8-installer udev linux-image-3.19.0-28-generic linux-image-generic-lts-vivid base-files linux-firmware chromium-browser google-chrome-stable g++-4.8 gcc-4.8 cpp-4.8 openjdk-6-jre-headless openjdk-7-jre-headless
apt-get upgrade -y $APT_OPTIONS
# Install Zulip
env TRAVIS=1 /root/zulip/scripts/setup/install

cat >>/etc/zulip/settings.py <<EOF
# Travis CI override settings above
EXTERNAL_HOST = '127.0.0.1'
ZULIP_ADMINISTRATOR = 'zulip-travis-admin@travis.example.com'
ADMIN_DOMAIN = 'travis.example.com'
AUTHENTICATION_BACKENDS = ( 'zproject.backends.EmailAuthBackend', )
NOREPLY_EMAIL_ADDRESS = 'noreply@travis.example.com'
DEFAULT_FROM_EMAIL = "Zulip <zulip@travis.example.com>"
EOF

su zulip -c /home/zulip/deployments/current/scripts/setup/initialize-database

# TODO: Ideally this would test actually logging in, but this is a start.
echo; echo "Now testing that the newly installed server's homepage loads"; echo

wget https://localhost -O /tmp/index.html --no-check-certificate -S 2> /tmp/wget-output
grep -vi '\(Vary\|Content-Language\|expires\|modified\|saved\|[.][.][.]\|Date\|[-][-]\)' /tmp/wget-output > /tmp/http-headers-processed
if ! diff -ur /tmp/http-headers-processed ~/success-http-headers.txt; then
    set +x
    echo
    echo "FAILURE: The HTTP Headers returned from loading the homepage on the server do not match the contents of tools/travis/success-http-headers.txt.  Typically, this means that the server threw a 500 when trying to load the homepage."
    echo "Displaying the contents of the server's error log:"
    echo
    cat /var/log/zulip/errors.log
    exit 1
fi

echo; echo "Now testing that the supervisord jobs are running properly"; echo
sleep 10 # Guaranteed to have a working supervisord process get an extra digit
if supervisorctl status | grep -vq RUNNING || supervisorctl status | sed 's/^.*uptime //' | grep -q 0:00:0; then
    set +x
    echo
    echo "FAILURE: Supervisor output shows daemons are crashing:"
    echo
    supervisorctl status
    echo
    echo "DEBUG: printing Zulip server's error log:"
    cat /var/log/zulip/errors.log
    echo
    echo "DEBUG: printing Zulip server's workers log:"
    cat /var/log/zulip/workers.log
    echo
    echo "DEBUG: printing Zulip server's tornado log:"
    cat /var/log/zulip/tornado.log
    exit 1
fi

echo; echo "Now running RabbitMQ consumer Nagios tests"; echo
for consumer in notify_tornado user_activity user_activity_interval user_presence invites signups message_sender feedback_messages error_reports digest_emails email_mirror missedmessage_mobile_notifications; do
    if ! /home/zulip/deployments/current/scripts/nagios/write-rabbitmq-consumers-state-file "$consumer"; then
        # Temporary section while we're debugging why this fails nondeterministically in CI
        STATE_DIR=/var/lib/nagios_state
        ls "$STATE_DIR"
        cat "$STATE_DIR"/*
        /home/zulip/deployments/current/scripts/nagios/write-rabbitmq-consumers-state-file "$consumer"
    fi
    if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_rabbitmq_consumers "$consumer"; then
        set +x
        echo
        echo "FAILURE: Missing Nagios consumer for $consumer; displaying full consumer output:"
        rabbitmqctl list_consumers
        echo
        exit 1
    fi
done

echo; echo "Now running additional Nagios tests"; echo
if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_queue_worker_errors || \
   ! su zulip -c /usr/lib/nagios/plugins/zulip_postgres_appdb/check_fts_update_log || \
   ! /usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://127.0.0.1/api --nagios --insecure; then
    set +x
    echo
    echo "FAILURE: Nagios checks don't pass:"
    echo
    echo "DEBUG: printing Zulip server's error log:"
    cat /var/log/zulip/errors.log
    exit 1
fi
echo "Production installation test successful!"
exit 0