provisioning: Use AWS CLI to automate provisioning

The previous steps for standing up a new host were somewhat manual.
This further scripts the process, by using the AWS CLI to start the
instance, and pass it a "user data" script to provision itself upon
boot.  This results in a hands-off provisioning process which
completes in 5min.

Additional settings are required for `~/.zulip-install-server.conf`.
It is not suited for all roles, as it assumes one instance type and
security group value.  Additionally, not all of the post-provision
process is currently automated -- Nagios SSH key verification, for
instance, is still a manual step.  There are also additional steps for
database or frontend servers.  Regardless, this is a move toward
automated provisioning.

tools/setup/bootstrap-aws-installer

@@ -0,0 +1,94 @@
+#!/bin/env bash
+
+# Prepended to this automatically are the following:
+#SERVER=
+#HOSTNAME=
+#ROLES=
+#REPO_URL=
+#BRANCH=
+#SSH_SECRET_ID=
+
+if ! curl -s -m 5 http://169.254.169.254/latest/dynamic/instance-identity/document | grep instanceId; then
+    echo "This should be run on AWS instances, not locally."
+    exit 1
+fi
+
+set -e
+set -x
+
+# Set the hostname early
+echo "$HOSTNAME" > /etc/hostname
+hostname "$HOSTNAME"
+sed -i "s/localhost$/localhost $HOSTNAME $SERVER/" /etc/hosts
+
+# Delete the ubuntu user
+userdel ubuntu
+
+# Make sure root doesn't have a password
+passwd -d root
+
+# Allow root logins
+sed -i 's/disable_root: true/disable_root: false/' /etc/cloud/cloud.cfg
+
+# Dependencies to install AWS CLI
+(
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get -qy update
+    apt-get -qy -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade
+    apt-get -qy install jq unzip wget
+    apt-get -qy autoclean
+)
+
+# The following line gets subbed in with the contents of bootstrap-awscli
+AWS=
+
+# Set up AWS so we can use the role credentials we were started with, which give secrets access
+mkdir -p /root/.aws
+cat >/root/.aws/config <<EOF
+[default]
+region = us-east-1
+output = text
+credential_source = Ec2InstanceMetadata
+EOF
+
+# Set up public keys for root, so we can fetch the repo; this is a
+# function so we do can it again later with the zulip user
+function install_keys() {
+    USERNAME="$1"
+    SSHDIR="$( getent passwd "$USERNAME" | cut -d: -f6 )/.ssh"
+    KEYDATA="$($AWS --output text \
+                    secretsmanager get-secret-value \
+                    --secret-id "$SSH_SECRET_ID" \
+                    --query SecretString)"
+    mkdir -p "$SSHDIR"
+    echo "$KEYDATA" | jq -r .public  | base64 -d > "$SSHDIR/id_rsa.pub"
+    echo "$KEYDATA" | jq -r .private | base64 -d > "$SSHDIR/id_rsa"
+    chown -R  "$USERNAME:$USERNAME" "$SSHDIR"
+    chmod 600 "$SSHDIR/id_rsa"
+}
+install_keys root
+
+# Provide GitHub known_hosts setup; you can verify against fingerprints at
+# https://docs.github.com/en/github/authenticating-to-github/githubs-ssh-key-fingerprints
+# via `ssh-keygen -lf`
+cat >/root/.ssh/known_hosts <<EOF
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+EOF
+
+cd /root
+git clone "$REPO_URL" zulip -b "$BRANCH"
+git -C zulip checkout "$BRANCH"
+
+(
+    VIRTUALENV_NEEDED=$(if echo "$ROLES" | grep -q app_frontend; then echo -n yes; else echo -n no; fi)
+    export VIRTUALENV_NEEDED
+    PUPPET_CLASSES="$ROLES"
+    export PUPPET_CLASSES
+    /root/zulip/scripts/setup/install \
+        --self-signed-cert \
+        --no-init-db
+)
+
+install_keys zulip
+
+reboot

tools/setup/bootstrap-awscli

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+AWS_CLI_VERSION="2.0.30"
+AWS_CLI_SHA="7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e"
+
+if [ ! -d "/srv/zulip-aws-tools/v2/$AWS_CLI_VERSION" ]; then
+    mkdir -p /srv/zulip-aws-tools
+    cd /srv/zulip-aws-tools || exit 1
+    rm -rf awscli.zip awscli.zip.sha256 aws/
+    wget -q "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -O awscli.zip
+    echo "$AWS_CLI_SHA  awscli.zip" > awscli.zip.sha256
+    sha256sum -c awscli.zip.sha256
+    unzip -q awscli.zip
+    (
+        cd ./aws || exit 1
+        ./install -i /srv/zulip-aws-tools -b /srv/zulip-aws-tools/bin -u
+    )
+    rm -rf awscli.zip awscli.zip.sha256 aws/
+fi
+
+# shellcheck disable=SC2034
+AWS="/srv/zulip-aws-tools/bin/aws"

tools/setup/install-aws-server

@@ -1,136 +1,120 @@
 #!/usr/bin/env bash
 set -e
 
-server=$1
+SERVER=$1
-type=$2
+ROLES=$2
-hostname=$3
+BRANCH=$3
-branch=$4
+if [ -z "$SERVER" ] || [ -z "$ROLES" ]; then
-if [ -z "$hostname" ]; then
+    echo "USAGE: $0 server roles [branch]"
-    echo "USAGE: $0 server type hostname [branch]"
+    echo
     echo "Installs an empty Ubuntu server in AWS with a Zulip server role."
-    echo "* hostname is the current hostname/IP of the server"
+    echo
-    echo "* type is a list of puppet rules to be passed to scripts/lib/install"
+    echo " * server is the local part of the hostname (e.g. postgres0)"
+    echo " * roles is a list of puppet rules to be passed to scripts/lib/install"
     echo "   E.g. 'zulip::base,zulip::apt_repository,zulip::postgres_common'"
-    echo "* hostname is to be the server's external hostname."
     echo " * branch is used to override the default branch to install from."
-    echo "Reads configuration from $HOME/.zulip-install-server.conf."
+    echo
-    exit 1
+    echo "Reads configuration from $HOME/.zulip-install-server.conf, which should look like:"
-fi
+    echo
-if ! echo "$hostname" | grep -q zulip; then
+    echo "[repo]"
-    echo "USAGE: $0 server type hostname [branch]"
+    echo "repo_url=git@github.com:zulip/zulip.git"
-    echo "Hostname must have zulip in it."
+    echo "branch=master"
+    echo "[aws]"
+    echo "zone_id=Z2U988IEXAMPLE"
+    echo "security_groups=sg-01234567"
+    echo "image_id=ami-0dc45e3d9be6ab7b5"
+    echo "instance_type=m4.large"
+    echo "ssh_secret_id=prod/git/deploy"
     exit 1
 fi
 set -x
 
-zulip_ssh_config_file="$HOME/.zulip-install-server.conf"
+cd "$(dirname "$0")"
-amazon_key_file=$(crudini --get "$zulip_ssh_config_file" ssh amazon_key_file)
+
-if ! [ -e "$amazon_key_file" ]; then
+source ./bootstrap-awscli
-    echo "You need the amazon ssh key at $amazon_key_file"
+
+zulip_install_config_file="$HOME/.zulip-install-server.conf"
+if [ ! -f "$zulip_install_config_file" ]; then
+    echo "No configuration file found in $zulip_install_config_file"
+    exit 1
+fi
+REPO_URL=$(crudini --get "$zulip_install_config_file" repo repo_url)
+if [ -z "$BRANCH" ]; then
+    BRANCH=$(crudini --get "$zulip_install_config_file" repo default_branch)
+fi
+
+AWS_ZONE_ID=$(    crudini --get "$zulip_install_config_file" aws zone_id)
+SECURITY_GROUPS=$(crudini --get "$zulip_install_config_file" aws security_groups)
+AMI_ID=$(         crudini --get "$zulip_install_config_file" aws image_id)
+INSTANCE_TYPE=$(  crudini --get "$zulip_install_config_file" aws instance_type)
+SSH_SECRET_ID=$(  crudini --get "$zulip_install_config_file" aws ssh_secret_id)
+
+# Verify it doesn't exist already
+ZONE_NAME=$($AWS route53 get-hosted-zone --id "$AWS_ZONE_ID" | jq -r '.HostedZone.Name' )
+HOSTNAME="$SERVER.${ZONE_NAME%?}"  # Remove trailing .
+EXISTING_RECORDS=$($AWS route53 list-resource-record-sets \
+                        --hosted-zone-id "$AWS_ZONE_ID" \
+                        --query "ResourceRecordSets[?Name == '$HOSTNAME.']" \
+                       | jq '. | length')
+if [ "$EXISTING_RECORDS" != "0" ]; then
+    echo "$HOSTNAME already exists!"
     exit 1
 fi
 
-server_private_key_file=$(crudini --get "$zulip_ssh_config_file" ssh server_private_key_file)
+# Build up the provisioning script
-if ! [ -e "$server_private_key_file" ]; then
+BOOTDATA=$(mktemp)
-    echo "You need a server ssh key at $server_private_key_file"
+{
-    exit 1
+    echo "#!/bin/bash"
-fi
+    echo "SERVER=$SERVER"
+    echo "HOSTNAME=$HOSTNAME"
+    echo "ROLES=$ROLES"
+    echo "REPO_URL=$REPO_URL"
+    echo "BRANCH=$BRANCH"
+    echo "SSH_SECRET_ID=$SSH_SECRET_ID"
+    sed '/^AWS=/ r ./bootstrap-awscli' bootstrap-aws-installer
+} >> "$BOOTDATA"
 
-if [ -n "${zulip_confdir-}" ]; then
+TAGS="[{Key=Name,Value=$SERVER},{Key=role,Value=\"$ROLES\"}]"
-    zulipconf_file="$zulip_confdir/zulip.conf"
+INSTANCE_DATA=$($AWS ec2 run-instances \
-    secrets_file="$zulip_confdir/zulip-secrets.conf"
+                     --iam-instance-profile 'Name="EC2ProdInstance"' \
-    settings_file="$zulip_confdir/settings.py"
+                     --image-id "$AMI_ID" \
-fi
+                     --instance-type "$INSTANCE_TYPE" \
-if [ -z "$secrets_file" ]; then
+                     --security-group-ids "$SECURITY_GROUPS" \
-    echo "Specify secrets_file via environment."
+                     --tag-specifications "ResourceType=instance,Tags=$TAGS" \
-    exit 1
+                     --user-data "file://$BOOTDATA")
-fi
+INSTANCEID=$(echo "$INSTANCE_DATA" | jq -r .Instances[0].InstanceId)
 
-zulip_repo=$(crudini --get "$zulip_ssh_config_file" repo repo_url)
+# Wait for public IP assignment
+PUBLIC_DNS_NAME=""
+while [ -z "$PUBLIC_DNS_NAME" ]; do
+    sleep 1
+    PUBLIC_DNS_NAME=$($AWS ec2 describe-instances --instance-ids "$INSTANCEID" \
+                          | jq -r .Reservations[0].Instances[0].PublicDnsName )
+done
 
-if [ -z "$branch" ]; then
+# Add the hostname to the zone
-    branch=$(crudini --get "$zulip_ssh_config_file" repo default_branch)
+ROUTE53_CHANGES=$(mktemp)
-fi
+cat > "$ROUTE53_CHANGES" <<EOF
-
+{
-VIRTUALENV_NEEDED=$(if echo "$type" | grep -q app_frontend; then echo -n yes; else echo -n no; fi)
+    "Comment": "Add the $HOSTNAME CNAME record",
-
+    "Changes": [
-# Force RSA keys. We do this because the ECDSA key is not printed on syslog,
+        {
-# and our puppet configuration does not use ECDSA. If we don't do this,
+            "Action": "CREATE",
-# we'll get key errors after puppet apply.
+            "ResourceRecordSet": {
-SSH_OPTS=(-o 'HostKeyAlgorithms=ssh-rsa')
+                "Name": "$HOSTNAME",
-
+                "Type": "CNAME",
-set +e
+                "TTL": 300,
-
+                "ResourceRecords": [{"Value": "$PUBLIC_DNS_NAME"}]
-ssh "${SSH_OPTS[@]}" "$server" -i "$amazon_key_file" -lubuntu -o "ControlMaster=no" /bin/bash <<EOF
+            }
-sudo mkdir -p ~root/.ssh && sudo cp .ssh/authorized_keys /root/.ssh/authorized_keys
+        }
-sudo sed -i 's/disable_root: true/disable_root: false/' /etc/cloud/cloud.cfg
+    ]
-sudo mkdir -p /etc/zulip
+}
-EOF
-# shellcheck disable=SC2029 disable=SC2087
-ssh "${SSH_OPTS[@]}" "$server" -i "$amazon_key_file" -lroot /bin/bash <<EOF
-# Set the hostname early
-echo "$hostname" > /etc/hostname
-hostname "$hostname"
-sed -i 's/localhost$/localhost $hostname/' /etc/hosts
 EOF
+$AWS route53 change-resource-record-sets --hosted-zone-id "$AWS_ZONE_ID" --change-batch "file://$ROUTE53_CHANGES"
+rm "$ROUTE53_CHANGES"
 
-set -e
-
-# Give server its SSH keys
-scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$server_private_key_file" root@"$server":/root/.ssh/id_rsa
-scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$server_private_key_file".pub root@"$server":/root/.ssh/id_rsa.pub
-scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$secrets_file" root@"$server":/etc/zulip/zulip-secrets.conf
-if [ -e "$zulipconf_file" ]; then
-    scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$zulipconf_file" root@"$server":/etc/zulip/zulip.conf
-fi
-if [ -e "$settings_file" ]; then
-    scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$settings_file" root@"$server":/etc/zulip/settings.py
-fi
-
-# shellcheck disable=SC2029 disable=SC2087
-ssh "${SSH_OPTS[@]}" "$server" -i "$amazon_key_file" -lroot /bin/bash <<EOF
-set -x
-# Finish setting up the SSH private key
-chmod 600 /root/.ssh/id_rsa
-# Delete the ubuntu user
-if grep -q '^ubuntu:' /etc/passwd; then
-    userdel ubuntu
-fi
-# Make sure root doesn't have a password
-passwd -d root
-
-apt-get update
-apt-get -y upgrade
-
-cd "\$(mktemp -d)"
-
-# Get GitHub known_hosts setup
-# TODO: Replace this with hardcoding the actual GitHub keys
-ssh -oStrictHostKeyChecking=no git@github.com </dev/null >/dev/null || true
-
-if ! [ -e "zulip" ]; then
-    # need to install git to clone the repo
-    apt-get install -y git crudini
-    git clone "$zulip_repo" zulip
-fi
-cd zulip
-git fetch
-git checkout origin/$branch
-# The main Zulip production install script can take things from here!
-env VIRTUALENV_NEEDED=$VIRTUALENV_NEEDED PUPPET_CLASSES="$type" \
-  ./scripts/setup/install --self-signed-cert --no-init-db --no-overwrite-settings
-EOF
-
-scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$server_private_key_file" root@"$server":/home/zulip/.ssh/id_rsa
-scp "${SSH_OPTS[@]}" -i "$amazon_key_file" "$server_private_key_file".pub root@"$server":/home/zulip/.ssh/id_rsa.pub
-# shellcheck disable=SC2029
-ssh "${SSH_OPTS[@]}" "$server" -i "$amazon_key_file" -lroot /bin/bash <<EOF
-chown zulip:zulip /home/zulip/.ssh/id_rsa*
-chmod 600 /home/zulip/.ssh/id_rsa*
-EOF
 set +x
-cat <<EOF
+echo
-
+echo
- Done.
+echo ">>> Install started successfully!  Provisioning takes 5-6min."
-
+echo "    sleep 360 && ssh root@$HOSTNAME"
-EOF