kandra: Automate the second step of configuring database replicas.

If there is a replication primary configured, and no current database,
then we check all of the required secrets are in place, then pull down
the latest backup and trigger a PostgreSQL restart, which will pick up
downloading the remaining WAL logs to catch up, then start streaming
from the configured primary.

puppet/kandra/files/postgresql/setup_data.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -eux
+
+service postgresql stop
+
+cert_file="$(crudini --get /etc/zulip/zulip.conf postgresql ssl_cert_file)"
+if [ -z "$cert_file" ] || [ ! -f "$cert_file" ]; then
+    echo "Certificate file is not set or does not exist!"
+    exit 1
+fi
+
+key_file="$(crudini --get /etc/zulip/zulip.conf postgresql ssl_key_file)"
+if [ -z "$key_file" ] || [ ! -f "$key_file" ]; then
+    echo "Key file is not set or does not exist!"
+    exit 1
+fi
+
+cert_cn="$(openssl x509 -noout -subject -in "$cert_file" | sed -n '/^subject/s/^.*CN\s*=\s*//p')"
+
+if [ "$cert_cn" != "$(hostname)" ]; then
+    echo "Configured certificate does not match host!"
+    exit 1
+fi
+
+echo "Checking for S3 secrets..."
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_region >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_bucket >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_key >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_secret_key >/dev/null
+
+if [ ! -f "/var/lib/postgresql/.postgresql/postgresql.crt" ]; then
+    echo "Replication certificate file is not set or does not exist!"
+    exit 1
+fi
+if [ ! -f "/var/lib/postgresql/.postgresql/postgresql.key" ]; then
+    echo "Replication key file is not set or does not exist!"
+    exit 1
+fi
+
+version="$(crudini --get /etc/zulip/zulip.conf postgresql version)"
+mkdir -p "/srv/data/postgresql/$version"
+chown postgres.postgres "/srv/data/postgresql/$version"
+chmod 700 "/srv/data/postgresql/$version"
+
+/usr/local/bin/env-wal-g backup-fetch "/var/lib/postgresql/$version/main" LATEST
+chown -R postgres.postgres "/var/lib/postgresql/$version/main"

puppet/kandra/manifests/profile/postgresql.pp

@@ -34,6 +34,25 @@ class kandra::profile::postgresql inherits kandra::profile::base {
     unless  => 'test /var/lib/postgresql/ -ef /srv/data/postgresql/',
   }
 
+  # This is the second stage, after secrets are configured
+  $replication_primary = zulipconf('postgresql', 'replication_primary', undef)
+  if $replication_primary != undef {
+      file { '/root/setup_data.sh':
+        ensure => file,
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0744',
+        source => 'puppet:///modules/kandra/postgresql/setup_data.sh',
+      }
+      exec { 'setup_data':
+        command => '/root/setup_data.sh',
+        require => [File['/usr/local/bin/env-wal-g'], Exec['setup_disks']],
+        unless  => "test -d /srv/data/postgresql/${zulip::postgresql_common::version}/main",
+        timeout => 0,
+        notify  => Exec[$zulip::postgresql_base::postgresql_restart],
+      }
+  }
+
   file { "${zulip::postgresql_base::postgresql_confdir}/pg_hba.conf":
     ensure  => file,
     require => Package["postgresql-${zulip::postgresql_common::version}"],