From d7b59c86cec2333979a86b7c89c1f5dd4e068363 Mon Sep 17 00:00:00 2001
From: Alex Vandiver <alexmv@zulip.com>
Date: Mon, 14 Mar 2022 20:24:21 +0000
Subject: [PATCH] puppet: Build wal-g from source for aarch64.

Since wal-g does not provide binaries for aarch64, build them from
source.  While building them from source for arm64 would better ensure
that build process is tested, the build process takes 7min and 700M of
temp files, which is an unacceptable cost; we thus only build on
aarch64.

Since the wal-g build process uses submodules, which are not in the
Github export, we clone the full wal-g repository.  Because the
repository is relatively small, we clone it anew on each new version,
rather than attempt to manage the remotes.

Fixes #21070.
---
 puppet/zulip/manifests/common.pp             |  9 ++--
 puppet/zulip/manifests/postgresql_backups.pp | 51 ++++++++++++++++----
 scripts/lib/build-wal-g                      | 35 ++++++++++++++
 3 files changed, 82 insertions(+), 13 deletions(-)
 create mode 100755 scripts/lib/build-wal-g

diff --git a/puppet/zulip/manifests/common.pp b/puppet/zulip/manifests/common.pp
index fa254baee1..9d8ddcfd8b 100644
--- a/puppet/zulip/manifests/common.pp
+++ b/puppet/zulip/manifests/common.pp
@@ -68,14 +68,15 @@ class zulip::common {
 
     # https://github.com/wal-g/wal-g/releases
     'wal-g' => {
-      'version' => '1.1.3-rc-with-build',
-      'sha256' => {
+      'version'       => '1.1.3-rc-with-build',
+      'sha256'        => {
         'amd64' => '109a80f4c019e0f1d52602e90d2a181eb844494ece2d099a149cf9204b71113e',
-        # No aarch64 builds
+        # aarch64 builds from source, below
       },
+      # This is a Git commit hash, not a sha256sum, for when building from source.
+      'git_commit_id' => '52f990fe87679ee4651fe3fb7629a2ac799f50c6',
     },
 
-
     ### zulip_ops packages
 
     # https://grafana.com/grafana/download?edition=oss
diff --git a/puppet/zulip/manifests/postgresql_backups.pp b/puppet/zulip/manifests/postgresql_backups.pp
index 828110c580..2eb44660f6 100644
--- a/puppet/zulip/manifests/postgresql_backups.pp
+++ b/puppet/zulip/manifests/postgresql_backups.pp
@@ -4,19 +4,52 @@ class zulip::postgresql_backups {
   include zulip::postgresql_common
 
   $wal_g_version = $zulip::common::versions['wal-g']['version']
+  $wal_g_binary_hash = $zulip::common::versions['wal-g']['sha256'][$::os['architecture']]
+  $wal_g_commit_id = $zulip::common::versions['wal-g']['git_commit_id']
   $bin = "/srv/zulip-wal-g-${wal_g_version}"
-  $package = "wal-g-pg-ubuntu-20.04-${zulip::common::goarch}"
 
-  # This tarball contains only a single file
-  zulip::external_dep { 'wal-g':
-    version        => $wal_g_version,
-    url            => "https://github.com/wal-g/wal-g/releases/download/v${wal_g_version}/${package}.tar.gz",
-    tarball_prefix => $package,
+  if $wal_g_binary_hash != undef {
+    # We have a binary for this arch
+    $package = "wal-g-pg-ubuntu-20.04-${zulip::common::goarch}"
+    # This tarball contains only a single file, which is extracted as $bin
+    zulip::external_dep { 'wal-g':
+      version        => $wal_g_version,
+      url            => "https://github.com/wal-g/wal-g/releases/download/v${wal_g_version}/${package}.tar.gz",
+      tarball_prefix => $package,
+      before         => File['/usr/local/bin/wal-g'],
+    }
+  } else {
+    include zulip::golang
+    $source_dir = "/srv/zulip-wal-g-src-${wal_g_version}"
+    exec { 'clone wal-g':
+      command => "git clone https://github.com/wal-g/wal-g.git --branch v${wal_g_version} ${source_dir}",
+      cwd     => '/srv',
+      creates => $source_dir,
+      require => Package['git'],
+    }
+    exec { 'compile wal-g':
+      command     => "${::zulip_scripts_path}/lib/build-wal-g ${wal_g_version} ${wal_g_commit_id}",
+      environment => ["GOBIN=${zulip::golang::dir}/bin"],
+      cwd         => $source_dir,
+      creates     => $bin,
+      require     => [
+        Zulip::External_Dep['golang'],
+        Exec['clone wal-g'],
+      ],
+      timeout     => 600,
+      before      => File['/usr/local/bin/wal-g'],
+    }
+    tidy { '/srv/zulip-wal-g-*':
+      path    => '/srv/',
+      recurse => 1,
+      rmdirs  => true,
+      matches => 'zulip-wal-g-*',
+      require => Exec['compile wal-g'],
+    }
   }
   file { '/usr/local/bin/wal-g':
-    ensure  => 'link',
-    target  => $bin,
-    require => Zulip::External_Dep['wal-g'],
+    ensure => 'link',
+    target => $bin,
   }
   # We used to install versions into /usr/local/bin/wal-g-VERSION,
   # until we moved to using Zulip::External_Dep which places them in
diff --git a/scripts/lib/build-wal-g b/scripts/lib/build-wal-g
new file mode 100755
index 0000000000..4e037245cb
--- /dev/null
+++ b/scripts/lib/build-wal-g
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eux
+
+apt-get install -y \
+    liblzo2-dev \
+    libbrotli-dev \
+    libsodium-dev \
+    build-essential \
+    gcc \
+    cmake \
+    libc-dev
+
+tmpdir="$(mktemp -d)"
+trap 'rm -r "$tmpdir"' EXIT
+
+export GOCACHE="$tmpdir/cache"
+export GOPATH="$tmpdir/build"
+
+src_dir="/srv/zulip-wal-g-src-$1"
+dst="/srv/zulip-wal-g-$1"
+
+cd "$src_dir"
+
+if [ "$(git rev-parse HEAD)" != "$2" ]; then
+    echo "Commit tag has changed; expected $2, got $(git rev-parse HEAD)"
+    exit 1
+fi
+
+export PATH="$PATH:$GOBIN"
+export USE_LZO=1
+export USE_LIBSODIUM=1
+make deps pg_build
+
+mv "main/pg/wal-g" "$dst"