install-yarn: Rewrite Yarn installer.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

docs/subsystems/dependencies.md

@@ -244,17 +244,14 @@ reasoning here.
 
 These are installed by `scripts/lib/install-node` (which in turn uses
 the standard third-party `nvm` installer to download `node` and pin
-its version) and `scripts/lib/third/install-yarn.sh` (the standard
+its version) and `scripts/lib/install-yarn`.
-installer for `yarn`, modified to support installing to a path that is
-not the current user's home directory).
 
 * `nvm` has its own system for installing each version of `node` at
 its own path, which we use, though we install a `/usr/local/bin/node`
 wrapper to access the desired version conveniently and efficiently
 (`nvm` has a lot of startup overhead).
-* `install-yarn.sh` is configured to install `yarn` at
+* We install `yarn` at `/srv/zulip-yarn`.  We don't do anything
-`/srv/zulip-yarn`.  We don't do anything special to try to manage
+special to try to manage multiple versions of `yarn`.
-multiple versions of `yarn`.
 
 ## ShellCheck and shfmt
 

scripts/lib/install

@@ -381,6 +381,7 @@ if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
 fi
 
 "$ZULIP_PATH"/scripts/lib/install-node
+"$ZULIP_PATH"/scripts/lib/install-yarn
 
 # Generate /etc/zulip/zulip.conf .
 mkdir -p /etc/zulip

scripts/lib/install-node

@@ -1,11 +1,7 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
-ZULIP_PATH="$(dirname "$0")/../.."
-ZULIP_SRV="/srv"
-YARN_PACKAGE_JSON="$ZULIP_SRV/zulip-yarn/package.json"
 node_version=14.17.0
-yarn_version=1.22.10
 nvm_version=0.38.0
 
 # This is a fix for the fact that nvm uses $HOME to determine which
@@ -18,13 +14,8 @@ if node_wrapper_path="$(command -v node)"; then
     current_node_version="$(node --version)"
 fi
 
-current_yarn_version="none"
+if [ "$current_node_version" = "v$node_version" ] && [ -L "$node_wrapper_path" ]; then
-if [ -e "$YARN_PACKAGE_JSON" ]; then
+    echo "Node version $node_version is already installed."
-    current_yarn_version=$(jq -r '.version' "$YARN_PACKAGE_JSON")
-fi
-
-if [ "$current_yarn_version" = "$yarn_version" ] && [ "$current_node_version" = "v$node_version" ] && [ -L "$node_wrapper_path" ]; then
-    echo "Node version $node_version and yarn version $yarn_version are already installed."
     exit 0
 fi
 
@@ -62,12 +53,3 @@ if [ "$current_node_version" != "v$node_version" ] || ! [ -L "$node_wrapper_path
     ln -nsf "$(dirname "$NODE_BIN")/npm" /usr/local/bin/npm
     ln -nsf "$(dirname "$NODE_BIN")/npx" /usr/local/bin/npx
 fi
-
-# Work around the fact that apparently sudo doesn't clear the HOME
-# environment variable in some cases; we don't want root
-# accessing/storing yarn configuration in the non-root user's home
-# directory.
-export HOME=/root
-
-# Install yarn if not installed
-bash "$ZULIP_PATH/scripts/lib/third/install-yarn.sh" "$ZULIP_SRV" --version "$yarn_version"

scripts/lib/install-yarn

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version=1.22.10
+sha256=05a22fff30d7d8e8005bed277bf20d55111ba2bed65a6b91a0fcd1307b71fd8d
+tarball="yarn-$version.tgz"
+
+check_version() {
+    # Reading the version of Yarn from its package.json is much faster
+    # than running yarn --version.
+    link="$(command -v yarn)" \
+        && bin="$(readlink -f "$link")" \
+        && current_version="$(jq -r '.version' "${bin%/*/*}/package.json")" \
+        && [ "$current_version" = "$version" ]
+}
+
+if ! check_version; then
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -LO "https://registry.npmjs.org/yarn/-/$tarball"
+    sha256sum -c <<<"$sha256 $tarball"
+    rm -rf /srv/zulip-yarn
+    mkdir /srv/zulip-yarn
+    tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-yarn
+    ln -nsf /srv/zulip-yarn/bin/yarn /usr/bin/yarn
+    check_version
+fi

scripts/lib/setup_venv.py

@@ -33,7 +33,7 @@ VENV_DEPENDENCIES = [
     # because we don't have another place that we install apt packages
     # on upgrade of a production server, and it's not worth adding
     # another call to `apt install` for.
-    "jq",  # Used by scripts/lib/install-node to check yarn version
+    "jq",  # Used by scripts/lib/install-yarn to check yarn version
     "libsasl2-dev",  # For building python-ldap from source
 ]
 

scripts/lib/third/install-yarn.sh

@@ -1,162 +0,0 @@
-#!/bin/sh
-set -e
-
-#####################
-# install-yarn.sh was patched to install yarn in a custom directory.
-# The following changes were made:
-# * yarn_link now just simlinks to /usr/bin
-# * yarn_detect_profile was removed
-# * Paths were changed to variables declared at the top
-# * Most of the non error coloration was removed to not distract during installs.
-# #######################
-
-reset="\033[0m"
-red="\033[31m"
-yellow="\033[33m"
-gpg_key=9D41F3C3
-
-ZULIP_ROOT="$1"
-YARN_DIR_NAME="zulip-yarn"
-YARN_DIR="$ZULIP_ROOT/$YARN_DIR_NAME"
-YARN_BIN="$YARN_DIR/bin/yarn"
-
-yarn_get_tarball() {
-  printf "Downloading tarball...\n"
-  if [ "$1" = '--nightly' ]; then
-    url=https://nightly.yarnpkg.com/latest.tar.gz
-  elif [ "$1" = '--rc' ]; then
-    url=https://yarnpkg.com/latest-rc.tar.gz
-  elif [ "$1" = '--version' ]; then
-    # Validate that the version matches MAJOR.MINOR.PATCH to avoid garbage-in/garbage-out behavior
-    version=$2
-    if echo $version | grep -qE "^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$"; then
-      url="https://yarnpkg.com/downloads/$version/yarn-v$version.tar.gz"
-    else
-      printf "$red> Version number must match MAJOR.MINOR.PATCH.$reset\n"
-      exit 1;
-    fi
-  else
-    url=https://yarnpkg.com/latest.tar.gz
-  fi
-  # Get both the tarball and its GPG signature
-  tarball_tmp=`mktemp -t yarn.tar.gz.XXXXXXXXXX`
-  if curl --fail -L -o "$tarball_tmp#1" "$url{,.asc}"; then
-    yarn_verify_integrity $tarball_tmp
-
-    printf "Extracting to $YARN_DIR...\n"
-    mkdir "$YARN_DIR_NAME"
-    tar zxf $tarball_tmp -C "$YARN_DIR_NAME" --strip 1 # extract tarball
-    rm $tarball_tmp*
-  else
-    printf "$red> Failed to download $url.$reset\n"
-    exit 1;
-  fi
-}
-
-# Verifies the GPG signature of the tarball
-yarn_verify_integrity() {
-  # Check if GPG is installed
-  if [[ -z "$(command -v gpg)" ]]; then
-    printf "$yellow> WARNING: GPG is not installed, integrity cannot be verified!$reset\n"
-    return
-  fi
-
-  if [ "$YARN_GPG" == "no" ]; then
-    printf "WARNING: Skipping GPG integrity check!\n"
-    return
-  fi
-
-  printf "Verifying integrity...\n"
-  # Grab the public key if it doesn't already exist
-  # Zulip patch: Fix the fact that Yarn has extended this keyring and we should always redownload.
-  curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --import
-
-  if [ ! -f "$1.asc" ]; then
-    printf "$red> Could not download GPG signature for this Yarn release. This means the release cannot be verified!$reset\n"
-    yarn_verify_or_quit "> Do you really want to continue?"
-    return
-  fi
-
-  # Actually perform the verification
-  if gpg --verify "$1.asc" $1; then
-    printf "GPG signature looks good\n"
-  else
-    printf "$red> GPG signature for this Yarn release is invalid! This is BAD and may mean the release has been tampered with. It is strongly recommended that you report this to the Yarn developers.$reset\n"
-    yarn_verify_or_quit "> Do you really want to continue?"
-  fi
-}
-
-yarn_link() {
-  printf "Adding to /usr/bin\n"
-
-  version=`$YARN_BIN --version` || (
-    printf "$red> Yarn was installed, but doesn't seem to be working :(.$reset\n"
-    exit 1;
-  )
-
-  ln -nsf "$YARN_BIN" /usr/bin/yarn
-
-  printf "Successfully installed Yarn $version!\n"
-}
-
-
-yarn_reset() {
-  unset -f yarn_install yarn_reset yarn_get_tarball yarn_link yarn_verify_integrity yarn_verify_or_quit
-}
-
-yarn_install() {
-  printf "Installing Yarn!\n"
-
-  if [ -d "$YARN_DIR" ]; then
-    if [ -e "$YARN_BIN" ] ; then
-      local latest_url
-      local specified_version
-      local version_type
-      if [ "$1" = '--nightly' ]; then
-        latest_url=https://nightly.yarnpkg.com/latest-tar-version
-        specified_version=`curl -sS $latest_url`
-        version_type='latest'
-      elif [ "$1" = '--version' ]; then
-        specified_version=$2
-        version_type='specified'
-      elif [ "$1" = '--rc' ]; then
-        latest_url=https://yarnpkg.com/latest-rc-version
-        specified_version=`curl -sS $latest_url`
-        version_type='rc'
-      else
-        latest_url=https://yarnpkg.com/latest-version
-        specified_version=`curl -sS $latest_url`
-        version_type='latest'
-      fi
-      yarn_version=`$YARN_BIN -V`
-      yarn_alt_version=`$YARN_BIN --version`
-      if [ "$specified_version" = "$yarn_version" -o "$specified_version" = "$yarn_alt_version" ]; then
-        printf "Yarn is already at the $specified_version version.\n"
-        exit 0
-      else
-        rm -rf "$YARN_DIR"
-      fi
-    else
-      printf "$red> $YARN_DIR already exists, possibly from a past Yarn install.$reset\n"
-      printf "$red> Remove it (rm -rf $YARN_DIR) and run this script again.$reset\n"
-      exit 0
-    fi
-  fi
-
-  yarn_get_tarball $1 $2
-  yarn_link
-  yarn_reset
-}
-
-yarn_verify_or_quit() {
-  read -p "$1 [y/N] " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]
-  then
-    printf "$red> Aborting$reset\n"
-    exit 1
-  fi
-}
-
-cd $ZULIP_ROOT
-yarn_install $2 $3

scripts/lib/upgrade-zulip-stage-2

@@ -167,7 +167,8 @@ subprocess.check_call(
 )
 
 # Make sure the right version of node is installed
-subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node"), deploy_path])
+subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")])
+subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-yarn")])
 
 # Generate any new secrets that were added in the new version required.
 # TODO: Do caching to only run this when it has changed.

tools/lib/provision.py

@@ -419,6 +419,7 @@ def main(options: argparse.Namespace) -> "NoReturn":
         "no_proxy=" + os.environ.get("no_proxy", ""),
     ]
     run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["-H"])
+    run_as_root([*proxy_env, "scripts/lib/install-yarn"])
 
     if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK):
         run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])

tools/linter_lib/exclude.py

@@ -2,7 +2,6 @@
 EXCLUDED_FILES = [
     # Third-party code that doesn't match our style
     "puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh",
-    "scripts/lib/third",
     "static/third",
     # Transifex syncs translation.json files without trailing
     # newlines; there's nothing other than trailing newlines we'd be