diff --git a/docs/subsystems/dependencies.md b/docs/subsystems/dependencies.md index 29c2358218..672d44f683 100644 --- a/docs/subsystems/dependencies.md +++ b/docs/subsystems/dependencies.md @@ -244,17 +244,14 @@ reasoning here. These are installed by `scripts/lib/install-node` (which in turn uses the standard third-party `nvm` installer to download `node` and pin -its version) and `scripts/lib/third/install-yarn.sh` (the standard -installer for `yarn`, modified to support installing to a path that is -not the current user's home directory). +its version) and `scripts/lib/install-yarn`. * `nvm` has its own system for installing each version of `node` at its own path, which we use, though we install a `/usr/local/bin/node` wrapper to access the desired version conveniently and efficiently (`nvm` has a lot of startup overhead). -* `install-yarn.sh` is configured to install `yarn` at -`/srv/zulip-yarn`. We don't do anything special to try to manage -multiple versions of `yarn`. +* We install `yarn` at `/srv/zulip-yarn`. We don't do anything +special to try to manage multiple versions of `yarn`. ## ShellCheck and shfmt diff --git a/scripts/lib/install b/scripts/lib/install index 139662fc69..3e91f40bd2 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -381,6 +381,7 @@ if [ "$VIRTUALENV_NEEDED" = "yes" ]; then fi "$ZULIP_PATH"/scripts/lib/install-node +"$ZULIP_PATH"/scripts/lib/install-yarn # Generate /etc/zulip/zulip.conf . mkdir -p /etc/zulip diff --git a/scripts/lib/install-node b/scripts/lib/install-node index c69b413454..cbfcede3a5 100755 --- a/scripts/lib/install-node +++ b/scripts/lib/install-node @@ -1,11 +1,7 @@ #!/usr/bin/env bash set -eo pipefail -ZULIP_PATH="$(dirname "$0")/../.." -ZULIP_SRV="/srv" -YARN_PACKAGE_JSON="$ZULIP_SRV/zulip-yarn/package.json" node_version=14.17.0 -yarn_version=1.22.10 nvm_version=0.38.0 # This is a fix for the fact that nvm uses $HOME to determine which @@ -18,13 +14,8 @@ if node_wrapper_path="$(command -v node)"; then current_node_version="$(node --version)" fi -current_yarn_version="none" -if [ -e "$YARN_PACKAGE_JSON" ]; then - current_yarn_version=$(jq -r '.version' "$YARN_PACKAGE_JSON") -fi - -if [ "$current_yarn_version" = "$yarn_version" ] && [ "$current_node_version" = "v$node_version" ] && [ -L "$node_wrapper_path" ]; then - echo "Node version $node_version and yarn version $yarn_version are already installed." +if [ "$current_node_version" = "v$node_version" ] && [ -L "$node_wrapper_path" ]; then + echo "Node version $node_version is already installed." exit 0 fi @@ -62,12 +53,3 @@ if [ "$current_node_version" != "v$node_version" ] || ! [ -L "$node_wrapper_path ln -nsf "$(dirname "$NODE_BIN")/npm" /usr/local/bin/npm ln -nsf "$(dirname "$NODE_BIN")/npx" /usr/local/bin/npx fi - -# Work around the fact that apparently sudo doesn't clear the HOME -# environment variable in some cases; we don't want root -# accessing/storing yarn configuration in the non-root user's home -# directory. -export HOME=/root - -# Install yarn if not installed -bash "$ZULIP_PATH/scripts/lib/third/install-yarn.sh" "$ZULIP_SRV" --version "$yarn_version" diff --git a/scripts/lib/install-yarn b/scripts/lib/install-yarn new file mode 100755 index 0000000000..048d2cffb8 --- /dev/null +++ b/scripts/lib/install-yarn @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +version=1.22.10 +sha256=05a22fff30d7d8e8005bed277bf20d55111ba2bed65a6b91a0fcd1307b71fd8d +tarball="yarn-$version.tgz" + +check_version() { + # Reading the version of Yarn from its package.json is much faster + # than running yarn --version. + link="$(command -v yarn)" \ + && bin="$(readlink -f "$link")" \ + && current_version="$(jq -r '.version' "${bin%/*/*}/package.json")" \ + && [ "$current_version" = "$version" ] +} + +if ! check_version; then + tmpdir="$(mktemp -d)" + trap 'rm -r "$tmpdir"' EXIT + cd "$tmpdir" + curl -LO "https://registry.npmjs.org/yarn/-/$tarball" + sha256sum -c <<<"$sha256 $tarball" + rm -rf /srv/zulip-yarn + mkdir /srv/zulip-yarn + tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-yarn + ln -nsf /srv/zulip-yarn/bin/yarn /usr/bin/yarn + check_version +fi diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index e275c9144f..9a03079203 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -33,7 +33,7 @@ VENV_DEPENDENCIES = [ # because we don't have another place that we install apt packages # on upgrade of a production server, and it's not worth adding # another call to `apt install` for. - "jq", # Used by scripts/lib/install-node to check yarn version + "jq", # Used by scripts/lib/install-yarn to check yarn version "libsasl2-dev", # For building python-ldap from source ] diff --git a/scripts/lib/third/install-yarn.sh b/scripts/lib/third/install-yarn.sh deleted file mode 100644 index f26e01706e..0000000000 --- a/scripts/lib/third/install-yarn.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/sh -set -e - -##################### -# install-yarn.sh was patched to install yarn in a custom directory. -# The following changes were made: -# * yarn_link now just simlinks to /usr/bin -# * yarn_detect_profile was removed -# * Paths were changed to variables declared at the top -# * Most of the non error coloration was removed to not distract during installs. -# ####################### - -reset="\033[0m" -red="\033[31m" -yellow="\033[33m" -gpg_key=9D41F3C3 - -ZULIP_ROOT="$1" -YARN_DIR_NAME="zulip-yarn" -YARN_DIR="$ZULIP_ROOT/$YARN_DIR_NAME" -YARN_BIN="$YARN_DIR/bin/yarn" - -yarn_get_tarball() { - printf "Downloading tarball...\n" - if [ "$1" = '--nightly' ]; then - url=https://nightly.yarnpkg.com/latest.tar.gz - elif [ "$1" = '--rc' ]; then - url=https://yarnpkg.com/latest-rc.tar.gz - elif [ "$1" = '--version' ]; then - # Validate that the version matches MAJOR.MINOR.PATCH to avoid garbage-in/garbage-out behavior - version=$2 - if echo $version | grep -qE "^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$"; then - url="https://yarnpkg.com/downloads/$version/yarn-v$version.tar.gz" - else - printf "$red> Version number must match MAJOR.MINOR.PATCH.$reset\n" - exit 1; - fi - else - url=https://yarnpkg.com/latest.tar.gz - fi - # Get both the tarball and its GPG signature - tarball_tmp=`mktemp -t yarn.tar.gz.XXXXXXXXXX` - if curl --fail -L -o "$tarball_tmp#1" "$url{,.asc}"; then - yarn_verify_integrity $tarball_tmp - - printf "Extracting to $YARN_DIR...\n" - mkdir "$YARN_DIR_NAME" - tar zxf $tarball_tmp -C "$YARN_DIR_NAME" --strip 1 # extract tarball - rm $tarball_tmp* - else - printf "$red> Failed to download $url.$reset\n" - exit 1; - fi -} - -# Verifies the GPG signature of the tarball -yarn_verify_integrity() { - # Check if GPG is installed - if [[ -z "$(command -v gpg)" ]]; then - printf "$yellow> WARNING: GPG is not installed, integrity cannot be verified!$reset\n" - return - fi - - if [ "$YARN_GPG" == "no" ]; then - printf "WARNING: Skipping GPG integrity check!\n" - return - fi - - printf "Verifying integrity...\n" - # Grab the public key if it doesn't already exist - # Zulip patch: Fix the fact that Yarn has extended this keyring and we should always redownload. - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --import - - if [ ! -f "$1.asc" ]; then - printf "$red> Could not download GPG signature for this Yarn release. This means the release cannot be verified!$reset\n" - yarn_verify_or_quit "> Do you really want to continue?" - return - fi - - # Actually perform the verification - if gpg --verify "$1.asc" $1; then - printf "GPG signature looks good\n" - else - printf "$red> GPG signature for this Yarn release is invalid! This is BAD and may mean the release has been tampered with. It is strongly recommended that you report this to the Yarn developers.$reset\n" - yarn_verify_or_quit "> Do you really want to continue?" - fi -} - -yarn_link() { - printf "Adding to /usr/bin\n" - - version=`$YARN_BIN --version` || ( - printf "$red> Yarn was installed, but doesn't seem to be working :(.$reset\n" - exit 1; - ) - - ln -nsf "$YARN_BIN" /usr/bin/yarn - - printf "Successfully installed Yarn $version!\n" -} - - -yarn_reset() { - unset -f yarn_install yarn_reset yarn_get_tarball yarn_link yarn_verify_integrity yarn_verify_or_quit -} - -yarn_install() { - printf "Installing Yarn!\n" - - if [ -d "$YARN_DIR" ]; then - if [ -e "$YARN_BIN" ] ; then - local latest_url - local specified_version - local version_type - if [ "$1" = '--nightly' ]; then - latest_url=https://nightly.yarnpkg.com/latest-tar-version - specified_version=`curl -sS $latest_url` - version_type='latest' - elif [ "$1" = '--version' ]; then - specified_version=$2 - version_type='specified' - elif [ "$1" = '--rc' ]; then - latest_url=https://yarnpkg.com/latest-rc-version - specified_version=`curl -sS $latest_url` - version_type='rc' - else - latest_url=https://yarnpkg.com/latest-version - specified_version=`curl -sS $latest_url` - version_type='latest' - fi - yarn_version=`$YARN_BIN -V` - yarn_alt_version=`$YARN_BIN --version` - if [ "$specified_version" = "$yarn_version" -o "$specified_version" = "$yarn_alt_version" ]; then - printf "Yarn is already at the $specified_version version.\n" - exit 0 - else - rm -rf "$YARN_DIR" - fi - else - printf "$red> $YARN_DIR already exists, possibly from a past Yarn install.$reset\n" - printf "$red> Remove it (rm -rf $YARN_DIR) and run this script again.$reset\n" - exit 0 - fi - fi - - yarn_get_tarball $1 $2 - yarn_link - yarn_reset -} - -yarn_verify_or_quit() { - read -p "$1 [y/N] " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]] - then - printf "$red> Aborting$reset\n" - exit 1 - fi -} - -cd $ZULIP_ROOT -yarn_install $2 $3 diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index 2b0407c902..5aae283d32 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -167,7 +167,8 @@ subprocess.check_call( ) # Make sure the right version of node is installed -subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node"), deploy_path]) +subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")]) +subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-yarn")]) # Generate any new secrets that were added in the new version required. # TODO: Do caching to only run this when it has changed. diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 229a571d7b..974884ff06 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -419,6 +419,7 @@ def main(options: argparse.Namespace) -> "NoReturn": "no_proxy=" + os.environ.get("no_proxy", ""), ] run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["-H"]) + run_as_root([*proxy_env, "scripts/lib/install-yarn"]) if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK): run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH]) diff --git a/tools/linter_lib/exclude.py b/tools/linter_lib/exclude.py index 86dbc95bd2..d1a0b8dd65 100644 --- a/tools/linter_lib/exclude.py +++ b/tools/linter_lib/exclude.py @@ -2,7 +2,6 @@ EXCLUDED_FILES = [ # Third-party code that doesn't match our style "puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh", - "scripts/lib/third", "static/third", # Transifex syncs translation.json files without trailing # newlines; there's nothing other than trailing newlines we'd be