install-node: Upgrade Node.js to 18.14.0; manage Yarn with Corepack.

Corepack manages multiple per-project version of Yarn and PNPM, which
means we have to maintain less installation code, and could help us
switch away from Yarn 1 without making the system unusable for
development of other Yarn 1 projects.

https://nodejs.org/api/corepack.html

The Unicode spaces in the timerender test resulted from an ICU
upgrade: https://github.com/nodejs/node/pull/45068.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

docs/subsystems/dependencies.md

@@ -250,16 +250,15 @@ reasoning here.
 ## Node.js and Yarn
 
 Node.js is installed by `scripts/lib/install-node` to
-`/srv/zulip-node` and symlinked to `/usr/local/bin/node`. Yarn is
+`/srv/zulip-node` and symlinked to `/usr/local/bin/node`. A Yarn
-installed by `scripts/lib/install-yarn` to `/srv/zulip-yarn` and
+symlink at `/usr/local/bin/yarn` is managed by
-symlinked to `/usr/bin/yarn`.
+[Corepack](https://nodejs.org/api/corepack.html).
 
 We don't do anything special to try to manage multiple versions of
-Node.js or Yarn. (Previous versions of Zulip installed multiple
+Node.js. (Previous versions of Zulip installed multiple versions of
-versions of Node.js using the third-party `nvm` installer, but the
+Node.js using the third-party `nvm` installer, but the current version
-current version no longer uses `nvm`; if it’s present in
+no longer uses `nvm`; if it’s present in `/usr/local/nvm` where
-`/usr/local/nvm` where previous versions installed it, it will now be
+previous versions installed it, it will now be removed.)
-removed.)
 
 ## ShellCheck and shfmt
 

frontend_tests/node_tests/timerender.js

@@ -158,7 +158,7 @@ run_test("format_time_modern_different_timezones", () => {
     assert.equal(timerender.format_time_modern(yesterday, today), "translated: Yesterday");
 
     process.env.TZ = "America/Juneau";
-    let expected = "translated: 5/16/2017 at 11:12:53 PM AKDT (UTC-08:00)";
+    let expected = "translated: 5/16/2017 at 11:12:53 PM AKDT (UTC-08:00)";
     assert.equal(timerender.get_full_datetime(yesterday), expected);
     assert.equal(timerender.format_time_modern(yesterday, today), "Tuesday");
     process.env.TZ = utc_tz;
@@ -169,7 +169,7 @@ run_test("format_time_modern_different_timezones", () => {
     assert.equal(timerender.format_time_modern(yesterday, today), "Tuesday");
 
     process.env.TZ = "Asia/Brunei";
-    expected = "translated: 5/17/2017 at 5:12:53 AM (UTC+08:00)";
+    expected = "translated: 5/17/2017 at 5:12:53 AM (UTC+08:00)";
     assert.equal(timerender.get_full_datetime(yesterday), expected);
     assert.equal(timerender.format_time_modern(yesterday, today), "translated: Yesterday");
     process.env.TZ = utc_tz;
@@ -180,7 +180,7 @@ run_test("format_time_modern_different_timezones", () => {
     assert.equal(timerender.format_time_modern(yesterday, today), "Friday");
 
     process.env.TZ = "America/Juneau";
-    expected = "translated: 5/11/2017 at 11:12:53 PM AKDT (UTC-08:00)";
+    expected = "translated: 5/11/2017 at 11:12:53 PM AKDT (UTC-08:00)";
     assert.equal(timerender.get_full_datetime(yesterday), expected);
     assert.equal(timerender.format_time_modern(yesterday, today), "May 11");
     process.env.TZ = utc_tz;
@@ -340,7 +340,7 @@ run_test("absolute_time_24_hour", () => {
 run_test("get_full_datetime", () => {
     const time = date_2017_PM;
 
-    let expected = "translated: 5/18/2017 at 9:12:53 PM UTC";
+    let expected = "translated: 5/18/2017 at 9:12:53 PM UTC";
     assert.equal(timerender.get_full_datetime(time), expected);
 
     // test 24 hour time setting.
@@ -353,7 +353,7 @@ run_test("get_full_datetime", () => {
     // Test the GMT[+-]x:y logic.
     const previous_env_tz = process.env.TZ;
     process.env.TZ = "Asia/Kolkata";
-    expected = "translated: 5/19/2017 at 2:42:53 AM (UTC+05:30)";
+    expected = "translated: 5/19/2017 at 2:42:53 AM (UTC+05:30)";
     assert.equal(timerender.get_full_datetime(time), expected);
     process.env.TZ = previous_env_tz;
 });

package.json

@@ -5,6 +5,7 @@
   "description": "",
   "private": true,
   "main": "",
+  "packageManager": "yarn@1.22.19+sha256.732620bac8b1690d507274f025f3c6cfdc3627a84d9642e38a07452cc00e0f2e",
   "dependencies": {
     "@babel/core": "^7.5.5",
     "@babel/preset-env": "^7.5.5",

scripts/lib/install

@@ -390,7 +390,6 @@ if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
 fi
 
 "$ZULIP_PATH"/scripts/lib/install-node
-"$ZULIP_PATH"/scripts/lib/install-yarn
 
 # Generate /etc/zulip/zulip.conf .
 mkdir -p /etc/zulip

scripts/lib/install-node

@@ -1,23 +1,24 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-version=18.12.1
+version=18.14.0
 arch="$(uname -m)"
 
 case $arch in
     x86_64)
         tarball="node-v$version-linux-x64.tar.xz"
-        sha256=4481a34bf32ddb9a9ff9540338539401320e8c3628af39929b4211ea3552a19e
+        sha256=1ccec74b6240fce8754813e31fdbc93ad520df2e814729cea29efe9075e48350
         ;;
 
     aarch64)
         tarball="node-v$version-linux-arm64.tar.xz"
-        sha256=3904869935b7ecc51130b4b86486d2356539a174d11c9181180cab649f32cd2a
+        sha256=30ef375f0b8006759c0e08bee9d4b74915b95abfa924006c289d2d474a8b152e
         ;;
 esac
 
 check_version() {
-    out="$(node --version)" && [ "$out" = "v$version" ]
+    out="$(node --version)" && [ "$out" = "v$version" ] \
+        && [ /usr/local/bin/yarn -ef /srv/zulip-node/lib/node_modules/corepack/dist/yarn.js ]
 }
 
 if ! check_version 2>/dev/null; then
@@ -33,7 +34,11 @@ if ! check_version 2>/dev/null; then
     rm -rf /srv/zulip-node
     mkdir -p /srv/zulip-node
     tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-node
-    ln -sf /srv/zulip-node/bin/{node,npm,npx} /usr/local/bin
+    ln -sf /srv/zulip-node/bin/{corepack,node,npm,npx} /usr/local/bin
-    rm -rf /usr/local/nvm
+    COREPACK_DEFAULT_TO_LATEST=0 /usr/local/bin/corepack enable
+
+    # Clean up after previous versions of this script
+    rm -rf /srv/zulip-yarn /usr/bin/yarn /usr/local/nvm
+
     check_version
 fi

scripts/lib/install-yarn

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-version=1.22.19
-sha256=732620bac8b1690d507274f025f3c6cfdc3627a84d9642e38a07452cc00e0f2e
-tarball="yarn-$version.tgz"
-
-check_version() {
-    # Reading the version of Yarn from its package.json is much faster
-    # than running yarn --version.
-    current_version="$(jq -r '.version' /srv/zulip-yarn/package.json)" \
-        && [ "$current_version" = "$version" ]
-}
-
-if ! check_version; then
-    tmpdir="$(mktemp -d)"
-    trap 'rm -r "$tmpdir"' EXIT
-    cd "$tmpdir"
-    curl -fLO --retry 3 "https://registry.npmjs.org/yarn/-/$tarball"
-    sha256sum -c <<<"$sha256 $tarball"
-    rm -rf /srv/zulip-yarn
-    mkdir /srv/zulip-yarn
-    tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-yarn
-    check_version
-fi

scripts/lib/node_cache.py

@@ -11,8 +11,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 ZULIP_SRV_PATH = "/srv"
 
 NODE_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-npm-cache")
-YARN_BIN = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/bin/yarn")
+YARN_BIN = "/usr/local/bin/yarn"
-YARN_PACKAGE_JSON = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/package.json")
 
 DEFAULT_PRODUCTION = False
 
@@ -40,8 +39,6 @@ def generate_sha1sum_node_modules(
         # For backwards compatibility, we can't assume yarn.lock exists
         with open(YARN_LOCK_FILE_PATH) as f:
             data[YARN_LOCK_FILE_PATH] = f.read().strip()
-    with open(YARN_PACKAGE_JSON) as f:
-        data["yarn-package-version"] = json.load(f)["version"]
     data["node-version"] = subprocess.check_output(["node", "--version"], text=True).strip()
     data["yarn-args"] = get_yarn_args(production=production)
 

scripts/lib/setup_venv.py

@@ -29,11 +29,7 @@ VENV_DEPENDENCIES = [
     # Needed by python-xmlsec:
     "libxmlsec1-dev",
     "pkg-config",
-    # This is technically a node dependency, but we add it here
+    "jq",  # No longer used in production (clean me up later)
-    # because we don't have another place that we install apt packages
-    # on upgrade of a production server, and it's not worth adding
-    # another call to `apt install` for.
-    "jq",  # Used by scripts/lib/install-yarn to check yarn version
     "libsasl2-dev",  # For building python-ldap from source
 ]
 

scripts/lib/upgrade-zulip-stage-2

@@ -263,7 +263,6 @@ if not args.skip_downgrade_check:
 
 # Make sure the right version of node is installed
 subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")])
-subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-yarn")])
 
 # Generate any new secrets that were added in the new version required.
 # TODO: Do caching to only run this when it has changed.

tools/ci/production-build

@@ -45,7 +45,7 @@ cp -a \
 
 # Check that webpack bundles use only ES2019 syntax.
 # Use the yarn binary installed by tools/provision.
-YARN="/srv/zulip-yarn/bin/yarn"
+YARN="/usr/local/bin/yarn"
 tar -C /tmp -xzf /tmp/production-build/zulip-server-test.tar.gz zulip-server-test/prod-static/serve/webpack-bundles
 (
     GLOBIGNORE=/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex-cli.js

tools/lib/provision.py

@@ -399,7 +399,6 @@ def main(options: argparse.Namespace) -> NoReturn:
         "no_proxy=" + os.environ.get("no_proxy", ""),
     ]
     run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["-H"])
-    run_as_root([*proxy_env, "scripts/lib/install-yarn"])
 
     if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK):
         run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH])

tools/setup/optimize-svg

@@ -42,7 +42,7 @@ if [ "$#" -gt 0 ]; then
 fi
 
 ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
-YARN="/srv/zulip-yarn/bin/yarn"
+YARN="/usr/local/bin/yarn"
 
 if [ -n "$CHECK_UNOPTIMIZED" ]; then
     if [ "$(node_modules/.bin/svgo -f static/images/integrations/logos | grep -o '\.[0-9]% = ' | wc -l)" -ge 1 ]; then

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 162
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (221, 0)
+PROVISION_VERSION = (222, 0)