From ec58b6790d9e29c2acaf94152f1c07187c9bece9 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 8 Feb 2023 16:13:42 -0800 Subject: [PATCH] install-node: Upgrade Node.js to 18.14.0; manage Yarn with Corepack. Corepack manages multiple per-project version of Yarn and PNPM, which means we have to maintain less installation code, and could help us switch away from Yarn 1 without making the system unusable for development of other Yarn 1 projects. https://nodejs.org/api/corepack.html The Unicode spaces in the timerender test resulted from an ICU upgrade: https://github.com/nodejs/node/pull/45068. Signed-off-by: Anders Kaseorg --- docs/subsystems/dependencies.md | 15 +++++++-------- frontend_tests/node_tests/timerender.js | 10 +++++----- package.json | 1 + scripts/lib/install | 1 - scripts/lib/install-node | 17 +++++++++++------ scripts/lib/install-yarn | 25 ------------------------- scripts/lib/node_cache.py | 5 +---- scripts/lib/setup_venv.py | 6 +----- scripts/lib/upgrade-zulip-stage-2 | 1 - tools/ci/production-build | 2 +- tools/lib/provision.py | 1 - tools/setup/optimize-svg | 2 +- version.py | 2 +- 13 files changed, 29 insertions(+), 59 deletions(-) delete mode 100755 scripts/lib/install-yarn diff --git a/docs/subsystems/dependencies.md b/docs/subsystems/dependencies.md index 4fa3512275..8e63cc51b0 100644 --- a/docs/subsystems/dependencies.md +++ b/docs/subsystems/dependencies.md @@ -250,16 +250,15 @@ reasoning here. ## Node.js and Yarn Node.js is installed by `scripts/lib/install-node` to -`/srv/zulip-node` and symlinked to `/usr/local/bin/node`. Yarn is -installed by `scripts/lib/install-yarn` to `/srv/zulip-yarn` and -symlinked to `/usr/bin/yarn`. +`/srv/zulip-node` and symlinked to `/usr/local/bin/node`. A Yarn +symlink at `/usr/local/bin/yarn` is managed by +[Corepack](https://nodejs.org/api/corepack.html). We don't do anything special to try to manage multiple versions of -Node.js or Yarn. (Previous versions of Zulip installed multiple -versions of Node.js using the third-party `nvm` installer, but the -current version no longer uses `nvm`; if it’s present in -`/usr/local/nvm` where previous versions installed it, it will now be -removed.) +Node.js. (Previous versions of Zulip installed multiple versions of +Node.js using the third-party `nvm` installer, but the current version +no longer uses `nvm`; if it’s present in `/usr/local/nvm` where +previous versions installed it, it will now be removed.) ## ShellCheck and shfmt diff --git a/frontend_tests/node_tests/timerender.js b/frontend_tests/node_tests/timerender.js index c3c94310e7..6716923bad 100644 --- a/frontend_tests/node_tests/timerender.js +++ b/frontend_tests/node_tests/timerender.js @@ -158,7 +158,7 @@ run_test("format_time_modern_different_timezones", () => { assert.equal(timerender.format_time_modern(yesterday, today), "translated: Yesterday"); process.env.TZ = "America/Juneau"; - let expected = "translated: 5/16/2017 at 11:12:53 PM AKDT (UTC-08:00)"; + let expected = "translated: 5/16/2017 at 11:12:53 PM AKDT (UTC-08:00)"; assert.equal(timerender.get_full_datetime(yesterday), expected); assert.equal(timerender.format_time_modern(yesterday, today), "Tuesday"); process.env.TZ = utc_tz; @@ -169,7 +169,7 @@ run_test("format_time_modern_different_timezones", () => { assert.equal(timerender.format_time_modern(yesterday, today), "Tuesday"); process.env.TZ = "Asia/Brunei"; - expected = "translated: 5/17/2017 at 5:12:53 AM (UTC+08:00)"; + expected = "translated: 5/17/2017 at 5:12:53 AM (UTC+08:00)"; assert.equal(timerender.get_full_datetime(yesterday), expected); assert.equal(timerender.format_time_modern(yesterday, today), "translated: Yesterday"); process.env.TZ = utc_tz; @@ -180,7 +180,7 @@ run_test("format_time_modern_different_timezones", () => { assert.equal(timerender.format_time_modern(yesterday, today), "Friday"); process.env.TZ = "America/Juneau"; - expected = "translated: 5/11/2017 at 11:12:53 PM AKDT (UTC-08:00)"; + expected = "translated: 5/11/2017 at 11:12:53 PM AKDT (UTC-08:00)"; assert.equal(timerender.get_full_datetime(yesterday), expected); assert.equal(timerender.format_time_modern(yesterday, today), "May 11"); process.env.TZ = utc_tz; @@ -340,7 +340,7 @@ run_test("absolute_time_24_hour", () => { run_test("get_full_datetime", () => { const time = date_2017_PM; - let expected = "translated: 5/18/2017 at 9:12:53 PM UTC"; + let expected = "translated: 5/18/2017 at 9:12:53 PM UTC"; assert.equal(timerender.get_full_datetime(time), expected); // test 24 hour time setting. @@ -353,7 +353,7 @@ run_test("get_full_datetime", () => { // Test the GMT[+-]x:y logic. const previous_env_tz = process.env.TZ; process.env.TZ = "Asia/Kolkata"; - expected = "translated: 5/19/2017 at 2:42:53 AM (UTC+05:30)"; + expected = "translated: 5/19/2017 at 2:42:53 AM (UTC+05:30)"; assert.equal(timerender.get_full_datetime(time), expected); process.env.TZ = previous_env_tz; }); diff --git a/package.json b/package.json index f54fed3a92..8db4d3745e 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "description": "", "private": true, "main": "", + "packageManager": "yarn@1.22.19+sha256.732620bac8b1690d507274f025f3c6cfdc3627a84d9642e38a07452cc00e0f2e", "dependencies": { "@babel/core": "^7.5.5", "@babel/preset-env": "^7.5.5", diff --git a/scripts/lib/install b/scripts/lib/install index 6e5df0aa59..3c48e7c04e 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -390,7 +390,6 @@ if [ "$VIRTUALENV_NEEDED" = "yes" ]; then fi "$ZULIP_PATH"/scripts/lib/install-node -"$ZULIP_PATH"/scripts/lib/install-yarn # Generate /etc/zulip/zulip.conf . mkdir -p /etc/zulip diff --git a/scripts/lib/install-node b/scripts/lib/install-node index 58296f4b98..0a1df94f6b 100755 --- a/scripts/lib/install-node +++ b/scripts/lib/install-node @@ -1,23 +1,24 @@ #!/usr/bin/env bash set -euo pipefail -version=18.12.1 +version=18.14.0 arch="$(uname -m)" case $arch in x86_64) tarball="node-v$version-linux-x64.tar.xz" - sha256=4481a34bf32ddb9a9ff9540338539401320e8c3628af39929b4211ea3552a19e + sha256=1ccec74b6240fce8754813e31fdbc93ad520df2e814729cea29efe9075e48350 ;; aarch64) tarball="node-v$version-linux-arm64.tar.xz" - sha256=3904869935b7ecc51130b4b86486d2356539a174d11c9181180cab649f32cd2a + sha256=30ef375f0b8006759c0e08bee9d4b74915b95abfa924006c289d2d474a8b152e ;; esac check_version() { - out="$(node --version)" && [ "$out" = "v$version" ] + out="$(node --version)" && [ "$out" = "v$version" ] \ + && [ /usr/local/bin/yarn -ef /srv/zulip-node/lib/node_modules/corepack/dist/yarn.js ] } if ! check_version 2>/dev/null; then @@ -33,7 +34,11 @@ if ! check_version 2>/dev/null; then rm -rf /srv/zulip-node mkdir -p /srv/zulip-node tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-node - ln -sf /srv/zulip-node/bin/{node,npm,npx} /usr/local/bin - rm -rf /usr/local/nvm + ln -sf /srv/zulip-node/bin/{corepack,node,npm,npx} /usr/local/bin + COREPACK_DEFAULT_TO_LATEST=0 /usr/local/bin/corepack enable + + # Clean up after previous versions of this script + rm -rf /srv/zulip-yarn /usr/bin/yarn /usr/local/nvm + check_version fi diff --git a/scripts/lib/install-yarn b/scripts/lib/install-yarn deleted file mode 100755 index c457428998..0000000000 --- a/scripts/lib/install-yarn +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version=1.22.19 -sha256=732620bac8b1690d507274f025f3c6cfdc3627a84d9642e38a07452cc00e0f2e -tarball="yarn-$version.tgz" - -check_version() { - # Reading the version of Yarn from its package.json is much faster - # than running yarn --version. - current_version="$(jq -r '.version' /srv/zulip-yarn/package.json)" \ - && [ "$current_version" = "$version" ] -} - -if ! check_version; then - tmpdir="$(mktemp -d)" - trap 'rm -r "$tmpdir"' EXIT - cd "$tmpdir" - curl -fLO --retry 3 "https://registry.npmjs.org/yarn/-/$tarball" - sha256sum -c <<<"$sha256 $tarball" - rm -rf /srv/zulip-yarn - mkdir /srv/zulip-yarn - tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-yarn - check_version -fi diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 5857d5a74a..dc0e15ed84 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -11,8 +11,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f ZULIP_SRV_PATH = "/srv" NODE_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-npm-cache") -YARN_BIN = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/bin/yarn") -YARN_PACKAGE_JSON = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/package.json") +YARN_BIN = "/usr/local/bin/yarn" DEFAULT_PRODUCTION = False @@ -40,8 +39,6 @@ def generate_sha1sum_node_modules( # For backwards compatibility, we can't assume yarn.lock exists with open(YARN_LOCK_FILE_PATH) as f: data[YARN_LOCK_FILE_PATH] = f.read().strip() - with open(YARN_PACKAGE_JSON) as f: - data["yarn-package-version"] = json.load(f)["version"] data["node-version"] = subprocess.check_output(["node", "--version"], text=True).strip() data["yarn-args"] = get_yarn_args(production=production) diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index 4f016f388f..2f0346100b 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -29,11 +29,7 @@ VENV_DEPENDENCIES = [ # Needed by python-xmlsec: "libxmlsec1-dev", "pkg-config", - # This is technically a node dependency, but we add it here - # because we don't have another place that we install apt packages - # on upgrade of a production server, and it's not worth adding - # another call to `apt install` for. - "jq", # Used by scripts/lib/install-yarn to check yarn version + "jq", # No longer used in production (clean me up later) "libsasl2-dev", # For building python-ldap from source ] diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index d619d8e151..c0031a0b1d 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -263,7 +263,6 @@ if not args.skip_downgrade_check: # Make sure the right version of node is installed subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")]) -subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-yarn")]) # Generate any new secrets that were added in the new version required. # TODO: Do caching to only run this when it has changed. diff --git a/tools/ci/production-build b/tools/ci/production-build index a524afec75..a097dc30eb 100755 --- a/tools/ci/production-build +++ b/tools/ci/production-build @@ -45,7 +45,7 @@ cp -a \ # Check that webpack bundles use only ES2019 syntax. # Use the yarn binary installed by tools/provision. -YARN="/srv/zulip-yarn/bin/yarn" +YARN="/usr/local/bin/yarn" tar -C /tmp -xzf /tmp/production-build/zulip-server-test.tar.gz zulip-server-test/prod-static/serve/webpack-bundles ( GLOBIGNORE=/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex-cli.js diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 9849f1009a..4c8e70866f 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -399,7 +399,6 @@ def main(options: argparse.Namespace) -> NoReturn: "no_proxy=" + os.environ.get("no_proxy", ""), ] run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["-H"]) - run_as_root([*proxy_env, "scripts/lib/install-yarn"]) if not os.access(NODE_MODULES_CACHE_PATH, os.W_OK): run_as_root(["mkdir", "-p", NODE_MODULES_CACHE_PATH]) diff --git a/tools/setup/optimize-svg b/tools/setup/optimize-svg index 23efbb4295..8dc6f28db7 100755 --- a/tools/setup/optimize-svg +++ b/tools/setup/optimize-svg @@ -42,7 +42,7 @@ if [ "$#" -gt 0 ]; then fi ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)" -YARN="/srv/zulip-yarn/bin/yarn" +YARN="/usr/local/bin/yarn" if [ -n "$CHECK_UNOPTIMIZED" ]; then if [ "$(node_modules/.bin/svgo -f static/images/integrations/logos | grep -o '\.[0-9]% = ' | wc -l)" -ge 1 ]; then diff --git a/version.py b/version.py index 2d36d9eb12..23a8ecec1b 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 162 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (221, 0) +PROVISION_VERSION = (222, 0)