From 91bfebca7dfea485f4b5edcab8ae9d0161365e8c Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 24 Jun 2021 16:28:27 -0700 Subject: [PATCH] install: Replace wget with curl. curl uses Happy Eyeballs to avoid long timeouts on systems with broken IPv6. Signed-off-by: Anders Kaseorg --- docs/production/install-existing-server.md | 2 +- docs/production/install.md | 2 +- docs/production/upgrade-or-modify.md | 2 +- docs/tutorials/shell-tips.md | 2 +- puppet/zulip/manifests/profile/base.pp | 3 +- scripts/lib/build-pgroonga | 2 +- scripts/lib/install | 4 +- scripts/lib/install-node | 6 +-- scripts/lib/setup-apt-repo | 2 +- scripts/setup/apt-repos/zulip/custom.sh | 4 +- scripts/setup/sha256-tarball-to | 2 +- .../zerver/help/desktop-app-install-guide.md | 2 +- tools/ci/.gitattributes | 1 + tools/ci/Dockerfile.prod | 2 +- tools/ci/production-build | 1 - tools/ci/production-verify | 31 ++++-------- .../success-http-headers.template.debian.txt | 29 ------------ tools/ci/success-http-headers.template.txt | 47 +++++++++---------- tools/lib/provision.py | 4 +- tools/linter_lib/custom_check.py | 5 +- tools/setup/bootstrap-aws-installer | 2 +- tools/setup/bootstrap-awscli | 2 +- tools/setup/install-shellcheck | 2 +- tools/setup/install-shfmt | 2 +- tools/test-install/prepare-base | 2 +- 25 files changed, 56 insertions(+), 107 deletions(-) create mode 100644 tools/ci/.gitattributes delete mode 100644 tools/ci/success-http-headers.template.debian.txt diff --git a/docs/production/install-existing-server.md b/docs/production/install-existing-server.md index 143f165848..ea2a46c64f 100644 --- a/docs/production/install-existing-server.md +++ b/docs/production/install-existing-server.md @@ -28,7 +28,7 @@ one created by Zulip into it: ```shell sudo cp /etc/nginx/nginx.conf /etc/nginx.conf.before-zulip-install -sudo wget -O /etc/nginx/nginx.conf.zulip \ +sudo curl -Lo /etc/nginx/nginx.conf.zulip \ https://raw.githubusercontent.com/zulip/zulip/master/puppet/zulip/templates/nginx.conf.template.erb sudo meld /etc/nginx/nginx.conf /etc/nginx/nginx.conf.zulip # be sure to merge to the right ``` diff --git a/docs/production/install.md b/docs/production/install.md index 3f15269b64..bc27675d27 100644 --- a/docs/production/install.md +++ b/docs/production/install.md @@ -20,7 +20,7 @@ with the following commands: ``` cd $(mktemp -d) -wget https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz +curl -LO https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz tar -xf zulip-server-latest.tar.gz ``` diff --git a/docs/production/upgrade-or-modify.md b/docs/production/upgrade-or-modify.md index 4dfdd888cc..53c1396417 100644 --- a/docs/production/upgrade-or-modify.md +++ b/docs/production/upgrade-or-modify.md @@ -28,7 +28,7 @@ to a new Zulip release: release with: ``` - wget https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz + curl -LO https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz ``` You also have the option of upgrading Zulip [to a version in a Git diff --git a/docs/tutorials/shell-tips.md b/docs/tutorials/shell-tips.md index 4c905cea02..08de9f5c2f 100644 --- a/docs/tutorials/shell-tips.md +++ b/docs/tutorials/shell-tips.md @@ -178,7 +178,7 @@ environment: ``` sudo apt-get -y purge vagrant && \ -wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb && \ +curl -LO https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb && \ sudo dpkg -i vagrant*.deb && \ sudo apt-get -y install build-essential git ruby lxc lxc-templates cgroup-lite redir && \ vagrant plugin install vagrant-lxc && \ diff --git a/puppet/zulip/manifests/profile/base.pp b/puppet/zulip/manifests/profile/base.pp index 5b019ab7d3..4eb23b4cec 100644 --- a/puppet/zulip/manifests/profile/base.pp +++ b/puppet/zulip/manifests/profile/base.pp @@ -39,8 +39,9 @@ class zulip::profile::base { 'python3-yaml', 'puppet', 'git', - # Used in scripts including install-yarn.sh + # Used for most downloads 'curl', + # Used in check_website_response.sh 'wget', 'jq', 'procps', diff --git a/scripts/lib/build-pgroonga b/scripts/lib/build-pgroonga index 9f74ff776b..aac59f0a65 100755 --- a/scripts/lib/build-pgroonga +++ b/scripts/lib/build-pgroonga @@ -6,7 +6,7 @@ PGROONGA_VERSION="2.2.8" cd "$(mktemp -d)" -wget https://packages.groonga.org/source/pgroonga/pgroonga-"$PGROONGA_VERSION".tar.gz +curl -LO https://packages.groonga.org/source/pgroonga/pgroonga-"$PGROONGA_VERSION".tar.gz tar xf pgroonga-"$PGROONGA_VERSION".tar.gz cd pgroonga-"$PGROONGA_VERSION" diff --git a/scripts/lib/install b/scripts/lib/install index da897788fa..139662fc69 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -344,7 +344,7 @@ if [ "$package_system" = apt ]; then # `zulip::profile::base` such that the new dependency is seen by # upgrades, as well as new installs. if ! apt-get install -y \ - python3 python3-yaml puppet git curl wget jq crudini \ + python3 python3-yaml puppet git curl jq crudini \ "${ADDITIONAL_PACKAGES[@]}"; then set +x echo -e '\033[0;31m' >&2 @@ -355,7 +355,7 @@ if [ "$package_system" = apt ]; then fi elif [ "$package_system" = yum ]; then if ! yum install -y \ - python3 python3-pyyaml puppet git curl wget jq crudini \ + python3 python3-pyyaml puppet git curl jq crudini \ "${ADDITIONAL_PACKAGES[@]}"; then set +x echo -e '\033[0;31m' >&2 diff --git a/scripts/lib/install-node b/scripts/lib/install-node index 4f3256823f..c69b413454 100755 --- a/scripts/lib/install-node +++ b/scripts/lib/install-node @@ -36,11 +36,11 @@ if [ "$current_node_version" != "v$node_version" ] || ! [ -L "$node_wrapper_path [ "$(nvm --version)" != "$nvm_version" ] }; then mkdir -p "$NVM_DIR" - wget_opts=(-nv) + curl_opts=(-L) if [ -n "${CUSTOM_CA_CERTIFICATES:-}" ]; then - wget_opts+=(--ca-certificate "${CUSTOM_CA_CERTIFICATES}") + curl_opts+=(--cacert "${CUSTOM_CA_CERTIFICATES}") fi - wget "${wget_opts[@]}" -O- "https://raw.githubusercontent.com/nvm-sh/nvm/v$nvm_version/install.sh" | bash + curl "${curl_opts[@]}" "https://raw.githubusercontent.com/nvm-sh/nvm/v$nvm_version/install.sh" | bash # shellcheck source=/dev/null . "$NVM_DIR/nvm.sh" fi diff --git a/scripts/lib/setup-apt-repo b/scripts/lib/setup-apt-repo index 55bf412229..910b35f17c 100755 --- a/scripts/lib/setup-apt-repo +++ b/scripts/lib/setup-apt-repo @@ -72,7 +72,7 @@ touch "$SOURCES_FILE" # Hash it to check if the sources file is changed by the script later. zulip_source_hash=$(sha1sum "$SOURCES_FILE") -pre_setup_deps=(lsb-release apt-transport-https ca-certificates gnupg wget) +pre_setup_deps=(lsb-release apt-transport-https ca-certificates gnupg curl) if ! apt-get -dy install "${pre_setup_deps[@]}"; then apt-get update fi diff --git a/scripts/setup/apt-repos/zulip/custom.sh b/scripts/setup/apt-repos/zulip/custom.sh index bd89cb688f..fa0f561491 100755 --- a/scripts/setup/apt-repos/zulip/custom.sh +++ b/scripts/setup/apt-repos/zulip/custom.sh @@ -23,8 +23,8 @@ if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then distribution=$(lsb_release -si | tr '[:upper:]' '[:lower:]') groonga_apt_source_deb="groonga-apt-source-latest-$release.deb" groonga_apt_source_deb_sign="$groonga_apt_source_deb.asc.$pgroonga_apt_sign_key_fingerprint" - wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb" - wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb_sign" + curl -LO "https://packages.groonga.org/$distribution/$groonga_apt_source_deb" + curl -LO "https://packages.groonga.org/$distribution/$groonga_apt_source_deb_sign" gpg \ --homedir="$tmp_gpg_home" \ --verify \ diff --git a/scripts/setup/sha256-tarball-to b/scripts/setup/sha256-tarball-to index 1edbbd1498..033fa5f84c 100755 --- a/scripts/setup/sha256-tarball-to +++ b/scripts/setup/sha256-tarball-to @@ -25,7 +25,7 @@ cd "$tmpdir" # Fetch to a predictable name, not whatever curl guesses from the URL LOCALFILE="archive.tar.gz" -wget -qO "$LOCALFILE" "$URL" +curl -Lo "$LOCALFILE" "$URL" # Check the hash against what was passed in echo "$SHA256 $LOCALFILE" >"$LOCALFILE.sha256" diff --git a/templates/zerver/help/desktop-app-install-guide.md b/templates/zerver/help/desktop-app-install-guide.md index e99003edf6..d28b75706b 100644 --- a/templates/zerver/help/desktop-app-install-guide.md +++ b/templates/zerver/help/desktop-app-install-guide.md @@ -55,7 +55,7 @@ to future versions. 1. Enter the following commands into a terminal: - sudo wget -O /etc/apt/trusted.gpg.d/zulip-desktop.asc \ + sudo curl -Lo /etc/apt/trusted.gpg.d/zulip-desktop.asc \ https://download.zulip.com/desktop/apt/zulip-desktop.asc echo "deb https://download.zulip.com/desktop/apt stable main" | \ sudo tee /etc/apt/sources.list.d/zulip-desktop.list diff --git a/tools/ci/.gitattributes b/tools/ci/.gitattributes new file mode 100644 index 0000000000..2b2d664639 --- /dev/null +++ b/tools/ci/.gitattributes @@ -0,0 +1 @@ +success-http-headers.template.txt eol=crlf diff --git a/tools/ci/Dockerfile.prod b/tools/ci/Dockerfile.prod index e3ebf1724e..dfaca7061a 100644 --- a/tools/ci/Dockerfile.prod +++ b/tools/ci/Dockerfile.prod @@ -17,7 +17,7 @@ RUN if (. /etc/os-release && [ "$ID $VERSION_ID" = 'ubuntu 18.04' ]); then \ # Download the release tarball, start rabbitmq server and install the server ARG VERSION RUN cd $(mktemp -d) \ - && wget "https://www.zulip.org/dist/releases/zulip-server-$VERSION.tar.gz" \ + && curl -LO "https://www.zulip.org/dist/releases/zulip-server-$VERSION.tar.gz" \ && tar -xf "zulip-server-$VERSION.tar.gz" \ && sudo service rabbitmq-server start \ && sudo service rabbitmq-server status \ diff --git a/tools/ci/production-build b/tools/ci/production-build index 6d42419d71..a61276dc93 100755 --- a/tools/ci/production-build +++ b/tools/ci/production-build @@ -34,7 +34,6 @@ mkdir /tmp/production-build mv /tmp/tmp.*/zulip-server-test.tar.gz /tmp/production-build cp -a \ tools/ci/success-http-headers.template.txt \ - tools/ci/success-http-headers.template.debian.txt \ tools/ci/production-install \ tools/ci/production-verify \ tools/ci/production-upgrade \ diff --git a/tools/ci/production-verify b/tools/ci/production-verify index c14a5d1aa0..9c508b0fa2 100755 --- a/tools/ci/production-verify +++ b/tools/ci/production-verify @@ -12,17 +12,9 @@ NOREPLY_EMAIL_ADDRESS = 'noreply@circleci.example.com' ALLOWED_HOSTS = [] EOF -os_info="$( - . /etc/os-release - printf '%s\n' "$ID" -)" -{ - read -r os_id -} <<<"$os_info" - check_header() { - sed -i -e 's|Length: [0-9]\+\( ([0-9.]\+K)\)\?|Length: |' -e "s|{nginx_version_string}|$nginx_version|g" "$success_header_file" - if ! diff -ur /tmp/http-headers-processed "$success_header_file"; then + if ! sed "s|{nginx_version_string}|$nginx_version|g" "$success_header_file" \ + | diff -ur - /tmp/http-headers-processed; then set +x echo echo "FAILURE: The HTTP headers returned from loading the homepage on the server do not match the contents of tools/ci/success-http-headers.template.txt. Typically, this means that the server threw a 500 when trying to load the homepage." @@ -67,20 +59,13 @@ echo echo "Now testing that the newly installed server's homepage loads" echo -wget https://localhost -O /tmp/index.html --no-check-certificate -S 2>/tmp/wget-output || true # || true so we see errors.log if this 500s -grep -vi '\(Vary\|Content-Language\|expires\|issued by\|modified\|saved\|[.][.][.]\|Date\|[-][-]\)' /tmp/wget-output >/tmp/http-headers-processed +curl -ILk https://localhost -o /tmp/http-headers +grep -vi -e '^content-length:' -e '^date:' -e '^expires:' -e '^set-cookie:' /tmp/http-headers >/tmp/http-headers-processed -nginx_version="$(nginx -v 2>&1 | awk '{print $3, $4}' | xargs)" - -# Simplify the diff by getting replacing 4-5 digit length numbers with . -sed -i 's|Length: [0-9]\+\( ([0-9.]\+K)\)\?|Length: |' /tmp/http-headers-processed -if [ "$os_id" = debian ]; then - success_header_file="/tmp/success-http-headers.template.debian.txt" - check_header -else - success_header_file="/tmp/success-http-headers.template.txt" - check_header -fi +nginx_version="$(nginx -v 2>&1)" +nginx_version="${nginx_version#nginx version: }" +success_header_file="/tmp/success-http-headers.template.txt" +check_header # Start the RabbitMQ queue worker related section echo diff --git a/tools/ci/success-http-headers.template.debian.txt b/tools/ci/success-http-headers.template.debian.txt deleted file mode 100644 index 67ccf09af8..0000000000 --- a/tools/ci/success-http-headers.template.debian.txt +++ /dev/null @@ -1,29 +0,0 @@ -WARNING: The certificate of ‘localhost’ is not trusted. -WARNING: The certificate of ‘localhost’ doesn't have a known issuer. -The certificate's owner does not match hostname ‘localhost’ - HTTP/1.1 302 Found - Server: {nginx_version_string} - Content-Type: text/html; charset=utf-8 - Content-Length: 0 - Connection: keep-alive - Location: /login/ - Strict-Transport-Security: max-age=15768000 - X-Frame-Options: DENY - X-Content-Type-Options: nosniff - X-XSS-Protection: 1; mode=block -Location: /login/ [following] -Reusing existing connection to localhost:443. - HTTP/1.1 200 OK - Server: {nginx_version_string} - Content-Type: text/html; charset=utf-8 - Content-Length: 6361 - Connection: keep-alive - Strict-Transport-Security: max-age=15768000 - X-Frame-Options: DENY - X-Content-Type-Options: nosniff - X-XSS-Protection: 1; mode=block -Length: 6361 (6.2K) [text/html] -Saving to: ‘/tmp/index.html’ - - - diff --git a/tools/ci/success-http-headers.template.txt b/tools/ci/success-http-headers.template.txt index 113cb9da65..a3e854e4c8 100644 --- a/tools/ci/success-http-headers.template.txt +++ b/tools/ci/success-http-headers.template.txt @@ -1,28 +1,23 @@ - Self-signed certificate encountered. - WARNING: certificate common name ‘127.0.0.1’ doesn't match requested host name ‘localhost’. - HTTP/1.1 302 Found - Server: {nginx_version_string} - Content-Type: text/html; charset=utf-8 - Content-Length: 0 - Connection: keep-alive - Location: /login/ - Strict-Transport-Security: max-age=15768000 - X-Frame-Options: DENY - X-Content-Type-Options: nosniff - X-XSS-Protection: 1; mode=block -Location: /login/ [following] -Reusing existing connection to localhost:443. - HTTP/1.1 200 OK - Server: {nginx_version_string} - Content-Type: text/html; charset=utf-8 - Content-Length: 6361 - Connection: keep-alive - Strict-Transport-Security: max-age=15768000 - X-Frame-Options: DENY - X-Content-Type-Options: nosniff - X-XSS-Protection: 1; mode=block -Length: 6361 (6.2K) [text/html] -Saving to: ‘/tmp/index.html’ - +HTTP/2 302 +server: {nginx_version_string} +content-type: text/html; charset=utf-8 +location: /login/ +vary: Accept-Language, Cookie +content-language: en +strict-transport-security: max-age=15768000 +x-frame-options: DENY +x-content-type-options: nosniff +x-xss-protection: 1; mode=block +HTTP/2 200 +server: {nginx_version_string} +content-type: text/html; charset=utf-8 +vary: Accept-Encoding +cache-control: max-age=0, no-cache, no-store, must-revalidate, private +vary: Cookie, Accept-Language +content-language: en +strict-transport-security: max-age=15768000 +x-frame-options: DENY +x-content-type-options: nosniff +x-xss-protection: 1; mode=block diff --git a/tools/lib/provision.py b/tools/lib/provision.py index b1bd48fd9b..229a571d7b 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -122,8 +122,8 @@ COMMON_DEPENDENCIES = [ "rabbitmq-server", "supervisor", "git", - "wget", - "ca-certificates", # Explicit dependency in case e.g. wget is already installed + "curl", + "ca-certificates", # Explicit dependency in case e.g. curl is already installed "puppet", # Used by lint (`puppet parser validate`) "gettext", # Used by makemessages i18n "transifex-client", # Needed to sync translations from transifex diff --git a/tools/linter_lib/custom_check.py b/tools/linter_lib/custom_check.py index 2d7ae29ef0..4161352e03 100644 --- a/tools/linter_lib/custom_check.py +++ b/tools/linter_lib/custom_check.py @@ -58,6 +58,7 @@ shebang_rules: List["Rule"] = [ trailing_whitespace_rule: "Rule" = { "pattern": r"\s+$", "strip": "\n", + "exclude": {"tools/ci/success-http-headers.template.txt"}, "description": "Fix trailing whitespace", } whitespace_rules: List["Rule"] = [ @@ -70,10 +71,6 @@ whitespace_rules: List["Rule"] = [ { "pattern": "\t", "strip": "\n", - "exclude": { - "tools/ci/success-http-headers.template.txt", - "tools/ci/success-http-headers.template.debian.txt", - }, "description": "Fix tab-based whitespace", }, ] diff --git a/tools/setup/bootstrap-aws-installer b/tools/setup/bootstrap-aws-installer index d407b77e85..c0a5219853 100644 --- a/tools/setup/bootstrap-aws-installer +++ b/tools/setup/bootstrap-aws-installer @@ -37,7 +37,7 @@ export DEBIAN_FRONTEND=noninteractive ( apt-get -qy update apt-get -qy -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade - apt-get -qy install jq unzip wget + apt-get -qy install jq unzip curl apt-get -qy autoclean ) diff --git a/tools/setup/bootstrap-awscli b/tools/setup/bootstrap-awscli index 678df76f4c..0958fe0711 100755 --- a/tools/setup/bootstrap-awscli +++ b/tools/setup/bootstrap-awscli @@ -7,7 +7,7 @@ if [ ! -d "/srv/zulip-aws-tools/v2/$AWS_CLI_VERSION" ]; then mkdir -p /srv/zulip-aws-tools cd /srv/zulip-aws-tools || exit 1 rm -rf awscli.zip awscli.zip.sha256 aws/ - wget -q "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -O awscli.zip + curl -L "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -o awscli.zip echo "$AWS_CLI_SHA awscli.zip" >awscli.zip.sha256 sha256sum -c awscli.zip.sha256 unzip -q awscli.zip diff --git a/tools/setup/install-shellcheck b/tools/setup/install-shellcheck index 528598b273..296cd155e9 100755 --- a/tools/setup/install-shellcheck +++ b/tools/setup/install-shellcheck @@ -19,7 +19,7 @@ if ! check_version; then tmpdir="$(mktemp -d)" trap 'rm -r "$tmpdir"' EXIT cd "$tmpdir" - wget -nv "https://github.com/koalaman/shellcheck/releases/download/v$version/$tarball" + curl -LO "https://github.com/koalaman/shellcheck/releases/download/v$version/$tarball" sha256sum -c <<<"${sha256[$arch]} $tarball" tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /usr/local/bin "shellcheck-v$version/shellcheck" check_version diff --git a/tools/setup/install-shfmt b/tools/setup/install-shfmt index e509b7fcf4..91ec2593df 100755 --- a/tools/setup/install-shfmt +++ b/tools/setup/install-shfmt @@ -24,7 +24,7 @@ if ! check_version; then tmpdir="$(mktemp -d)" trap 'rm -r "$tmpdir"' EXIT cd "$tmpdir" - wget -nv "https://github.com/mvdan/sh/releases/download/v$version/$binary" + curl -LO "https://github.com/mvdan/sh/releases/download/v$version/$binary" sha256sum -c <<<"$sha256 $binary" chmod +x "$binary" mv "$binary" /usr/local/bin/shfmt diff --git a/tools/test-install/prepare-base b/tools/test-install/prepare-base index d72c689931..33afb18888 100755 --- a/tools/test-install/prepare-base +++ b/tools/test-install/prepare-base @@ -48,7 +48,7 @@ run apt-get dist-upgrade -y # As an optimization, we install a bunch of packages the installer # would install for itself. run apt-get install -y --no-install-recommends \ - xvfb parallel unzip zip jq python3-pip wget curl eatmydata \ + xvfb parallel unzip zip jq python3-pip curl eatmydata \ git crudini openssl ssl-cert \ build-essential python3-dev \ memcached redis-server \