install: Replace wget with curl.

curl uses Happy Eyeballs to avoid long timeouts on systems with broken
IPv6.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

docs/production/install-existing-server.md

@@ -28,7 +28,7 @@ one created by Zulip into it:
 
 ```shell
 sudo cp /etc/nginx/nginx.conf /etc/nginx.conf.before-zulip-install
-sudo wget -O /etc/nginx/nginx.conf.zulip \
+sudo curl -Lo /etc/nginx/nginx.conf.zulip \
     https://raw.githubusercontent.com/zulip/zulip/master/puppet/zulip/templates/nginx.conf.template.erb
 sudo meld /etc/nginx/nginx.conf /etc/nginx/nginx.conf.zulip  # be sure to merge to the right
 ```

docs/production/install.md

@@ -20,7 +20,7 @@ with the following commands:
 
 ```
 cd $(mktemp -d)
-wget https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz
+curl -LO https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz
 tar -xf zulip-server-latest.tar.gz
 ```
 

docs/production/upgrade-or-modify.md

@@ -28,7 +28,7 @@ to a new Zulip release:
     release with:
 
     ```
-    wget https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz
+    curl -LO https://www.zulip.org/dist/releases/zulip-server-latest.tar.gz
     ```
 
     You also have the option of upgrading Zulip [to a version in a Git

docs/tutorials/shell-tips.md

@@ -178,7 +178,7 @@ environment:
 
 ```
 sudo apt-get -y purge vagrant && \
-wget https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb && \
+curl -LO https://releases.hashicorp.com/vagrant/2.0.2/vagrant_2.0.2_x86_64.deb && \
 sudo dpkg -i vagrant*.deb && \
 sudo apt-get -y install build-essential git ruby lxc lxc-templates cgroup-lite redir && \
 vagrant plugin install vagrant-lxc && \

puppet/zulip/manifests/profile/base.pp

@@ -39,8 +39,9 @@ class zulip::profile::base {
         'python3-yaml',
         'puppet',
         'git',
-        # Used in scripts including install-yarn.sh
+        # Used for most downloads
         'curl',
+        # Used in check_website_response.sh
         'wget',
         'jq',
         'procps',

scripts/lib/build-pgroonga

@@ -6,7 +6,7 @@ PGROONGA_VERSION="2.2.8"
 
 cd "$(mktemp -d)"
 
-wget https://packages.groonga.org/source/pgroonga/pgroonga-"$PGROONGA_VERSION".tar.gz
+curl -LO https://packages.groonga.org/source/pgroonga/pgroonga-"$PGROONGA_VERSION".tar.gz
 tar xf pgroonga-"$PGROONGA_VERSION".tar.gz
 cd pgroonga-"$PGROONGA_VERSION"
 

scripts/lib/install

@@ -344,7 +344,7 @@ if [ "$package_system" = apt ]; then
     # `zulip::profile::base` such that the new dependency is seen by
     # upgrades, as well as new installs.
     if ! apt-get install -y \
-        python3 python3-yaml puppet git curl wget jq crudini \
+        python3 python3-yaml puppet git curl jq crudini \
         "${ADDITIONAL_PACKAGES[@]}"; then
         set +x
         echo -e '\033[0;31m' >&2
@@ -355,7 +355,7 @@ if [ "$package_system" = apt ]; then
     fi
 elif [ "$package_system" = yum ]; then
     if ! yum install -y \
-        python3 python3-pyyaml puppet git curl wget jq crudini \
+        python3 python3-pyyaml puppet git curl jq crudini \
         "${ADDITIONAL_PACKAGES[@]}"; then
         set +x
         echo -e '\033[0;31m' >&2

scripts/lib/install-node

@@ -36,11 +36,11 @@ if [ "$current_node_version" != "v$node_version" ] || ! [ -L "$node_wrapper_path
         [ "$(nvm --version)" != "$nvm_version" ]
     }; then
         mkdir -p "$NVM_DIR"
-        wget_opts=(-nv)
+        curl_opts=(-L)
         if [ -n "${CUSTOM_CA_CERTIFICATES:-}" ]; then
-            wget_opts+=(--ca-certificate "${CUSTOM_CA_CERTIFICATES}")
+            curl_opts+=(--cacert "${CUSTOM_CA_CERTIFICATES}")
         fi
-        wget "${wget_opts[@]}" -O- "https://raw.githubusercontent.com/nvm-sh/nvm/v$nvm_version/install.sh" | bash
+        curl "${curl_opts[@]}" "https://raw.githubusercontent.com/nvm-sh/nvm/v$nvm_version/install.sh" | bash
         # shellcheck source=/dev/null
         . "$NVM_DIR/nvm.sh"
     fi

scripts/lib/setup-apt-repo

@@ -72,7 +72,7 @@ touch "$SOURCES_FILE"
 # Hash it to check if the sources file is changed by the script later.
 zulip_source_hash=$(sha1sum "$SOURCES_FILE")
 
-pre_setup_deps=(lsb-release apt-transport-https ca-certificates gnupg wget)
+pre_setup_deps=(lsb-release apt-transport-https ca-certificates gnupg curl)
 if ! apt-get -dy install "${pre_setup_deps[@]}"; then
     apt-get update
 fi

scripts/setup/apt-repos/zulip/custom.sh

@@ -23,8 +23,8 @@ if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then
         distribution=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
         groonga_apt_source_deb="groonga-apt-source-latest-$release.deb"
         groonga_apt_source_deb_sign="$groonga_apt_source_deb.asc.$pgroonga_apt_sign_key_fingerprint"
-        wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb"
-        wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb_sign"
+        curl -LO "https://packages.groonga.org/$distribution/$groonga_apt_source_deb"
+        curl -LO "https://packages.groonga.org/$distribution/$groonga_apt_source_deb_sign"
         gpg \
             --homedir="$tmp_gpg_home" \
             --verify \

scripts/setup/sha256-tarball-to

@@ -25,7 +25,7 @@ cd "$tmpdir"
 
 # Fetch to a predictable name, not whatever curl guesses from the URL
 LOCALFILE="archive.tar.gz"
-wget -qO "$LOCALFILE" "$URL"
+curl -Lo "$LOCALFILE" "$URL"
 
 # Check the hash against what was passed in
 echo "$SHA256  $LOCALFILE" >"$LOCALFILE.sha256"

templates/zerver/help/desktop-app-install-guide.md

@@ -55,7 +55,7 @@ to future versions. <!-- TODO fact check -->
 
 1. Enter the following commands into a terminal:
 
-        sudo wget -O /etc/apt/trusted.gpg.d/zulip-desktop.asc \
+        sudo curl -Lo /etc/apt/trusted.gpg.d/zulip-desktop.asc \
             https://download.zulip.com/desktop/apt/zulip-desktop.asc
         echo "deb https://download.zulip.com/desktop/apt stable main" | \
             sudo tee /etc/apt/sources.list.d/zulip-desktop.list

tools/ci/.gitattributes

@@ -0,0 +1 @@
+success-http-headers.template.txt eol=crlf

tools/ci/Dockerfile.prod

@@ -17,7 +17,7 @@ RUN if (. /etc/os-release && [ "$ID $VERSION_ID" = 'ubuntu 18.04' ]); then \
 # Download the release tarball, start rabbitmq server and install the server
 ARG VERSION
 RUN cd $(mktemp -d) \
-  && wget "https://www.zulip.org/dist/releases/zulip-server-$VERSION.tar.gz" \
+  && curl -LO "https://www.zulip.org/dist/releases/zulip-server-$VERSION.tar.gz" \
   && tar -xf "zulip-server-$VERSION.tar.gz" \
   && sudo service rabbitmq-server start \
   && sudo service rabbitmq-server status \

tools/ci/production-build

@@ -34,7 +34,6 @@ mkdir /tmp/production-build
 mv /tmp/tmp.*/zulip-server-test.tar.gz /tmp/production-build
 cp -a \
     tools/ci/success-http-headers.template.txt \
-    tools/ci/success-http-headers.template.debian.txt \
     tools/ci/production-install \
     tools/ci/production-verify \
     tools/ci/production-upgrade \

tools/ci/production-verify

@@ -12,17 +12,9 @@ NOREPLY_EMAIL_ADDRESS = 'noreply@circleci.example.com'
 ALLOWED_HOSTS = []
 EOF
 
-os_info="$(
-    . /etc/os-release
-    printf '%s\n' "$ID"
-)"
-{
-    read -r os_id
-} <<<"$os_info"
-
 check_header() {
-    sed -i -e 's|Length: [0-9]\+\( ([0-9.]\+K)\)\?|Length: <Length>|' -e "s|{nginx_version_string}|$nginx_version|g" "$success_header_file"
-    if ! diff -ur /tmp/http-headers-processed "$success_header_file"; then
+    if ! sed "s|{nginx_version_string}|$nginx_version|g" "$success_header_file" \
+        | diff -ur - /tmp/http-headers-processed; then
         set +x
         echo
         echo "FAILURE: The HTTP headers returned from loading the homepage on the server do not match the contents of tools/ci/success-http-headers.template.txt.  Typically, this means that the server threw a 500 when trying to load the homepage."
@@ -67,20 +59,13 @@ echo
 echo "Now testing that the newly installed server's homepage loads"
 echo
 
-wget https://localhost -O /tmp/index.html --no-check-certificate -S 2>/tmp/wget-output || true # || true so we see errors.log if this 500s
-grep -vi '\(Vary\|Content-Language\|expires\|issued by\|modified\|saved\|[.][.][.]\|Date\|[-][-]\)' /tmp/wget-output >/tmp/http-headers-processed
+curl -ILk https://localhost -o /tmp/http-headers
+grep -vi -e '^content-length:' -e '^date:' -e '^expires:' -e '^set-cookie:' /tmp/http-headers >/tmp/http-headers-processed
 
-nginx_version="$(nginx -v 2>&1 | awk '{print $3, $4}' | xargs)"
-
-# Simplify the diff by getting replacing 4-5 digit length numbers with <Length>.
-sed -i 's|Length: [0-9]\+\( ([0-9.]\+K)\)\?|Length: <Length>|' /tmp/http-headers-processed
-if [ "$os_id" = debian ]; then
-    success_header_file="/tmp/success-http-headers.template.debian.txt"
-    check_header
-else
-    success_header_file="/tmp/success-http-headers.template.txt"
-    check_header
-fi
+nginx_version="$(nginx -v 2>&1)"
+nginx_version="${nginx_version#nginx version: }"
+success_header_file="/tmp/success-http-headers.template.txt"
+check_header
 
 # Start the RabbitMQ queue worker related section
 echo

tools/ci/success-http-headers.template.debian.txt

@@ -1,29 +0,0 @@
-WARNING: The certificate of ‘localhost’ is not trusted.
-WARNING: The certificate of ‘localhost’ doesn't have a known issuer.
-The certificate's owner does not match hostname ‘localhost’
-  HTTP/1.1 302 Found
-  Server: {nginx_version_string}
-  Content-Type: text/html; charset=utf-8
-  Content-Length: 0
-  Connection: keep-alive
-  Location: /login/
-  Strict-Transport-Security: max-age=15768000
-  X-Frame-Options: DENY
-  X-Content-Type-Options: nosniff
-  X-XSS-Protection: 1; mode=block
-Location: /login/ [following]
-Reusing existing connection to localhost:443.
-  HTTP/1.1 200 OK
-  Server: {nginx_version_string}
-  Content-Type: text/html; charset=utf-8
-  Content-Length: 6361
-  Connection: keep-alive
-  Strict-Transport-Security: max-age=15768000
-  X-Frame-Options: DENY
-  X-Content-Type-Options: nosniff
-  X-XSS-Protection: 1; mode=block
-Length: 6361 (6.2K) [text/html]
-Saving to: ‘/tmp/index.html’
-
-
-

tools/ci/success-http-headers.template.txt

@@ -1,28 +1,23 @@
-  Self-signed certificate encountered.
-    WARNING: certificate common name ‘127.0.0.1’ doesn't match requested host name ‘localhost’.
-  HTTP/1.1 302 Found
-  Server: {nginx_version_string}
-  Content-Type: text/html; charset=utf-8
-  Content-Length: 0
-  Connection: keep-alive
-  Location: /login/
-  Strict-Transport-Security: max-age=15768000
-  X-Frame-Options: DENY
-  X-Content-Type-Options: nosniff
-  X-XSS-Protection: 1; mode=block
-Location: /login/ [following]
-Reusing existing connection to localhost:443.
-  HTTP/1.1 200 OK
-  Server: {nginx_version_string}
-  Content-Type: text/html; charset=utf-8
-  Content-Length: 6361
-  Connection: keep-alive
-  Strict-Transport-Security: max-age=15768000
-  X-Frame-Options: DENY
-  X-Content-Type-Options: nosniff
-  X-XSS-Protection: 1; mode=block
-Length: 6361 (6.2K) [text/html]
-Saving to: ‘/tmp/index.html’
-
+HTTP/2 302 
+server: {nginx_version_string}
+content-type: text/html; charset=utf-8
+location: /login/
+vary: Accept-Language, Cookie
+content-language: en
+strict-transport-security: max-age=15768000
+x-frame-options: DENY
+x-content-type-options: nosniff
+x-xss-protection: 1; mode=block
 
+HTTP/2 200 
+server: {nginx_version_string}
+content-type: text/html; charset=utf-8
+vary: Accept-Encoding
+cache-control: max-age=0, no-cache, no-store, must-revalidate, private
+vary: Cookie, Accept-Language
+content-language: en
+strict-transport-security: max-age=15768000
+x-frame-options: DENY
+x-content-type-options: nosniff
+x-xss-protection: 1; mode=block
 

tools/lib/provision.py

@@ -122,8 +122,8 @@ COMMON_DEPENDENCIES = [
     "rabbitmq-server",
     "supervisor",
     "git",
-    "wget",
-    "ca-certificates",  # Explicit dependency in case e.g. wget is already installed
+    "curl",
+    "ca-certificates",  # Explicit dependency in case e.g. curl is already installed
     "puppet",  # Used by lint (`puppet parser validate`)
     "gettext",  # Used by makemessages i18n
     "transifex-client",  # Needed to sync translations from transifex

tools/linter_lib/custom_check.py

@@ -58,6 +58,7 @@ shebang_rules: List["Rule"] = [
 trailing_whitespace_rule: "Rule" = {
     "pattern": r"\s+$",
     "strip": "\n",
+    "exclude": {"tools/ci/success-http-headers.template.txt"},
     "description": "Fix trailing whitespace",
 }
 whitespace_rules: List["Rule"] = [
@@ -70,10 +71,6 @@ whitespace_rules: List["Rule"] = [
     {
         "pattern": "\t",
         "strip": "\n",
-        "exclude": {
-            "tools/ci/success-http-headers.template.txt",
-            "tools/ci/success-http-headers.template.debian.txt",
-        },
         "description": "Fix tab-based whitespace",
     },
 ]

tools/setup/bootstrap-aws-installer

@@ -37,7 +37,7 @@ export DEBIAN_FRONTEND=noninteractive
 (
     apt-get -qy update
     apt-get -qy -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade
-    apt-get -qy install jq unzip wget
+    apt-get -qy install jq unzip curl
     apt-get -qy autoclean
 )
 

tools/setup/bootstrap-awscli

@@ -7,7 +7,7 @@ if [ ! -d "/srv/zulip-aws-tools/v2/$AWS_CLI_VERSION" ]; then
     mkdir -p /srv/zulip-aws-tools
     cd /srv/zulip-aws-tools || exit 1
     rm -rf awscli.zip awscli.zip.sha256 aws/
-    wget -q "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -O awscli.zip
+    curl -L "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -o awscli.zip
     echo "$AWS_CLI_SHA  awscli.zip" >awscli.zip.sha256
     sha256sum -c awscli.zip.sha256
     unzip -q awscli.zip

tools/setup/install-shellcheck

@@ -19,7 +19,7 @@ if ! check_version; then
     tmpdir="$(mktemp -d)"
     trap 'rm -r "$tmpdir"' EXIT
     cd "$tmpdir"
-    wget -nv "https://github.com/koalaman/shellcheck/releases/download/v$version/$tarball"
+    curl -LO "https://github.com/koalaman/shellcheck/releases/download/v$version/$tarball"
     sha256sum -c <<<"${sha256[$arch]} $tarball"
     tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /usr/local/bin "shellcheck-v$version/shellcheck"
     check_version

tools/setup/install-shfmt

@@ -24,7 +24,7 @@ if ! check_version; then
     tmpdir="$(mktemp -d)"
     trap 'rm -r "$tmpdir"' EXIT
     cd "$tmpdir"
-    wget -nv "https://github.com/mvdan/sh/releases/download/v$version/$binary"
+    curl -LO "https://github.com/mvdan/sh/releases/download/v$version/$binary"
     sha256sum -c <<<"$sha256 $binary"
     chmod +x "$binary"
     mv "$binary" /usr/local/bin/shfmt

tools/test-install/prepare-base

@@ -48,7 +48,7 @@ run apt-get dist-upgrade -y
 # As an optimization, we install a bunch of packages the installer
 # would install for itself.
 run apt-get install -y --no-install-recommends \
-    xvfb parallel unzip zip jq python3-pip wget curl eatmydata \
+    xvfb parallel unzip zip jq python3-pip curl eatmydata \
     git crudini openssl ssl-cert \
     build-essential python3-dev \
     memcached redis-server \