tools: Add a tool to upload release tarballs to S3.

docs/subsystems/release-checklist.md

@@ -55,9 +55,7 @@ preparing a new release.
 * Tag that commit with an unsigned Git tag named the release number.
 * Use `build-release-tarball` to generate a final release tarball.
 * Push the tag and release commit.
-* Copy the tarball to `zulip.org`, and run
-  `/etc/zulip/ship-release.sh` on it; this will put it in place,
-  update the latest symlink, and update the SHA256 sums.
+* Upload the tarball using `tools/upload-release`.
 * Post the release by [editing the latest tag on
   GitHub](https://github.com/zulip/zulip/tags); use the text from
   `changelog.md` for the release notes.

pyproject.toml

@@ -72,6 +72,7 @@ module = [
     "ldap.*",
     "markdown_include.*",
     "moto.*",
+    "natsort.*",
     "netifaces.*",
     "onelogin.*",
     "openapi_core.*",

requirements/dev.in

@@ -77,3 +77,6 @@ semgrep
 
 # Contains Pysa, a security-focused static analyzer
 pyre-check
+
+# For sorting versions when uploading releases
+natsort

requirements/dev.txt

@@ -847,6 +847,10 @@ myst-parser==0.15.1 \
     --hash=sha256:7c3c78a36c4bc30ce6a67933ebd800a880c8d81f1688fad5c2ebd82cddbc1603 \
     --hash=sha256:e8baa9959dac0bcf0f3ea5fc32a1a28792959471d8a8094e3ed5ee0de9733ade
     # via -r requirements/docs.in
+natsort==7.1.1 \
+    --hash=sha256:00c603a42365830c4722a2eb7663a25919551217ec09a243d3399fa8dd4ac403 \
+    --hash=sha256:d0f4fc06ca163fa4a5ef638d9bf111c67f65eedcc7920f98dec08e489045b67e
+    # via -r requirements/dev.in
 oauthlib==3.1.1 \
     --hash=sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc \
     --hash=sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3

tools/upload-release

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import os
+import sys
+import tempfile
+from typing import IO, Dict
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import boto3
+from mypy_boto3_s3 import S3Client
+from natsort import natsorted
+
+
+def sha256_contents(f: IO[bytes]) -> str:
+    sha256_hash = hashlib.sha256()
+    for byte_block in iter(lambda: f.read(4096), b""):
+        sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+parser = argparse.ArgumentParser(description="Upload a Zulip release")
+parser.add_argument(
+    dest="filename",
+    help="Tarball to upload",
+    metavar="FILE",
+)
+args = parser.parse_args()
+if not os.path.exists(args.filename):
+    parser.error(f"File does not exist: {args.filename}")
+new_basename = os.path.basename(args.filename)
+if not new_basename.startswith("zulip-server-") or not new_basename.endswith(".tar.gz"):
+    parser.error("File does not match zulip-server-*.tar.gz")
+
+session = boto3.Session()
+client: S3Client = session.client("s3")
+bucket = session.resource("s3", region_name="us-east-1").Bucket("zulip-download")
+
+file_hashes: Dict[str, str] = {}
+with open(args.filename, "rb") as new_file:
+    print(f"Hashing {new_basename}..")
+    file_hashes[new_basename] = sha256_contents(new_file)
+
+print("Fetching existing hashes..")
+for obj_summary in bucket.objects.filter(Prefix="server/zulip-server-"):
+    head = client.head_object(Bucket=bucket.name, Key=obj_summary.key)
+    assert obj_summary.key.startswith("server/")
+    filename = obj_summary.key[len("server/") :]
+    metadata = head["Metadata"]
+    if "sha256sum" not in metadata:
+        print(f"  {filename} does not have SHA256 metadata!")
+        with tempfile.TemporaryFile() as obj_contents:
+            print("    Downloading..")
+            bucket.download_fileobj(Key=obj_summary.key, Fileobj=obj_contents)
+            obj_contents.seek(0)
+            print("    Hashing..")
+            metadata["sha256sum"] = sha256_contents(obj_contents)
+            print(f"    Got {metadata['sha256sum']}")
+
+        print("    Updating..")
+        obj_summary.copy_from(
+            ContentType=head["ContentType"],
+            CopySource=f"{bucket.name}/{obj_summary.key}",
+            Metadata=metadata,
+            MetadataDirective="REPLACE",
+        )
+
+    file_hashes[filename] = metadata["sha256sum"]
+
+ordered_filenames = natsorted(file_hashes.keys(), reverse=True)
+assert ordered_filenames[0] == "zulip-server-latest.tar.gz"
+
+print(f"Uploading {new_basename}..")
+extra = {
+    "Metadata": {"sha256sum": file_hashes[new_basename]},
+    "ACL": "public-read",
+    "ContentType": "application/gzip",
+}
+bucket.upload_file(
+    Filename=args.filename,
+    Key=f"server/{new_basename}",
+    ExtraArgs=extra,
+)
+
+if ordered_filenames[1] == new_basename:
+    print("Copying to zulip-server-latest.tar.gz..")
+    bucket.copy(
+        CopySource={"Bucket": bucket.name, "Key": f"server/{new_basename}"},
+        Key="server/zulip-server-latest.tar.gz",
+        ExtraArgs=extra,
+    )
+    file_hashes["zulip-server-latest.tar.gz"] = file_hashes[new_basename]
+
+print("Updating SHA256SUMS.txt..")
+contents = b""
+for filename in ordered_filenames:
+    contents += f"{file_hashes[filename]}  {filename}\n".encode()
+bucket.put_object(
+    ACL="public-read",
+    Body=contents,
+    ContentType="text/plain",
+    Key="server/SHA256SUMS.txt",
+)

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 93
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "155.0"
+PROVISION_VERSION = "155.1"