From 710a92f48c7c07255c9d478788538b64867c7b72 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Thu, 26 Aug 2021 02:12:16 +0000 Subject: [PATCH] tools: Add a tool to upload release tarballs to S3. --- docs/subsystems/release-checklist.md | 4 +- pyproject.toml | 1 + requirements/dev.in | 3 + requirements/dev.txt | 4 + tools/upload-release | 107 +++++++++++++++++++++++++++ version.py | 2 +- 6 files changed, 117 insertions(+), 4 deletions(-) create mode 100755 tools/upload-release diff --git a/docs/subsystems/release-checklist.md b/docs/subsystems/release-checklist.md index bfc42e0b94..77a80a42ff 100644 --- a/docs/subsystems/release-checklist.md +++ b/docs/subsystems/release-checklist.md @@ -55,9 +55,7 @@ preparing a new release. * Tag that commit with an unsigned Git tag named the release number. * Use `build-release-tarball` to generate a final release tarball. * Push the tag and release commit. -* Copy the tarball to `zulip.org`, and run - `/etc/zulip/ship-release.sh` on it; this will put it in place, - update the latest symlink, and update the SHA256 sums. +* Upload the tarball using `tools/upload-release`. * Post the release by [editing the latest tag on GitHub](https://github.com/zulip/zulip/tags); use the text from `changelog.md` for the release notes. diff --git a/pyproject.toml b/pyproject.toml index 5ff7126222..5388e1f4ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ module = [ "ldap.*", "markdown_include.*", "moto.*", + "natsort.*", "netifaces.*", "onelogin.*", "openapi_core.*", diff --git a/requirements/dev.in b/requirements/dev.in index dc79b7dd8d..e2a80fca31 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -77,3 +77,6 @@ semgrep # Contains Pysa, a security-focused static analyzer pyre-check + +# For sorting versions when uploading releases +natsort diff --git a/requirements/dev.txt b/requirements/dev.txt index d9e86dccb9..d35fbad28f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -847,6 +847,10 @@ myst-parser==0.15.1 \ --hash=sha256:7c3c78a36c4bc30ce6a67933ebd800a880c8d81f1688fad5c2ebd82cddbc1603 \ --hash=sha256:e8baa9959dac0bcf0f3ea5fc32a1a28792959471d8a8094e3ed5ee0de9733ade # via -r requirements/docs.in +natsort==7.1.1 \ + --hash=sha256:00c603a42365830c4722a2eb7663a25919551217ec09a243d3399fa8dd4ac403 \ + --hash=sha256:d0f4fc06ca163fa4a5ef638d9bf111c67f65eedcc7920f98dec08e489045b67e + # via -r requirements/dev.in oauthlib==3.1.1 \ --hash=sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc \ --hash=sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3 diff --git a/tools/upload-release b/tools/upload-release new file mode 100755 index 0000000000..30060b4875 --- /dev/null +++ b/tools/upload-release @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import os +import sys +import tempfile +from typing import IO, Dict + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from scripts.lib.setup_path import setup_path + +setup_path() + +import boto3 +from mypy_boto3_s3 import S3Client +from natsort import natsorted + + +def sha256_contents(f: IO[bytes]) -> str: + sha256_hash = hashlib.sha256() + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +parser = argparse.ArgumentParser(description="Upload a Zulip release") +parser.add_argument( + dest="filename", + help="Tarball to upload", + metavar="FILE", +) +args = parser.parse_args() +if not os.path.exists(args.filename): + parser.error(f"File does not exist: {args.filename}") +new_basename = os.path.basename(args.filename) +if not new_basename.startswith("zulip-server-") or not new_basename.endswith(".tar.gz"): + parser.error("File does not match zulip-server-*.tar.gz") + +session = boto3.Session() +client: S3Client = session.client("s3") +bucket = session.resource("s3", region_name="us-east-1").Bucket("zulip-download") + +file_hashes: Dict[str, str] = {} +with open(args.filename, "rb") as new_file: + print(f"Hashing {new_basename}..") + file_hashes[new_basename] = sha256_contents(new_file) + +print("Fetching existing hashes..") +for obj_summary in bucket.objects.filter(Prefix="server/zulip-server-"): + head = client.head_object(Bucket=bucket.name, Key=obj_summary.key) + assert obj_summary.key.startswith("server/") + filename = obj_summary.key[len("server/") :] + metadata = head["Metadata"] + if "sha256sum" not in metadata: + print(f" {filename} does not have SHA256 metadata!") + with tempfile.TemporaryFile() as obj_contents: + print(" Downloading..") + bucket.download_fileobj(Key=obj_summary.key, Fileobj=obj_contents) + obj_contents.seek(0) + print(" Hashing..") + metadata["sha256sum"] = sha256_contents(obj_contents) + print(f" Got {metadata['sha256sum']}") + + print(" Updating..") + obj_summary.copy_from( + ContentType=head["ContentType"], + CopySource=f"{bucket.name}/{obj_summary.key}", + Metadata=metadata, + MetadataDirective="REPLACE", + ) + + file_hashes[filename] = metadata["sha256sum"] + +ordered_filenames = natsorted(file_hashes.keys(), reverse=True) +assert ordered_filenames[0] == "zulip-server-latest.tar.gz" + +print(f"Uploading {new_basename}..") +extra = { + "Metadata": {"sha256sum": file_hashes[new_basename]}, + "ACL": "public-read", + "ContentType": "application/gzip", +} +bucket.upload_file( + Filename=args.filename, + Key=f"server/{new_basename}", + ExtraArgs=extra, +) + +if ordered_filenames[1] == new_basename: + print("Copying to zulip-server-latest.tar.gz..") + bucket.copy( + CopySource={"Bucket": bucket.name, "Key": f"server/{new_basename}"}, + Key="server/zulip-server-latest.tar.gz", + ExtraArgs=extra, + ) + file_hashes["zulip-server-latest.tar.gz"] = file_hashes[new_basename] + +print("Updating SHA256SUMS.txt..") +contents = b"" +for filename in ordered_filenames: + contents += f"{file_hashes[filename]} {filename}\n".encode() +bucket.put_object( + ACL="public-read", + Body=contents, + ContentType="text/plain", + Key="server/SHA256SUMS.txt", +) diff --git a/version.py b/version.py index 9bcbed47eb..b510f55412 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 93 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = "155.0" +PROVISION_VERSION = "155.1"