diff --git a/docs/production/upload-backends.md b/docs/production/upload-backends.md index 7fe1c65e1e..b3609b7484 100644 --- a/docs/production/upload-backends.md +++ b/docs/production/upload-backends.md @@ -206,3 +206,38 @@ Congratulations! Your uploaded files are now migrated to S3. **Caveat**: The current version of this tool does not migrate an uploaded organization avatar or logo. + +## S3 data storage class + +In general, uploaded files in Zulip are accessed frequently at first, and then +age out of frequent access. The S3 backend provides the [S3 +Intelligent-Tiering][s3-it] [storage class][s3-storage-class] which provides +cheaper storage for less frequently accessed objects, and may provide overall +cost savings for large deployments. + +You can configure Zulip to store uploaded files using Intelligent-Tiering by +setting `S3_UPLOADS_STORAGE_CLASS` to `INTELLIGENT_TIERING` in `settings.py`. +This setting can take any of the following [storage class +value][s3-storage-class-constant] values: + +- `STANDARD` +- `STANDARD_IA` +- `ONEZONE_IA` +- `REDUCED_REDUNDANCY` +- `GLACIER_IR` +- `INTELLIGENT_TIERING` + +Setting `S3_UPLOADS_STORAGE_CLASS` does not affect the storage class of existing +objects. In order to change those, for example to `INTELLIGENT_TIERING`, perform +an in-place copy: + + aws s3 cp --storage-class INTELLIGENT_TIERING --recursive \ + s3://your-bucket-name/ s3://your-bucket-name/ + +Note that changing the lifecycle of existing objects will incur a [one-time +lifecycle transition cost][s3-pricing]. + +[s3-it]: https://aws.amazon.com/s3/storage-classes/intelligent-tiering/ +[s3-storage-class]: https://aws.amazon.com/s3/storage-classes/ +[s3-storage-class-constant]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass +[s3-pricing]: https://aws.amazon.com/s3/pricing/ diff --git a/zerver/lib/transfer.py b/zerver/lib/transfer.py index e5dfc1cce9..e80709f28b 100644 --- a/zerver/lib/transfer.py +++ b/zerver/lib/transfer.py @@ -65,6 +65,7 @@ def _transfer_message_files_to_s3(attachment: Attachment) -> None: guessed_type, attachment.owner, f.read(), + settings.S3_UPLOADS_STORAGE_CLASS, ) logging.info("Uploaded message file in path %s", file_path) except FileNotFoundError: # nocoverage diff --git a/zerver/lib/upload/s3.py b/zerver/lib/upload/s3.py index af9e484863..669f4e3eea 100644 --- a/zerver/lib/upload/s3.py +++ b/zerver/lib/upload/s3.py @@ -4,7 +4,7 @@ import secrets import urllib from datetime import datetime from mimetypes import guess_type -from typing import IO, Any, BinaryIO, Callable, Iterator, List, Optional, Tuple +from typing import IO, Any, BinaryIO, Callable, Iterator, List, Literal, Optional, Tuple import boto3 import botocore @@ -71,6 +71,14 @@ def upload_image_to_s3( content_type: Optional[str], user_profile: UserProfile, contents: bytes, + storage_class: Literal[ + "GLACIER_IR", + "INTELLIGENT_TIERING", + "ONEZONE_IA", + "REDUCED_REDUNDANCY", + "STANDARD", + "STANDARD_IA", + ] = "STANDARD", ) -> None: key = bucket.Object(file_name) metadata = { @@ -89,6 +97,7 @@ def upload_image_to_s3( Metadata=metadata, ContentType=content_type, ContentDisposition=content_disposition, + StorageClass=storage_class, ) @@ -224,6 +233,7 @@ class S3UploadBackend(ZulipUploadBackend): content_type, user_profile, file_data, + settings.S3_UPLOADS_STORAGE_CLASS, ) create_attachment( diff --git a/zproject/default_settings.py b/zproject/default_settings.py index 7523a97fbe..15d5513b39 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -1,6 +1,6 @@ import os from email.headerregistry import Address -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple from scripts.lib.zulip_tools import deport from zproject.settings_types import JwtAuthKey, OIDCIdPConfigDict, SAMLIdPConfigDict @@ -145,6 +145,14 @@ S3_AUTH_UPLOADS_BUCKET = "" S3_REGION: Optional[str] = None S3_ENDPOINT_URL: Optional[str] = None S3_SKIP_PROXY = True +S3_UPLOADS_STORAGE_CLASS: Literal[ + "GLACIER_IR", + "INTELLIGENT_TIERING", + "ONEZONE_IA", + "REDUCED_REDUNDANCY", + "STANDARD", + "STANDARD_IA", +] = "STANDARD" LOCAL_UPLOADS_DIR: Optional[str] = None LOCAL_AVATARS_DIR: Optional[str] = None LOCAL_FILES_DIR: Optional[str] = None diff --git a/zproject/prod_settings_template.py b/zproject/prod_settings_template.py index fd13e9aab3..80b16de867 100644 --- a/zproject/prod_settings_template.py +++ b/zproject/prod_settings_template.py @@ -768,6 +768,7 @@ LOCAL_UPLOADS_DIR = "/home/zulip/uploads" # S3_REGION = None # S3_ENDPOINT_URL = None # S3_SKIP_PROXY = True +# S3_UPLOADS_STORAGE_CLASS = "STANDARD" ## Maximum allowed size of uploaded files, in megabytes. This value is ## capped at 80MB in the nginx configuration, because the file upload