From 4df9b03451d8de6f2f850bc0da388afa79f0bfa2 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 17 Jun 2024 21:33:21 +0000 Subject: [PATCH] migrations: Re-thumbnail and rename existing emoji. --- static/images/bad-emoji.png | Bin 0 -> 1412 bytes zerver/lib/transfer.py | 4 +- zerver/migrations/0553_copy_emoji_images.py | 316 ++++++++++++++++++++ 3 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 static/images/bad-emoji.png create mode 100644 zerver/migrations/0553_copy_emoji_images.py diff --git a/static/images/bad-emoji.png b/static/images/bad-emoji.png new file mode 100644 index 0000000000000000000000000000000000000000..4a60b64e6abe70b20ff7cb3c8ec432303497c6b4 GIT binary patch literal 1412 zcmV-~1$+95P)2D?NY%?PN(TTo*ald{ z9g(8|00jt1L_t(|ob6iaaL^LXjE^ifKQZnukmzaJ~2 zNG04%K-ziEIf5X-YPG`QZ~*5#*GMy>m#!fqG@DIKr&H8wwL(cKqz#(R`@WC2w>KOP z2M8g~j5DR5rHHI9AcTPHx_Etkh39$ToEMe?HxZDIU>Js&&*$)cA3}(;Tu&QqR{X3Q zVqV8NM;L}!EEZ5o71kbaR)A^vdc8i2kT%dHOBX`UdHezpf1W16RG8;^*lae5-@Ro4 zRY(`$oMXS=W53@+2oW2tp`9WkSeAu;zmHa{1u11**upTx>2$(ww?hyFXEN7E&N&>% zL9f??l=5a3Tune)+Ml1FiTCNq6A^6N#$+-w<``H9yAH1enGg4u`lfnJ{yAJRZknq$6sc6A`48=yW=YRXf~T?PUfja zLocB2ISB#UL(t)bToCH$Wy^3b^HWZ6Oa`h^W0noW=(S@vSx|X z>6E!3(%0w~dTSco)CTh$GOy9dlXKQA;dve$$BCnywWw+~o8X*7DRr$20B%%(dHl&e z8d;GiBCJ*`D5VnX0P|>B7TWFhS(ms5m28Tp=Xo7@BEoz=$LVy+9e9Wcy3eU7z_%DK2S0NS(0arVYl01ynln{1jQvM^S|L_jhc!+jw6vE9Z#_olXZ&Pfxi7U5io`kS=zM#UkD~0nWRIUOA4( zV+@DGFWc!gD20HmsLtne9FNB{=S|7i>vc>f6SP{bM9vqn6{b;13g~fH?;53BDiNXE z?P4;SKnQU+=YbLkNU!f5$ARNGapW~j=ZC`~Mx#-j|LO6%NR+k?&^i6_@ezNP&SxUR zU@*XVJigm=fHdw-fGO|O=>*^R<0Y>7o!VHrz;r+lI9bjx#?bHgGkg56 zABVpKB8nf40q1*)mSv&QXe5@m_jFvN6asWKJDpBqb3`MGqWEEG*|<9arj&cV-o None: content_type = guess_type(emoji_path)[0] emoji_path = os.path.join(settings.LOCAL_AVATARS_DIR, emoji_path) + ".original" if content_type is None: # nocoverage + # This should not be possible after zerver/migrations/0553_copy_emoji_images.py logging.error("Emoji %d has no recognizable file extension", realm_emoji.id) return try: @@ -127,7 +128,8 @@ def _transfer_emoji_to_s3(realm_emoji: RealmEmoji) -> None: logging.info("Uploaded emoji file in path %s", emoji_path) except FileNotFoundError: # nocoverage logging.error("Emoji %d could not be loaded from local disk", realm_emoji.id) - except BadImageError as e: + except BadImageError as e: # nocoverage + # This should not be possible after zerver/migrations/0553_copy_emoji_images.py logging.error("Emoji %d is invalid: %s", realm_emoji.id, e) diff --git a/zerver/migrations/0553_copy_emoji_images.py b/zerver/migrations/0553_copy_emoji_images.py new file mode 100644 index 0000000000..ca05b0ad00 --- /dev/null +++ b/zerver/migrations/0553_copy_emoji_images.py @@ -0,0 +1,316 @@ +import contextlib +import hashlib +import logging +import os +from typing import Any, Iterator, Optional, Tuple + +import boto3 +import botocore +import pyvips +from botocore.client import Config +from django.conf import settings +from django.db import migrations +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + +from zerver.lib.mime_types import guess_extension, guess_type + +# From zerver.lib.thumbnail +DEFAULT_EMOJI_SIZE = 64 +IMAGE_BOMB_TOTAL_PIXELS = 90000000 +MAX_EMOJI_GIF_FILE_SIZE_BYTES = 128 * 1024 * 1024 # 128 kb + +# This is the intersection of INLINE_MIME_TYPES and THUMBNAIL_ACCEPT_IMAGE_TYPES +VALID_EMOJI_CONTENT_TYPE = frozenset( + [ + "image/avif", + "image/gif", + "image/jpeg", + "image/png", + "image/webp", + ] +) + + +class SkipImageError(Exception): + pass + + +# From zerver.lib.thumbnail, with minor exception changes +@contextlib.contextmanager +def libvips_check_image(image_data: bytes) -> Iterator[pyvips.Image]: + try: + source_image = pyvips.Image.new_from_buffer(image_data, "") + except pyvips.Error as e: + raise SkipImageError(f"Cannot process image: {e}") + + if source_image.width * source_image.height > IMAGE_BOMB_TOTAL_PIXELS: + raise SkipImageError(f"Image too big: {source_image.height} * {source_image.width}") + + try: + yield source_image + except pyvips.Error as e: + raise SkipImageError(f"Bad image data? {e}") + + +# From zerver.lib.thumbnail, with minor exception changes +def resize_emoji( + image_data: bytes, emoji_file_name: str, size: int = DEFAULT_EMOJI_SIZE +) -> Tuple[bytes, Optional[bytes]]: + if len(image_data) > MAX_EMOJI_GIF_FILE_SIZE_BYTES: + raise SkipImageError(f"Image has too many bytes: {len(image_data)}") + + # Square brackets are used for providing options to libvips' save + # operation; the extension on the filename comes from reversing + # the content-type, which removes most of the attacker control of + # this string, but assert it has no bracketed pieces for safety. + write_file_ext = os.path.splitext(emoji_file_name)[1] + assert "[" not in write_file_ext + + # This function returns two values: + # 1) Emoji image data. + # 2) If it is animated, the still image data i.e. first frame of gif. + with libvips_check_image(image_data) as source_image: + if source_image.get_n_pages() == 1: + return ( + pyvips.Image.thumbnail_buffer( + image_data, + size, + height=size, + crop=pyvips.Interesting.CENTRE, + ).write_to_buffer(write_file_ext), + None, + ) + first_still = pyvips.Image.thumbnail_buffer( + image_data, + size, + height=size, + crop=pyvips.Interesting.CENTRE, + ).write_to_buffer(".png") + + animated = pyvips.Image.thumbnail_buffer( + image_data, + size, + height=size, + # This is passed to the loader, and means "load all + # frames", instead of the default of just the first + option_string="n=-1", + ) + if animated.width != animated.get("page-height"): + # If the image is non-square, we have to iterate the + # frames to add padding to make it so + if not animated.hasalpha(): + animated = animated.addalpha() + frames = [ + frame.gravity( + pyvips.CompassDirection.CENTRE, + size, + size, + extend=pyvips.Extend.BACKGROUND, + background=[0, 0, 0, 0], + ) + for frame in animated.pagesplit() + ] + animated = frames[0].pagejoin(frames[1:]) + return (animated.write_to_buffer(write_file_ext), first_still) + + +# From zerver.lib.emoji +def get_emoji_file_name(content_type: str, emoji_id: int) -> str: + image_ext = guess_extension(content_type, strict=False) + # The only callsite of this pre-limits the content_type to a + # reasonable set that we know have extensions. + assert image_ext is not None + + # We salt this with a server-side secret so that it is not + # enumerable by clients, and will not collide on the server. New + # realm imports may pass a synthetic emoji_id, which is fine as + # long as it starts at 1, and as such later emoji cannot collide + # unless there is a legit hash collision. + # + # We truncate the hash at 8 characters, as this is enough entropy + # to make collisions vanishingly unlikely. In the event of a + # collusion, the id will advance and a manual retry will succeed. + hash_key = settings.AVATAR_SALT.encode() + b":" + str(emoji_id).encode() + return "".join((hashlib.sha256(hash_key).hexdigest()[0:8], image_ext)) + + +def thumbnail_local_emoji(apps: StateApps) -> None: + assert settings.LOCAL_AVATARS_DIR is not None + for total_processed, emoji in enumerate(thumbnail_iterator(apps)): + if total_processed % 100 == 0: + print(f"Processed {total_processed} custom emoji") + + try: + old_file_name = emoji.file_name + content_type = guess_type(old_file_name)[0] + if content_type not in VALID_EMOJI_CONTENT_TYPE: + raise SkipImageError(f"Invalid content-type: {content_type}") + + new_file_name = get_emoji_file_name(content_type, emoji.id) + if old_file_name == new_file_name: + continue + base_path = os.path.join( + settings.LOCAL_AVATARS_DIR, str(emoji.realm_id), "emoji/images" + ) + print(f"{base_path}/{old_file_name} -> {base_path}/{new_file_name}") + try: + if os.path.exists(f"{base_path}/{new_file_name}.original"): + os.unlink(f"{base_path}/{new_file_name}.original") + os.link( + f"{base_path}/{old_file_name}.original", f"{base_path}/{new_file_name}.original" + ) + with open(f"{base_path}/{new_file_name}.original", "rb") as fh: + original_bytes = fh.read() + except OSError as e: + raise SkipImageError(f"Failed to read original file: {e}") + + animated, still = resize_emoji(original_bytes, new_file_name) + try: + with open(f"{base_path}/{new_file_name}", "wb") as fh: + fh.write(animated) + + if still is not None: + filename_no_extension = os.path.splitext(new_file_name)[0] + with open(f"{base_path}/still/{filename_no_extension}.png", "wb") as fh: + fh.write(still) + except OSError as e: + raise SkipImageError(f"Failed to write new file: {e}") + + emoji.file_name = new_file_name + emoji.save(update_fields=["file_name"]) + except SkipImageError as e: + logging.warning( + "Failed to re-thumbnail %s/emoji/images/%s: %s", emoji.realm_id, emoji.file_name, e + ) + new_file_name = get_emoji_file_name("image/png", emoji.id) + try: + with open(f"{settings.DEPLOY_ROOT}/static/images/bad-emoji.png", "rb") as f: + with open(f"{base_path}/{new_file_name}", "wb") as new_f: + new_f.write(f.read()) + emoji.deactivated = True + emoji.is_animated = False + emoji.file_name = new_file_name + emoji.save(update_fields=["file_name", "is_animated", "deactivated"]) + except Exception as e: + logging.error("Failed to deactivate and replace with known-good image: %s", e) + + +def thumbnail_s3(apps: StateApps) -> None: + total_processed = 0 + avatar_bucket = boto3.resource( + "s3", + aws_access_key_id=settings.S3_KEY, + aws_secret_access_key=settings.S3_SECRET_KEY, + region_name=settings.S3_REGION, + endpoint_url=settings.S3_ENDPOINT_URL, + config=Config( + signature_version=None, + s3={"addressing_style": settings.S3_ADDRESSING_STYLE}, + ), + ).Bucket(settings.S3_AVATAR_BUCKET) + for total_processed, emoji in enumerate(thumbnail_iterator(apps)): + if total_processed % 100 == 0: + print(f"Processed {total_processed} custom emoji") + + old_file_name = emoji.file_name + try: + base_path = os.path.join(str(emoji.realm_id), "emoji/images") + try: + old_data = avatar_bucket.Object(f"{base_path}/{old_file_name}.original").get() + original_bytes = old_data["Body"].read() + except botocore.exceptions.ClientError as e: + raise SkipImageError(f"Failed to read original file: {e}") + + content_type = old_data["ContentType"] + if content_type not in VALID_EMOJI_CONTENT_TYPE: + raise SkipImageError(f"Invalid content-type: {content_type}") + + metadata = old_data["Metadata"] + # Make sure this metadata is up-to-date, while we're + # in here; some early emoji are missing it + metadata["realm_id"] = str(emoji.realm_id) + if emoji.author_id: + metadata["user_profile_id"] = str(emoji.author_id) + + new_file_name = get_emoji_file_name(content_type, emoji.id) + if old_file_name == new_file_name: + continue + + print(f"{base_path}/{old_file_name} -> {base_path}/{new_file_name}") + avatar_bucket.Object(f"{base_path}/{new_file_name}.original").copy_from( + CopySource=f"{settings.S3_AVATAR_BUCKET}/{base_path}/{old_file_name}.original", + MetadataDirective="REPLACE", + Metadata=metadata, + ContentType=content_type, + CacheControl="public, max-age=31536000, immutable", + ) + + animated, still = resize_emoji(original_bytes, new_file_name) + try: + avatar_bucket.Object(f"{base_path}/{new_file_name}").put( + Metadata=metadata, + ContentType=content_type, + CacheControl="public, max-age=31536000, immutable", + Body=animated, + ) + if still is not None: + filename_no_extension = os.path.splitext(new_file_name)[0] + avatar_bucket.Object(f"{base_path}/still/{filename_no_extension}.png").put( + Metadata=metadata, + ContentType="image/png", + CacheControl="public, max-age=31536000, immutable", + Body=still, + ) + except botocore.exceptions.ClientError as e: + raise SkipImageError(f"Failed to upload new file: {e}") + + emoji.file_name = new_file_name + emoji.save(update_fields=["file_name"]) + except SkipImageError as e: + logging.warning( + "Failed to re-thumbnail %s/emoji/images/%s: %s", emoji.realm_id, emoji.file_name, e + ) + new_file_name = get_emoji_file_name("image/png", emoji.id) + try: + with open(f"{settings.DEPLOY_ROOT}/static/images/bad-emoji.png", "rb") as f: + avatar_bucket.Object(f"{base_path}/{new_file_name}").put( + Metadata={ + "user_profile_id": str(emoji.author_id), + "realm_id": str(emoji.realm_id), + }, + ContentType="image/png", + CacheControl="public, max-age=31536000, immutable", + Body=f.read(), + ) + emoji.deactivated = True + emoji.is_animated = False + emoji.file_name = new_file_name + emoji.save(update_fields=["file_name", "is_animated", "deactivated"]) + except Exception as e: + logging.error("Failed to deactivate and replace with known-good image: %s", e) + + +def thumbnail_iterator(apps: StateApps) -> Iterator[Any]: + Realm = apps.get_model("zerver", "Realm") + RealmEmoji = apps.get_model("zerver", "RealmEmoji") + for realm in Realm.objects.filter(realmemoji__isnull=False).distinct().order_by("id"): + yield from RealmEmoji.objects.filter(realm=realm).order_by("id") + + +def thumbnail_emoji(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + if settings.LOCAL_AVATARS_DIR is not None: + thumbnail_local_emoji(apps) + else: + thumbnail_s3(apps) + + +class Migration(migrations.Migration): + atomic = False + elidable = True + + dependencies = [ + ("zerver", "0552_remove_realm_private_message_policy"), + ] + + operations = [migrations.RunPython(thumbnail_emoji, elidable=True)]