emoji: Convert spritesheets to webp.

This provides significant size savings:

| Emoji set   | png size | webp size | webp/png percent |
| ----------- | -------- | --------- | ---------------- |
| google-blob |  1968954 |   1373350 |           69.75% |
| twitter     |  2972820 |   2149672 |           72.31% |
| google      |  3455270 |   2327834 |           67.37% |

Since these are the largest assets that we ship to clients, it is
worth shaving off every byte we can.

scripts/lib/setup_venv.py

@@ -28,6 +28,7 @@ VENV_DEPENDENCIES = [
     "jq",  # No longer used in production (clean me up later)
     "libsasl2-dev",  # For building python-ldap from source
     "libvips",  # For thumbnailing
+    "libvips-tools",
 ]
 
 COMMON_YUM_VENV_DEPENDENCIES = [
@@ -46,6 +47,7 @@ COMMON_YUM_VENV_DEPENDENCIES = [
     "openssl-devel",
     "jq",
     "vips",  # For thumbnailing
+    "vips-tools",
 ]
 
 REDHAT_VENV_DEPENDENCIES = [

tools/setup/emoji/build_emoji

@@ -4,6 +4,7 @@
 # works.
 import os
 import shutil
+import subprocess
 import sys
 from collections.abc import Iterator, Sequence
 from typing import Any
@@ -49,7 +50,7 @@ div.emoji,
 span.emoji
 {{
     display: inline-block;
-    background-image: url(~emoji-datasource-{emojiset}/img/{alt_name}/sheets-256/64.png);
+    background-image: url(../emoji/{emojiset}.webp);
     background-size: {background_size};
     background-repeat: no-repeat;
 
@@ -116,15 +117,16 @@ def main() -> None:
 
     # /srv/zulip-emoji-cache/*/web gets copied to ZULIP_PATH/web/generated
     # These must not be symlinked so webpack can resolve module references.
-    TARGET_EMOJI_STYLES = os.path.join(ZULIP_PATH, "web", "generated", "emoji-styles")
+    for subdir in ("emoji", "emoji-styles"):
-    os.makedirs(TARGET_EMOJI_STYLES, exist_ok=True)
+        target_dir = os.path.join(ZULIP_PATH, "web", "generated", subdir)
-    to_remove = set(os.listdir(TARGET_EMOJI_STYLES))
+        os.makedirs(target_dir, exist_ok=True)
-    source_emoji_dump = os.path.join(emoji_cache_path, "web", "emoji-styles")
+        to_remove = set(os.listdir(target_dir))
+        source_emoji_dump = os.path.join(emoji_cache_path, "web", subdir)
         for filename in os.listdir(source_emoji_dump):
-        shutil.copy2(os.path.join(source_emoji_dump, filename), TARGET_EMOJI_STYLES)
+            shutil.copy2(os.path.join(source_emoji_dump, filename), target_dir)
             to_remove.discard(filename)
         for filename in to_remove:
-        os.remove(os.path.join(TARGET_EMOJI_STYLES, filename))
+            os.remove(os.path.join(target_dir, filename))
 
 
 def percent(f: float) -> str:
@@ -334,6 +336,26 @@ def setup_emoji_farms(cache_path: str, emoji_data: list[dict[str, Any]]) -> None
 
         generate_sprite_css_files(cache_path, emoji_data, emojiset, alt_name, fallback_emoji_data)
 
+        print(f"Converting {emojiset} sheet to webp...")
+        TARGET_EMOJI_SHEETS = os.path.join(cache_path, "web", "emoji")
+        os.makedirs(TARGET_EMOJI_SHEETS, exist_ok=True)
+
+        sheet_src = os.path.join(
+            NODE_MODULES_PATH,
+            f"emoji-datasource-{emojiset}",
+            "img",
+            alt_name,
+            "sheets-256",
+            "64.png",
+        )
+        sheet_dst = os.path.join(TARGET_EMOJI_SHEETS, f"{emojiset}.webp")
+        # From libwebp: [Q is] between 0 and 100. For lossy, 0 gives
+        #   the smallest size and 100 the largest. For lossless, this
+        #   parameter is the amount of effort put into the
+        #   compression: 0 is the fastest but gives larger files
+        #   compared to the slowest, but best, 100.
+        subprocess.check_call(["vips", "copy", sheet_src, f"{sheet_dst}[lossless=true,Q=100]"])
+
     # Set up standard emoji sets.
     for emojiset in ["google", "twitter"]:
         setup_emoji_farm(emojiset, emoji_data)

web/.gitignore

@@ -1,4 +1,5 @@
 # From emoji
+/generated/emoji
 /generated/emoji-styles
 # From passing pygments data to the frontend
 /generated/pygments_data.json

web/src/assets.d.ts

@@ -13,6 +13,11 @@ declare module "*.png" {
     export default url;
 }
 
+declare module "*.webp" {
+    const url: string;
+    export default url;
+}
+
 // Declare the style loader for CSS files.  This is used in the
 // `import` statements in the `emojisets.ts` file.
 declare module "!style-loader?*" {

web/src/emojisets.ts

@@ -1,8 +1,7 @@
-import google_sheet from "emoji-datasource-google/img/google/sheets-256/64.png";
-import google_blob_sheet from "emoji-datasource-google-blob/img/google/sheets-256/64.png";
-import twitter_sheet from "emoji-datasource-twitter/img/twitter/sheets-256/64.png";
-
 import octopus_url from "../../static/generated/emoji/images-google-64/1f419.png";
+import google_blob_sheet from "../generated/emoji/google-blob.webp";
+import google_sheet from "../generated/emoji/google.webp";
+import twitter_sheet from "../generated/emoji/twitter.webp";
 
 import * as blueslip from "./blueslip";
 import {user_settings} from "./user_settings";

web/tests/copy_and_paste.test.js

@@ -147,7 +147,7 @@ run_test("paste_handler_converter", () => {
 
     // Emojis
     input =
-        '<meta http-equiv="content-type" content="text/html; charset=utf-8"><span style="color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">emojis:<span> </span></span><span aria-label="smile" class="emoji emoji-1f642" role="img" title="smile" style="height: 20px; width: 20px; position: relative; margin-top: -7px; vertical-align: middle; top: 3px; background-position: 55% 46.667%; display: inline-block; background-image: url(&quot;http://localhost:9991/webpack/files/srv/zulip-npm-cache/287cb53c1a095fe79651f095d5d8d60f7060baa7/node_modules/emoji-datasource-google/img/google/sheets-256/64.png&quot;); background-size: 6100%; background-repeat: no-repeat; text-indent: 100%; white-space: nowrap; overflow: hidden; color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;">:smile:</span><span style="color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><span> </span></span><span aria-label="family man woman girl" class="emoji emoji-1f468-200d-1f469-200d-1f467" role="img" title="family man woman girl" style="height: 20px; width: 20px; position: relative; margin-top: -7px; vertical-align: middle; top: 3px; background-position: 23.333% 75%; display: inline-block; background-image: url(&quot;http://localhost:9991/webpack/files/srv/zulip-npm-cache/287cb53c1a095fe79651f095d5d8d60f7060baa7/node_modules/emoji-datasource-google/img/google/sheets-256/64.png&quot;); background-size: 6100%; background-repeat: no-repeat; text-indent: 100%; white-space: nowrap; overflow: hidden; color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;">:family_man_woman_girl:</span>';
+        '<meta http-equiv="content-type" content="text/html; charset=utf-8"><span style="color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">emojis:<span> </span></span><span aria-label="smile" class="emoji emoji-1f642" role="img" title="smile" style="height: 20px; width: 20px; position: relative; margin-top: -7px; vertical-align: middle; top: 3px; background-position: 55% 46.667%; display: inline-block; background-image: url(&quot;http://localhost:9991/webpack/files/generated/emoji/google.webp&quot;); background-size: 6100%; background-repeat: no-repeat; text-indent: 100%; white-space: nowrap; overflow: hidden; color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;">:smile:</span><span style="color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><span> </span></span><span aria-label="family man woman girl" class="emoji emoji-1f468-200d-1f469-200d-1f467" role="img" title="family man woman girl" style="height: 20px; width: 20px; position: relative; margin-top: -7px; vertical-align: middle; top: 3px; background-position: 23.333% 75%; display: inline-block; background-image: url(&quot;http://localhost:9991/webpack/files/generated/emoji/google.webp&quot;); background-size: 6100%; background-repeat: no-repeat; text-indent: 100%; white-space: nowrap; overflow: hidden; color: rgb(221, 222, 238); font-family: &quot;Source Sans 3&quot;, sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(33, 45, 59); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;">:family_man_woman_girl:</span>';
     assert.equal(
         copy_and_paste.paste_handler_converter(input),
         "emojis: :smile: :family_man_woman_girl:",

web/webpack.config.ts

@@ -192,7 +192,7 @@ const config = (
                 },
                 // load fonts and files
                 {
-                    test: /\.(eot|jpg|svg|ttf|otf|png|woff2?)$/,
+                    test: /\.(eot|jpg|svg|ttf|otf|png|webp|woff2?)$/,
                     type: "asset/resource",
                 },
             ],