From c13e3dee247c8fc78dbdc8236b89e2554af00d91 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 4 Mar 2024 20:00:20 +0000 Subject: [PATCH] katex: Replace subprocess call with minimal external service. Replace a separate call to subprocess, starting `node` from scratch, with an optional standalone node Express service which performs the rendering. In benchmarking, this reduces the overhead of a KaTeX call from 120ms to 2.8ms. This is notable because enough calls to KaTeX in a single message would previously time out the whole message rendering. The service is optional because he majority of deployments do not use enough LaTeX to merit the additional memory usage (60Mb). Fixes: #17425. --- .eslintrc.json | 6 + docs/production/system-configuration.md | 12 + package.json | 6 + pnpm-lock.yaml | 220 +++++++++++++++++- puppet/zulip/manifests/app_frontend_base.pp | 3 + .../supervisor/zulip.conf.template.erb | 17 ++ scripts/restart-server | 7 +- tools/ci/production-build | 2 +- tools/webpack | 2 +- version.py | 2 +- web/server/katex_server.ts | 78 +++++++ web/webpack.config.ts | 7 + zerver/lib/tex.py | 44 +++- zerver/tests/test_markdown.py | 76 ++++++ zproject/computed_settings.py | 8 + 15 files changed, 484 insertions(+), 6 deletions(-) create mode 100644 web/server/katex_server.ts diff --git a/.eslintrc.json b/.eslintrc.json index 355b9a1164..1b4e66b0f2 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -277,6 +277,12 @@ ], "unicorn/prefer-string-replace-all": "off" } + }, + { + "files": ["web/server/**"], + "env": { + "node": true + } } ] } diff --git a/docs/production/system-configuration.md b/docs/production/system-configuration.md index 430a355fe6..0871b8d1e4 100644 --- a/docs/production/system-configuration.md +++ b/docs/production/system-configuration.md @@ -190,6 +190,18 @@ more than 3.5GiB of RAM, 4 on hosts with less. Number of days of access logs to keep, for both nginx and the application. Defaults to 14 days. +#### `katex_server` + +Set to a true value to run a separate service for [rendering math with +LaTeX](https://zulip.com/help/latex). This is not necessary except on servers +with users who send several math blocks in a single message; it will address +issues with such messages occasionally failing to send, at cost of a small +amount of increased memory usage. + +#### `katex_server_port` + +Set to the port number for the KaTeX server, if enabled; defaults to port 9700. + ### `[postfix]` #### `mailname` diff --git a/package.json b/package.json index 1a1e131295..41787189eb 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,12 @@ "@formatjs/intl": "^2.0.0", "@giphy/js-components": "^5.0.5", "@giphy/js-fetch-api": "^5.0.0", + "@koa/bodyparser": "^5.0.0", "@sentry/browser": "^7.51.2", "@sentry/integrations": "^7.51.2", + "@types/co-body": "^6.1.3", + "@types/koa": "^2.15.0", + "@types/koa-bodyparser": "^4.3.12", "@uppy/core": "^3.0.2", "@uppy/progress-bar": "^3.0.1", "@uppy/xhr-upload": "^3.0.2", @@ -26,6 +30,7 @@ "clean-css": "^5.1.0", "clipboard": "^2.0.4", "colord": "^2.9.3", + "config-ini-parser": "^1.6.1", "core-js": "^3.36.0", "css-loader": "^6.2.0", "css-minimizer-webpack-plugin": "^6.0.0", @@ -50,6 +55,7 @@ "jquery-validation": "^1.19.0", "js-cookie": "^3.0.1", "katex": "^0.16.2", + "koa": "^2.15.0", "lodash": "^4.17.19", "micromodal": "^0.4.6", "mini-css-extract-plugin": "^2.2.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed177ab372..61ce854304 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,12 +37,24 @@ dependencies: '@giphy/js-fetch-api': specifier: ^5.0.0 version: 5.4.0 + '@koa/bodyparser': + specifier: ^5.0.0 + version: 5.0.0 '@sentry/browser': specifier: ^7.51.2 version: 7.106.1 '@sentry/integrations': specifier: ^7.51.2 version: 7.106.1 + '@types/co-body': + specifier: ^6.1.3 + version: 6.1.3 + '@types/koa': + specifier: ^2.15.0 + version: 2.15.0 + '@types/koa-bodyparser': + specifier: ^4.3.12 + version: 4.3.12 '@uppy/core': specifier: ^3.0.2 version: 3.9.3 @@ -85,6 +97,9 @@ dependencies: colord: specifier: ^2.9.3 version: 2.9.3 + config-ini-parser: + specifier: ^1.6.1 + version: 1.6.1 core-js: specifier: ^3.36.0 version: 3.36.0 @@ -157,6 +172,9 @@ dependencies: katex: specifier: ^0.16.2 version: 0.16.9 + koa: + specifier: ^2.15.0 + version: 2.15.0 lodash: specifier: ^4.17.19 version: 4.17.21 @@ -2605,6 +2623,15 @@ packages: resolution: {integrity: sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==} dev: true + /@koa/bodyparser@5.0.0: + resolution: {integrity: sha512-JEiZVe2e85qPOqA+Nw/SJC5fkFw3XSekh0RSoqz5F6lFYuhEspgqAb972rQRCJesv27QUsz96vU/Vb92wF1GUg==} + engines: {node: '>= 16'} + dependencies: + co-body: 6.1.0 + lodash.merge: 4.6.2 + type-is: 1.6.18 + dev: false + /@leichtgewicht/ip-codec@2.0.4: resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} @@ -2946,6 +2973,12 @@ packages: '@turf/helpers': 6.5.0 dev: false + /@types/accepts@1.3.7: + resolution: {integrity: sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==} + dependencies: + '@types/node': 20.11.27 + dev: false + /@types/autosize@4.0.3: resolution: {integrity: sha512-o0ZyU3ePp3+KRbhHsY4ogjc+ZQWgVN5h6j8BHW5RII4cFKi6PEKK9QPAcphJVkD0dGpyFnD3VRR0WMvHVjCv9w==} dev: true @@ -3007,6 +3040,13 @@ packages: source-map: /source-map-js@1.0.1 dev: true + /@types/co-body@6.1.3: + resolution: {integrity: sha512-UhuhrQ5hclX6UJctv5m4Rfp52AfG9o9+d9/HwjxhVB5NjXxr5t9oKgJxN8xRHgr35oo8meUEHUPFWiKg6y71aA==} + dependencies: + '@types/node': 20.11.27 + '@types/qs': 6.9.12 + dev: false + /@types/connect-history-api-fallback@1.5.4: resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} dependencies: @@ -3018,6 +3058,19 @@ packages: dependencies: '@types/node': 20.11.27 + /@types/content-disposition@0.5.8: + resolution: {integrity: sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==} + dev: false + + /@types/cookies@0.9.0: + resolution: {integrity: sha512-40Zk8qR147RABiQ7NQnBzWzDcjKzNrntB5BAmeGCb2p/MIyOE+4BVvc17wumsUqUw00bJYqoXFHYygQnEFh4/Q==} + dependencies: + '@types/connect': 3.4.38 + '@types/express': 4.17.21 + '@types/keygrip': 1.0.6 + '@types/node': 20.11.27 + dev: false + /@types/eslint-scope@3.7.7: resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} dependencies: @@ -3053,6 +3106,10 @@ packages: resolution: {integrity: sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==} dev: false + /@types/http-assert@1.5.5: + resolution: {integrity: sha512-4+tE/lwdAahgZT1g30Jkdm9PzFRde0xwxBNUyRsCitRvCQB90iuA2uJYdUnhnANRcqGXaWOGY4FEoxeElNAK2g==} + dev: false + /@types/http-errors@2.0.4: resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} @@ -3103,6 +3160,35 @@ packages: resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} dev: true + /@types/keygrip@1.0.6: + resolution: {integrity: sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==} + dev: false + + /@types/koa-bodyparser@4.3.12: + resolution: {integrity: sha512-hKMmRMVP889gPIdLZmmtou/BijaU1tHPyMNmcK7FAHAdATnRcGQQy78EqTTxLH1D4FTsrxIzklAQCso9oGoebQ==} + dependencies: + '@types/koa': 2.15.0 + dev: false + + /@types/koa-compose@3.2.8: + resolution: {integrity: sha512-4Olc63RY+MKvxMwVknCUDhRQX1pFQoBZ/lXcRLP69PQkEpze/0cr8LNqJQe5NFb/b19DWi2a5bTi2VAlQzhJuA==} + dependencies: + '@types/koa': 2.15.0 + dev: false + + /@types/koa@2.15.0: + resolution: {integrity: sha512-7QFsywoE5URbuVnG3loe03QXuGajrnotr3gQkXcEBShORai23MePfFYdhz90FEtBBpkyIYQbVD+evKtloCgX3g==} + dependencies: + '@types/accepts': 1.3.7 + '@types/content-disposition': 0.5.8 + '@types/cookies': 0.9.0 + '@types/http-assert': 1.5.5 + '@types/http-errors': 2.0.4 + '@types/keygrip': 1.0.6 + '@types/koa-compose': 3.2.8 + '@types/node': 20.11.27 + dev: false + /@types/lodash-es@4.17.12: resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} dependencies: @@ -4435,6 +4521,14 @@ packages: - bluebird dev: false + /cache-content-type@1.0.1: + resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} + engines: {node: '>= 6.0.0'} + dependencies: + mime-types: 2.1.35 + ylru: 1.3.2 + dev: false + /caching-transform@4.0.0: resolution: {integrity: sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==} engines: {node: '>=8'} @@ -4663,6 +4757,20 @@ packages: readable-stream: 2.3.8 dev: false + /co-body@6.1.0: + resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==} + dependencies: + inflation: 2.1.0 + qs: 6.11.0 + raw-body: 2.5.2 + type-is: 1.6.18 + dev: false + + /co@4.6.0: + resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + dev: false + /code-point@1.1.0: resolution: {integrity: sha512-L9JOfOolA/Y/YKVO+WUscMXuYUcHmkleTJkvl1G0XaGFj5JDXl02BobH8avoI/pjWvgOLivs2bizA0N6g8gM9Q==} dev: false @@ -4860,6 +4968,10 @@ packages: typedarray: 0.0.6 dev: false + /config-ini-parser@1.6.1: + resolution: {integrity: sha512-6YBVmXlWiUKDhjn+S/1ju2+8j8Qnj1PqR8yE5DmLrQRtBOc9kgMLyutFl2so1FNVHevR/A4nY7z9EWw8s6g/Cw==} + dev: false + /connect-history-api-fallback@2.0.0: resolution: {integrity: sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==} engines: {node: '>=0.8'} @@ -4891,6 +5003,14 @@ packages: resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} engines: {node: '>= 0.6'} + /cookies@0.9.1: + resolution: {integrity: sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==} + engines: {node: '>= 0.8'} + dependencies: + depd: 2.0.0 + keygrip: 1.1.0 + dev: false + /core-js-compat@3.36.0: resolution: {integrity: sha512-iV9Pd/PsgjNWBXeq8XRtWVSgz2tKAfhfvBs7qxYty+RlRd+OCksaWmOnc4JKrTc1cToXL1N0s3l/vwlxPtdElw==} dependencies: @@ -5415,6 +5535,10 @@ packages: resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} dev: true + /deep-equal@1.0.1: + resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} + dev: false + /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: true @@ -7218,7 +7342,6 @@ packages: engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.3 - dev: true /has-unicode@2.0.1: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} @@ -7332,6 +7455,14 @@ packages: entities: 2.2.0 dev: false + /http-assert@1.5.0: + resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} + engines: {node: '>= 0.8'} + dependencies: + deep-equal: 1.0.1 + http-errors: 1.8.1 + dev: false + /http-cache-semantics@4.1.1: resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} dev: false @@ -7348,6 +7479,17 @@ packages: setprototypeof: 1.1.0 statuses: 1.5.0 + /http-errors@1.8.1: + resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} + engines: {node: '>= 0.6'} + dependencies: + depd: 1.1.2 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 1.5.0 + toidentifier: 1.0.1 + dev: false + /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} @@ -7505,6 +7647,11 @@ packages: resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} dev: false + /inflation@2.1.0: + resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} + engines: {node: '>= 0.8.0'} + dev: false + /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: @@ -7665,6 +7812,13 @@ packages: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + /is-generator-function@1.0.10: + resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.2 + dev: false + /is-glob@3.1.0: resolution: {integrity: sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==} engines: {node: '>=0.10.0'} @@ -8193,6 +8347,13 @@ packages: resolution: {integrity: sha512-hRkd6/XW4HTsA9vjVpY9tuXJYLSlelnkTmVFu4M9/7MIYQtFcHpbugAU7UbOfjOiVSVYl2fqgBuJ32JUmRo5Ew==} dev: false + /keygrip@1.1.0: + resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} + engines: {node: '>= 0.6'} + dependencies: + tsscmp: 1.0.6 + dev: false + /keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} dependencies: @@ -8211,6 +8372,49 @@ packages: resolution: {integrity: sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==} dev: true + /koa-compose@4.1.0: + resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} + dev: false + + /koa-convert@2.0.0: + resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} + engines: {node: '>= 10'} + dependencies: + co: 4.6.0 + koa-compose: 4.1.0 + dev: false + + /koa@2.15.0: + resolution: {integrity: sha512-KEL/vU1knsoUvfP4MC4/GthpQrY/p6dzwaaGI6Rt4NQuFqkw3qrvsdYF5pz3wOfi7IGTvMPHC9aZIcUKYFNxsw==} + engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} + dependencies: + accepts: 1.3.8 + cache-content-type: 1.0.1 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookies: 0.9.1 + debug: 4.3.4 + delegates: 1.0.0 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + fresh: 0.5.2 + http-assert: 1.5.0 + http-errors: 1.8.1 + is-generator-function: 1.0.10 + koa-compose: 4.1.0 + koa-convert: 2.0.0 + on-finished: 2.4.1 + only: 0.0.2 + parseurl: 1.3.3 + statuses: 1.5.0 + type-is: 1.6.18 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + /kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} dev: true @@ -9138,6 +9342,10 @@ packages: dependencies: mimic-fn: 2.1.0 + /only@0.0.2: + resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} + dev: false + /open@10.1.0: resolution: {integrity: sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==} engines: {node: '>=18'} @@ -12033,6 +12241,11 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + /tsscmp@1.0.6: + resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} + engines: {node: '>=0.6.x'} + dev: false + /ttf2eot@3.1.0: resolution: {integrity: sha512-aHTbcYosNHVqb2Qtt9Xfta77ae/5y0VfdwNLUS6sGBeGr22cX2JDMo/i5h3uuOf+FAD3akYOr17+fYd5NK8aXw==} hasBin: true @@ -12972,6 +13185,11 @@ packages: fd-slicer: 1.1.0 dev: true + /ylru@1.3.2: + resolution: {integrity: sha512-RXRJzMiK6U2ye0BlGGZnmpwJDPgakn6aNQ0A7gHRbD4I0uvK4TW6UqkK1V0pp9jskjJBAXd3dRrbzWkqJ+6cxA==} + engines: {node: '>= 4.0.0'} + dev: false + /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} diff --git a/puppet/zulip/manifests/app_frontend_base.pp b/puppet/zulip/manifests/app_frontend_base.pp index 5d9c846495..20b8e84fe8 100644 --- a/puppet/zulip/manifests/app_frontend_base.pp +++ b/puppet/zulip/manifests/app_frontend_base.pp @@ -177,6 +177,9 @@ class zulip::app_frontend_base { include zulip::smokescreen } + $katex_server = zulipconf('application_server', 'katex_server', false) + $katex_server_port = zulipconf('application_server', 'katex_server_port', '9700') + if $proxy_host != '' and $proxy_port != '' { $proxy = "http://${proxy_host}:${proxy_port}" } else { diff --git a/puppet/zulip/templates/supervisor/zulip.conf.template.erb b/puppet/zulip/templates/supervisor/zulip.conf.template.erb index 1b5a00103b..d98472b647 100644 --- a/puppet/zulip/templates/supervisor/zulip.conf.template.erb +++ b/puppet/zulip/templates/supervisor/zulip.conf.template.erb @@ -106,6 +106,23 @@ programs=<% @queues.each_with_index do |queue, i| -%>zulip_events_<%= queue %><% programs=zulip_events <% end %> +<% if @katex_server %> +[program:zulip-katex] +command=/usr/local/bin/secret-env-wrapper SHARED_SECRET=shared_secret /srv/zulip-node/bin/node /home/zulip/deployments/current/static/webpack-bundles/katex_server.js <%= @katex_server_port %> +environment=NODE_ENV=production +priority=200 ; the relative start priority (default 999) +autostart=true ; start at supervisord start (default: true) +autorestart=true ; whether/when to restart (default: unexpected) +stopsignal=TERM ; signal used to kill process (default TERM) +stopwaitsecs=30 ; max num secs to wait b4 SIGKILL (default 10) +user=zulip ; setuid to this UNIX account to run the program +redirect_stderr=true ; redirect proc stderr to stdout (default false) +stdout_logfile=/var/log/zulip/katex.log ; stdout log path, NONE for none; default AUTO +stdout_logfile_maxbytes=20MB ; max # logfile bytes b4 rotation (default 50MB) +stdout_logfile_backups=3 ; # of stdout logfile backups (default 10) +directory=/home/zulip/deployments/current/ +<% end %> + ; The [include] section can just contain the "files" setting. This ; setting can list multiple files (separated by whitespace or ; newlines). It can also contain wildcards. The filenames are diff --git a/scripts/restart-server b/scripts/restart-server index 211286cb5b..46f9eb1854 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -101,12 +101,17 @@ if has_application_server(): ) ) + # This is an optional service, so may or may not exist + workers.extend(list_supervisor_processes(["zulip-katex"])) + if has_process_fts_updates(): workers.append("process-fts-updates") # Before we start (re)starting main services, make sure to start any # optional auxiliary services that we don't stop, but do expect to be -# running, and aren't currently. +# running, and aren't currently. These get new versions by getting +# updated supervisor files, and puppet restarts them -- so we never +# restart them in here, only start them. aux_services = list_supervisor_processes(["go-camo", "smokescreen"], only_running=False) if aux_services: subprocess.check_call(["supervisorctl", "start", *aux_services]) diff --git a/tools/ci/production-build b/tools/ci/production-build index 9d1b8040e8..9097096b4d 100755 --- a/tools/ci/production-build +++ b/tools/ci/production-build @@ -48,6 +48,6 @@ cp -a \ PNPM="/usr/local/bin/pnpm" tar -C /tmp -xzf /tmp/production-build/zulip-server-test.tar.gz zulip-server-test/prod-static/serve/webpack-bundles ( - GLOBIGNORE=/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex-cli.js + GLOBIGNORE="/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex*.js" "$PNPM" exec es-check es2020 /tmp/zulip-server-test/prod-static/serve/webpack-bundles/*.js ) diff --git a/tools/webpack b/tools/webpack index 009190f60c..70098fa556 100755 --- a/tools/webpack +++ b/tools/webpack @@ -46,7 +46,7 @@ def build_for_dev_server(host: str, port: str, minify: bool, disable_host_check: webpack_args = ["../node_modules/.bin/webpack-cli", "serve"] webpack_args += [ # webpack-cli has a bug where it ignores --watch-poll with - # multi-config, and we don't need the katex-cli part anyway. + # multi-config, and we don't need the katex part anyway. "--config-name=frontend", f"--host={host}", f"--port={port}", diff --git a/version.py b/version.py index 926c1b016e..3a6515e5ce 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 243 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (264, 3) +PROVISION_VERSION = (264, 4) diff --git a/web/server/katex_server.ts b/web/server/katex_server.ts new file mode 100644 index 0000000000..950465e4da --- /dev/null +++ b/web/server/katex_server.ts @@ -0,0 +1,78 @@ +import crypto from "node:crypto"; + +import bodyParser from "@koa/bodyparser"; +import katex from "katex"; +import Koa from "koa"; + +const host = "localhost"; +const port = Number(process.argv[2] ?? "9700"); +if (!Number.isInteger(port)) { + throw new TypeError("Invalid port"); +} + +const shared_secret = process.env.SHARED_SECRET; +if (typeof shared_secret !== "string") { + console.error("No SHARED_SECRET set!"); + process.exit(1); +} +const compare_secret = (given_secret: string): boolean => { + try { + // Throws an exception if the strings are unequal length + return crypto.timingSafeEqual( + Buffer.from(shared_secret, "utf8"), + Buffer.from(given_secret, "utf8"), + ); + } catch { + return false; + } +}; + +const app = new Koa(); +app.use(bodyParser()); + +app.use((ctx, _next) => { + if (ctx.request.method !== "POST" || ctx.request.path !== "/") { + ctx.status = 404; + return; + } + if (ctx.request.body === undefined) { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = "Missing POST body"; + return; + } + const given_secret = ctx.request.body.shared_secret; + if (typeof given_secret !== "string" || !compare_secret(given_secret)) { + ctx.status = 403; + ctx.type = "text/plain"; + ctx.body = "Invalid 'shared_secret' argument"; + return; + } + + const content = ctx.request.body.content; + const is_display = ctx.request.body.is_display === "true"; + + if (typeof content !== "string") { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = "Invalid 'content' argument"; + return; + } + + try { + ctx.body = katex.renderToString(content, {displayMode: is_display}); + } catch (error) { + if (error instanceof katex.ParseError) { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = error.message; + } else { + ctx.status = 500; + console.error(error); + } + } +}); + +app.listen(port, host, () => { + console.log(`Server started on http://${host}:${port}`); +}); diff --git a/web/webpack.config.ts b/web/webpack.config.ts index 702c907b31..e4c49aa7d1 100644 --- a/web/webpack.config.ts +++ b/web/webpack.config.ts @@ -255,11 +255,18 @@ export default ( name: "server", target: "node", entry: { + katex_server: "babel-loader!./server/katex_server.ts", "katex-cli": "shebang-loader!katex/cli", }, output: { path: path.resolve(__dirname, "../static/webpack-bundles"), }, + resolve: { + alias: { + // koa-body uses formidable 2.x, which suffers from https://github.com/node-formidable/formidable/issues/337 + hexoid: "hexoid/dist/index.js", + }, + }, }; return [frontendConfig, serverConfig]; diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 23de3a725f..6e2b399a91 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -1,14 +1,24 @@ import logging import os import subprocess -from typing import Optional +from typing import Any, Optional import lxml.html +import requests from django.conf import settings +from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.storage import static_path +class KatexSession(OutgoingSession): + def __init__(self, **kwargs: Any) -> None: + # We set a very short timeout because these requests are + # expected to be quite fast (milliseconds) and blocking on + # this affects message rendering performance. + super().__init__(role="katex", timeout=0.5, **kwargs) + + def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX @@ -23,6 +33,38 @@ def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: (default True) """ + if settings.KATEX_SERVER: + try: + resp = KatexSession().post( + # We explicitly disable the Smokescreen proxy for this + # call, since it intentionally connects to localhost. + # This is safe because the host is explicitly fixed, and + # the port is pulled from our own configuration. + f"http://localhost:{settings.KATEX_SERVER_PORT}/", + data={ + "content": tex, + "is_display": "false" if is_inline else "true", + "shared_secret": settings.SHARED_SECRET, + }, + proxies={"http": ""}, + ) + except requests.exceptions.Timeout: + logging.warning("KaTeX rendering service timed out with %d byte long input", len(tex)) + return None + except requests.exceptions.RequestException as e: + logging.warning("KaTeX rendering service failed: %s", type(e).__name__) + return None + + if resp.status_code == 200: + return resp.content.decode().strip() + elif resp.status_code == 400: + return None + else: + logging.warning( + "KaTeX rendering service failed: (%s) %s", resp.status_code, resp.content.decode() + ) + return None + katex_path = ( static_path("webpack-bundles/katex-cli.js") if settings.PRODUCTION diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index f1c899614b..9610ee7788 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -7,10 +7,13 @@ from typing import Any, Dict, List, Optional, Set, Tuple from unittest import mock import orjson +import requests +import responses from bs4 import BeautifulSoup from django.conf import settings from django.test import override_settings from markdown import Markdown +from responses import matchers from typing_extensions import override from zerver.actions.alert_words import do_add_alert_words @@ -297,6 +300,79 @@ class MarkdownMiscTest(ZulipTestCase): render_tex("random text") self.assertEqual(m.output, ["ERROR:root:Cannot find KaTeX for latex rendering!"]) + @responses.activate + @override_settings(KATEX_SERVER=True, SHARED_SECRET="foo") + def test_katex_server(self) -> None: + responses.post( + "http://localhost:9700/", + match=[ + matchers.urlencoded_params_matcher( + {"content": "foo", "is_display": "false", "shared_secret": "foo"} + ) + ], + content_type="text/html; charset=utf-8", + body="html", + ) + self.assertEqual(render_tex("foo"), "html") + + responses.post( + "http://localhost:9700/?", + match=[ + matchers.urlencoded_params_matcher( + {"content": "foo", "is_display": "true", "shared_secret": "foo"} + ) + ], + content_type="text/html; charset=utf-8", + body="other", + ) + self.assertEqual(render_tex("foo", is_inline=False), "other") + + responses.post( + "http://localhost:9700/", + content_type="text/html; charset=utf-8", + status=400, + body=r"KaTeX parse error: '\'", + ) + self.assertEqual(render_tex("bad"), None) + + responses.post( + "http://localhost:9700/", + content_type="text/html; charset=utf-8", + status=400, + body=r"KaTeX parse error: '\'", + ) + self.assertEqual(render_tex("bad"), None) + + responses.post("http://localhost:9700/", status=403, body="") + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: (403) "]) + + responses.post("http://localhost:9700/", status=500, body="") + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: (500) "]) + + responses.post("http://localhost:9700/", body=requests.exceptions.Timeout()) + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual( + m.output, ["WARNING:root:KaTeX rendering service timed out with 3 byte long input"] + ) + + responses.post("http://localhost:9700/", body=requests.exceptions.ConnectionError()) + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: ConnectionError"]) + + with override_settings(KATEX_SERVER_PORT=9701): + responses.post( + "http://localhost:9701/", + body="html", + content_type="text/html; charset=utf-8", + ) + self.assertEqual(render_tex("foo"), "html") + class MarkdownListPreprocessorTest(ZulipTestCase): # We test that the preprocessor inserts blank lines at correct places. diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 91035f42c9..94b6636172 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -520,6 +520,14 @@ INTERNAL_BOT_DOMAIN = "zulip.com" # This needs to be synced with the Camo installation CAMO_KEY = get_secret("camo_key") if CAMO_URI != "" else None +######################################################################## +# KATEX SERVER SETTINGS +######################################################################## + +KATEX_SERVER = get_config("application_server", "katex_server", False) +KATEX_SERVER_PORT = get_config("application_server", "katex_server_port", "9700") + + ######################################################################## # STATIC CONTENT AND MINIFICATION SETTINGS ########################################################################