diff --git a/.eslintrc.json b/.eslintrc.json index 355b9a1164..1b4e66b0f2 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -277,6 +277,12 @@ ], "unicorn/prefer-string-replace-all": "off" } + }, + { + "files": ["web/server/**"], + "env": { + "node": true + } } ] } diff --git a/docs/production/system-configuration.md b/docs/production/system-configuration.md index 430a355fe6..0871b8d1e4 100644 --- a/docs/production/system-configuration.md +++ b/docs/production/system-configuration.md @@ -190,6 +190,18 @@ more than 3.5GiB of RAM, 4 on hosts with less. Number of days of access logs to keep, for both nginx and the application. Defaults to 14 days. +#### `katex_server` + +Set to a true value to run a separate service for [rendering math with +LaTeX](https://zulip.com/help/latex). This is not necessary except on servers +with users who send several math blocks in a single message; it will address +issues with such messages occasionally failing to send, at cost of a small +amount of increased memory usage. + +#### `katex_server_port` + +Set to the port number for the KaTeX server, if enabled; defaults to port 9700. + ### `[postfix]` #### `mailname` diff --git a/package.json b/package.json index 1a1e131295..41787189eb 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,12 @@ "@formatjs/intl": "^2.0.0", "@giphy/js-components": "^5.0.5", "@giphy/js-fetch-api": "^5.0.0", + "@koa/bodyparser": "^5.0.0", "@sentry/browser": "^7.51.2", "@sentry/integrations": "^7.51.2", + "@types/co-body": "^6.1.3", + "@types/koa": "^2.15.0", + "@types/koa-bodyparser": "^4.3.12", "@uppy/core": "^3.0.2", "@uppy/progress-bar": "^3.0.1", "@uppy/xhr-upload": "^3.0.2", @@ -26,6 +30,7 @@ "clean-css": "^5.1.0", "clipboard": "^2.0.4", "colord": "^2.9.3", + "config-ini-parser": "^1.6.1", "core-js": "^3.36.0", "css-loader": "^6.2.0", "css-minimizer-webpack-plugin": "^6.0.0", @@ -50,6 +55,7 @@ "jquery-validation": "^1.19.0", "js-cookie": "^3.0.1", "katex": "^0.16.2", + "koa": "^2.15.0", "lodash": "^4.17.19", "micromodal": "^0.4.6", "mini-css-extract-plugin": "^2.2.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed177ab372..61ce854304 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,12 +37,24 @@ dependencies: '@giphy/js-fetch-api': specifier: ^5.0.0 version: 5.4.0 + '@koa/bodyparser': + specifier: ^5.0.0 + version: 5.0.0 '@sentry/browser': specifier: ^7.51.2 version: 7.106.1 '@sentry/integrations': specifier: ^7.51.2 version: 7.106.1 + '@types/co-body': + specifier: ^6.1.3 + version: 6.1.3 + '@types/koa': + specifier: ^2.15.0 + version: 2.15.0 + '@types/koa-bodyparser': + specifier: ^4.3.12 + version: 4.3.12 '@uppy/core': specifier: ^3.0.2 version: 3.9.3 @@ -85,6 +97,9 @@ dependencies: colord: specifier: ^2.9.3 version: 2.9.3 + config-ini-parser: + specifier: ^1.6.1 + version: 1.6.1 core-js: specifier: ^3.36.0 version: 3.36.0 @@ -157,6 +172,9 @@ dependencies: katex: specifier: ^0.16.2 version: 0.16.9 + koa: + specifier: ^2.15.0 + version: 2.15.0 lodash: specifier: ^4.17.19 version: 4.17.21 @@ -2605,6 +2623,15 @@ packages: resolution: {integrity: sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==} dev: true + /@koa/bodyparser@5.0.0: + resolution: {integrity: sha512-JEiZVe2e85qPOqA+Nw/SJC5fkFw3XSekh0RSoqz5F6lFYuhEspgqAb972rQRCJesv27QUsz96vU/Vb92wF1GUg==} + engines: {node: '>= 16'} + dependencies: + co-body: 6.1.0 + lodash.merge: 4.6.2 + type-is: 1.6.18 + dev: false + /@leichtgewicht/ip-codec@2.0.4: resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} @@ -2946,6 +2973,12 @@ packages: '@turf/helpers': 6.5.0 dev: false + /@types/accepts@1.3.7: + resolution: {integrity: sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==} + dependencies: + '@types/node': 20.11.27 + dev: false + /@types/autosize@4.0.3: resolution: {integrity: sha512-o0ZyU3ePp3+KRbhHsY4ogjc+ZQWgVN5h6j8BHW5RII4cFKi6PEKK9QPAcphJVkD0dGpyFnD3VRR0WMvHVjCv9w==} dev: true @@ -3007,6 +3040,13 @@ packages: source-map: /source-map-js@1.0.1 dev: true + /@types/co-body@6.1.3: + resolution: {integrity: sha512-UhuhrQ5hclX6UJctv5m4Rfp52AfG9o9+d9/HwjxhVB5NjXxr5t9oKgJxN8xRHgr35oo8meUEHUPFWiKg6y71aA==} + dependencies: + '@types/node': 20.11.27 + '@types/qs': 6.9.12 + dev: false + /@types/connect-history-api-fallback@1.5.4: resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} dependencies: @@ -3018,6 +3058,19 @@ packages: dependencies: '@types/node': 20.11.27 + /@types/content-disposition@0.5.8: + resolution: {integrity: sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==} + dev: false + + /@types/cookies@0.9.0: + resolution: {integrity: sha512-40Zk8qR147RABiQ7NQnBzWzDcjKzNrntB5BAmeGCb2p/MIyOE+4BVvc17wumsUqUw00bJYqoXFHYygQnEFh4/Q==} + dependencies: + '@types/connect': 3.4.38 + '@types/express': 4.17.21 + '@types/keygrip': 1.0.6 + '@types/node': 20.11.27 + dev: false + /@types/eslint-scope@3.7.7: resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} dependencies: @@ -3053,6 +3106,10 @@ packages: resolution: {integrity: sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==} dev: false + /@types/http-assert@1.5.5: + resolution: {integrity: sha512-4+tE/lwdAahgZT1g30Jkdm9PzFRde0xwxBNUyRsCitRvCQB90iuA2uJYdUnhnANRcqGXaWOGY4FEoxeElNAK2g==} + dev: false + /@types/http-errors@2.0.4: resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} @@ -3103,6 +3160,35 @@ packages: resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} dev: true + /@types/keygrip@1.0.6: + resolution: {integrity: sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==} + dev: false + + /@types/koa-bodyparser@4.3.12: + resolution: {integrity: sha512-hKMmRMVP889gPIdLZmmtou/BijaU1tHPyMNmcK7FAHAdATnRcGQQy78EqTTxLH1D4FTsrxIzklAQCso9oGoebQ==} + dependencies: + '@types/koa': 2.15.0 + dev: false + + /@types/koa-compose@3.2.8: + resolution: {integrity: sha512-4Olc63RY+MKvxMwVknCUDhRQX1pFQoBZ/lXcRLP69PQkEpze/0cr8LNqJQe5NFb/b19DWi2a5bTi2VAlQzhJuA==} + dependencies: + '@types/koa': 2.15.0 + dev: false + + /@types/koa@2.15.0: + resolution: {integrity: sha512-7QFsywoE5URbuVnG3loe03QXuGajrnotr3gQkXcEBShORai23MePfFYdhz90FEtBBpkyIYQbVD+evKtloCgX3g==} + dependencies: + '@types/accepts': 1.3.7 + '@types/content-disposition': 0.5.8 + '@types/cookies': 0.9.0 + '@types/http-assert': 1.5.5 + '@types/http-errors': 2.0.4 + '@types/keygrip': 1.0.6 + '@types/koa-compose': 3.2.8 + '@types/node': 20.11.27 + dev: false + /@types/lodash-es@4.17.12: resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} dependencies: @@ -4435,6 +4521,14 @@ packages: - bluebird dev: false + /cache-content-type@1.0.1: + resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} + engines: {node: '>= 6.0.0'} + dependencies: + mime-types: 2.1.35 + ylru: 1.3.2 + dev: false + /caching-transform@4.0.0: resolution: {integrity: sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==} engines: {node: '>=8'} @@ -4663,6 +4757,20 @@ packages: readable-stream: 2.3.8 dev: false + /co-body@6.1.0: + resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==} + dependencies: + inflation: 2.1.0 + qs: 6.11.0 + raw-body: 2.5.2 + type-is: 1.6.18 + dev: false + + /co@4.6.0: + resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + dev: false + /code-point@1.1.0: resolution: {integrity: sha512-L9JOfOolA/Y/YKVO+WUscMXuYUcHmkleTJkvl1G0XaGFj5JDXl02BobH8avoI/pjWvgOLivs2bizA0N6g8gM9Q==} dev: false @@ -4860,6 +4968,10 @@ packages: typedarray: 0.0.6 dev: false + /config-ini-parser@1.6.1: + resolution: {integrity: sha512-6YBVmXlWiUKDhjn+S/1ju2+8j8Qnj1PqR8yE5DmLrQRtBOc9kgMLyutFl2so1FNVHevR/A4nY7z9EWw8s6g/Cw==} + dev: false + /connect-history-api-fallback@2.0.0: resolution: {integrity: sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==} engines: {node: '>=0.8'} @@ -4891,6 +5003,14 @@ packages: resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} engines: {node: '>= 0.6'} + /cookies@0.9.1: + resolution: {integrity: sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==} + engines: {node: '>= 0.8'} + dependencies: + depd: 2.0.0 + keygrip: 1.1.0 + dev: false + /core-js-compat@3.36.0: resolution: {integrity: sha512-iV9Pd/PsgjNWBXeq8XRtWVSgz2tKAfhfvBs7qxYty+RlRd+OCksaWmOnc4JKrTc1cToXL1N0s3l/vwlxPtdElw==} dependencies: @@ -5415,6 +5535,10 @@ packages: resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} dev: true + /deep-equal@1.0.1: + resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} + dev: false + /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: true @@ -7218,7 +7342,6 @@ packages: engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.3 - dev: true /has-unicode@2.0.1: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} @@ -7332,6 +7455,14 @@ packages: entities: 2.2.0 dev: false + /http-assert@1.5.0: + resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} + engines: {node: '>= 0.8'} + dependencies: + deep-equal: 1.0.1 + http-errors: 1.8.1 + dev: false + /http-cache-semantics@4.1.1: resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} dev: false @@ -7348,6 +7479,17 @@ packages: setprototypeof: 1.1.0 statuses: 1.5.0 + /http-errors@1.8.1: + resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} + engines: {node: '>= 0.6'} + dependencies: + depd: 1.1.2 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 1.5.0 + toidentifier: 1.0.1 + dev: false + /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} @@ -7505,6 +7647,11 @@ packages: resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} dev: false + /inflation@2.1.0: + resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} + engines: {node: '>= 0.8.0'} + dev: false + /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: @@ -7665,6 +7812,13 @@ packages: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + /is-generator-function@1.0.10: + resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} + engines: {node: '>= 0.4'} + dependencies: + has-tostringtag: 1.0.2 + dev: false + /is-glob@3.1.0: resolution: {integrity: sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==} engines: {node: '>=0.10.0'} @@ -8193,6 +8347,13 @@ packages: resolution: {integrity: sha512-hRkd6/XW4HTsA9vjVpY9tuXJYLSlelnkTmVFu4M9/7MIYQtFcHpbugAU7UbOfjOiVSVYl2fqgBuJ32JUmRo5Ew==} dev: false + /keygrip@1.1.0: + resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} + engines: {node: '>= 0.6'} + dependencies: + tsscmp: 1.0.6 + dev: false + /keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} dependencies: @@ -8211,6 +8372,49 @@ packages: resolution: {integrity: sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==} dev: true + /koa-compose@4.1.0: + resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} + dev: false + + /koa-convert@2.0.0: + resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} + engines: {node: '>= 10'} + dependencies: + co: 4.6.0 + koa-compose: 4.1.0 + dev: false + + /koa@2.15.0: + resolution: {integrity: sha512-KEL/vU1knsoUvfP4MC4/GthpQrY/p6dzwaaGI6Rt4NQuFqkw3qrvsdYF5pz3wOfi7IGTvMPHC9aZIcUKYFNxsw==} + engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} + dependencies: + accepts: 1.3.8 + cache-content-type: 1.0.1 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookies: 0.9.1 + debug: 4.3.4 + delegates: 1.0.0 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + fresh: 0.5.2 + http-assert: 1.5.0 + http-errors: 1.8.1 + is-generator-function: 1.0.10 + koa-compose: 4.1.0 + koa-convert: 2.0.0 + on-finished: 2.4.1 + only: 0.0.2 + parseurl: 1.3.3 + statuses: 1.5.0 + type-is: 1.6.18 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + /kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} dev: true @@ -9138,6 +9342,10 @@ packages: dependencies: mimic-fn: 2.1.0 + /only@0.0.2: + resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} + dev: false + /open@10.1.0: resolution: {integrity: sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==} engines: {node: '>=18'} @@ -12033,6 +12241,11 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + /tsscmp@1.0.6: + resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} + engines: {node: '>=0.6.x'} + dev: false + /ttf2eot@3.1.0: resolution: {integrity: sha512-aHTbcYosNHVqb2Qtt9Xfta77ae/5y0VfdwNLUS6sGBeGr22cX2JDMo/i5h3uuOf+FAD3akYOr17+fYd5NK8aXw==} hasBin: true @@ -12972,6 +13185,11 @@ packages: fd-slicer: 1.1.0 dev: true + /ylru@1.3.2: + resolution: {integrity: sha512-RXRJzMiK6U2ye0BlGGZnmpwJDPgakn6aNQ0A7gHRbD4I0uvK4TW6UqkK1V0pp9jskjJBAXd3dRrbzWkqJ+6cxA==} + engines: {node: '>= 4.0.0'} + dev: false + /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} diff --git a/puppet/zulip/manifests/app_frontend_base.pp b/puppet/zulip/manifests/app_frontend_base.pp index 5d9c846495..20b8e84fe8 100644 --- a/puppet/zulip/manifests/app_frontend_base.pp +++ b/puppet/zulip/manifests/app_frontend_base.pp @@ -177,6 +177,9 @@ class zulip::app_frontend_base { include zulip::smokescreen } + $katex_server = zulipconf('application_server', 'katex_server', false) + $katex_server_port = zulipconf('application_server', 'katex_server_port', '9700') + if $proxy_host != '' and $proxy_port != '' { $proxy = "http://${proxy_host}:${proxy_port}" } else { diff --git a/puppet/zulip/templates/supervisor/zulip.conf.template.erb b/puppet/zulip/templates/supervisor/zulip.conf.template.erb index 1b5a00103b..d98472b647 100644 --- a/puppet/zulip/templates/supervisor/zulip.conf.template.erb +++ b/puppet/zulip/templates/supervisor/zulip.conf.template.erb @@ -106,6 +106,23 @@ programs=<% @queues.each_with_index do |queue, i| -%>zulip_events_<%= queue %><% programs=zulip_events <% end %> +<% if @katex_server %> +[program:zulip-katex] +command=/usr/local/bin/secret-env-wrapper SHARED_SECRET=shared_secret /srv/zulip-node/bin/node /home/zulip/deployments/current/static/webpack-bundles/katex_server.js <%= @katex_server_port %> +environment=NODE_ENV=production +priority=200 ; the relative start priority (default 999) +autostart=true ; start at supervisord start (default: true) +autorestart=true ; whether/when to restart (default: unexpected) +stopsignal=TERM ; signal used to kill process (default TERM) +stopwaitsecs=30 ; max num secs to wait b4 SIGKILL (default 10) +user=zulip ; setuid to this UNIX account to run the program +redirect_stderr=true ; redirect proc stderr to stdout (default false) +stdout_logfile=/var/log/zulip/katex.log ; stdout log path, NONE for none; default AUTO +stdout_logfile_maxbytes=20MB ; max # logfile bytes b4 rotation (default 50MB) +stdout_logfile_backups=3 ; # of stdout logfile backups (default 10) +directory=/home/zulip/deployments/current/ +<% end %> + ; The [include] section can just contain the "files" setting. This ; setting can list multiple files (separated by whitespace or ; newlines). It can also contain wildcards. The filenames are diff --git a/scripts/restart-server b/scripts/restart-server index 211286cb5b..46f9eb1854 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -101,12 +101,17 @@ if has_application_server(): ) ) + # This is an optional service, so may or may not exist + workers.extend(list_supervisor_processes(["zulip-katex"])) + if has_process_fts_updates(): workers.append("process-fts-updates") # Before we start (re)starting main services, make sure to start any # optional auxiliary services that we don't stop, but do expect to be -# running, and aren't currently. +# running, and aren't currently. These get new versions by getting +# updated supervisor files, and puppet restarts them -- so we never +# restart them in here, only start them. aux_services = list_supervisor_processes(["go-camo", "smokescreen"], only_running=False) if aux_services: subprocess.check_call(["supervisorctl", "start", *aux_services]) diff --git a/tools/ci/production-build b/tools/ci/production-build index 9d1b8040e8..9097096b4d 100755 --- a/tools/ci/production-build +++ b/tools/ci/production-build @@ -48,6 +48,6 @@ cp -a \ PNPM="/usr/local/bin/pnpm" tar -C /tmp -xzf /tmp/production-build/zulip-server-test.tar.gz zulip-server-test/prod-static/serve/webpack-bundles ( - GLOBIGNORE=/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex-cli.js + GLOBIGNORE="/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex*.js" "$PNPM" exec es-check es2020 /tmp/zulip-server-test/prod-static/serve/webpack-bundles/*.js ) diff --git a/tools/webpack b/tools/webpack index 009190f60c..70098fa556 100755 --- a/tools/webpack +++ b/tools/webpack @@ -46,7 +46,7 @@ def build_for_dev_server(host: str, port: str, minify: bool, disable_host_check: webpack_args = ["../node_modules/.bin/webpack-cli", "serve"] webpack_args += [ # webpack-cli has a bug where it ignores --watch-poll with - # multi-config, and we don't need the katex-cli part anyway. + # multi-config, and we don't need the katex part anyway. "--config-name=frontend", f"--host={host}", f"--port={port}", diff --git a/version.py b/version.py index 926c1b016e..3a6515e5ce 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 243 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (264, 3) +PROVISION_VERSION = (264, 4) diff --git a/web/server/katex_server.ts b/web/server/katex_server.ts new file mode 100644 index 0000000000..950465e4da --- /dev/null +++ b/web/server/katex_server.ts @@ -0,0 +1,78 @@ +import crypto from "node:crypto"; + +import bodyParser from "@koa/bodyparser"; +import katex from "katex"; +import Koa from "koa"; + +const host = "localhost"; +const port = Number(process.argv[2] ?? "9700"); +if (!Number.isInteger(port)) { + throw new TypeError("Invalid port"); +} + +const shared_secret = process.env.SHARED_SECRET; +if (typeof shared_secret !== "string") { + console.error("No SHARED_SECRET set!"); + process.exit(1); +} +const compare_secret = (given_secret: string): boolean => { + try { + // Throws an exception if the strings are unequal length + return crypto.timingSafeEqual( + Buffer.from(shared_secret, "utf8"), + Buffer.from(given_secret, "utf8"), + ); + } catch { + return false; + } +}; + +const app = new Koa(); +app.use(bodyParser()); + +app.use((ctx, _next) => { + if (ctx.request.method !== "POST" || ctx.request.path !== "/") { + ctx.status = 404; + return; + } + if (ctx.request.body === undefined) { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = "Missing POST body"; + return; + } + const given_secret = ctx.request.body.shared_secret; + if (typeof given_secret !== "string" || !compare_secret(given_secret)) { + ctx.status = 403; + ctx.type = "text/plain"; + ctx.body = "Invalid 'shared_secret' argument"; + return; + } + + const content = ctx.request.body.content; + const is_display = ctx.request.body.is_display === "true"; + + if (typeof content !== "string") { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = "Invalid 'content' argument"; + return; + } + + try { + ctx.body = katex.renderToString(content, {displayMode: is_display}); + } catch (error) { + if (error instanceof katex.ParseError) { + ctx.status = 400; + ctx.type = "text/plain"; + ctx.body = error.message; + } else { + ctx.status = 500; + console.error(error); + } + } +}); + +app.listen(port, host, () => { + console.log(`Server started on http://${host}:${port}`); +}); diff --git a/web/webpack.config.ts b/web/webpack.config.ts index 702c907b31..e4c49aa7d1 100644 --- a/web/webpack.config.ts +++ b/web/webpack.config.ts @@ -255,11 +255,18 @@ export default ( name: "server", target: "node", entry: { + katex_server: "babel-loader!./server/katex_server.ts", "katex-cli": "shebang-loader!katex/cli", }, output: { path: path.resolve(__dirname, "../static/webpack-bundles"), }, + resolve: { + alias: { + // koa-body uses formidable 2.x, which suffers from https://github.com/node-formidable/formidable/issues/337 + hexoid: "hexoid/dist/index.js", + }, + }, }; return [frontendConfig, serverConfig]; diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 23de3a725f..6e2b399a91 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -1,14 +1,24 @@ import logging import os import subprocess -from typing import Optional +from typing import Any, Optional import lxml.html +import requests from django.conf import settings +from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.storage import static_path +class KatexSession(OutgoingSession): + def __init__(self, **kwargs: Any) -> None: + # We set a very short timeout because these requests are + # expected to be quite fast (milliseconds) and blocking on + # this affects message rendering performance. + super().__init__(role="katex", timeout=0.5, **kwargs) + + def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX @@ -23,6 +33,38 @@ def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: (default True) """ + if settings.KATEX_SERVER: + try: + resp = KatexSession().post( + # We explicitly disable the Smokescreen proxy for this + # call, since it intentionally connects to localhost. + # This is safe because the host is explicitly fixed, and + # the port is pulled from our own configuration. + f"http://localhost:{settings.KATEX_SERVER_PORT}/", + data={ + "content": tex, + "is_display": "false" if is_inline else "true", + "shared_secret": settings.SHARED_SECRET, + }, + proxies={"http": ""}, + ) + except requests.exceptions.Timeout: + logging.warning("KaTeX rendering service timed out with %d byte long input", len(tex)) + return None + except requests.exceptions.RequestException as e: + logging.warning("KaTeX rendering service failed: %s", type(e).__name__) + return None + + if resp.status_code == 200: + return resp.content.decode().strip() + elif resp.status_code == 400: + return None + else: + logging.warning( + "KaTeX rendering service failed: (%s) %s", resp.status_code, resp.content.decode() + ) + return None + katex_path = ( static_path("webpack-bundles/katex-cli.js") if settings.PRODUCTION diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index f1c899614b..9610ee7788 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -7,10 +7,13 @@ from typing import Any, Dict, List, Optional, Set, Tuple from unittest import mock import orjson +import requests +import responses from bs4 import BeautifulSoup from django.conf import settings from django.test import override_settings from markdown import Markdown +from responses import matchers from typing_extensions import override from zerver.actions.alert_words import do_add_alert_words @@ -297,6 +300,79 @@ class MarkdownMiscTest(ZulipTestCase): render_tex("random text") self.assertEqual(m.output, ["ERROR:root:Cannot find KaTeX for latex rendering!"]) + @responses.activate + @override_settings(KATEX_SERVER=True, SHARED_SECRET="foo") + def test_katex_server(self) -> None: + responses.post( + "http://localhost:9700/", + match=[ + matchers.urlencoded_params_matcher( + {"content": "foo", "is_display": "false", "shared_secret": "foo"} + ) + ], + content_type="text/html; charset=utf-8", + body="html", + ) + self.assertEqual(render_tex("foo"), "html") + + responses.post( + "http://localhost:9700/?", + match=[ + matchers.urlencoded_params_matcher( + {"content": "foo", "is_display": "true", "shared_secret": "foo"} + ) + ], + content_type="text/html; charset=utf-8", + body="other", + ) + self.assertEqual(render_tex("foo", is_inline=False), "other") + + responses.post( + "http://localhost:9700/", + content_type="text/html; charset=utf-8", + status=400, + body=r"KaTeX parse error: '\'", + ) + self.assertEqual(render_tex("bad"), None) + + responses.post( + "http://localhost:9700/", + content_type="text/html; charset=utf-8", + status=400, + body=r"KaTeX parse error: '\'", + ) + self.assertEqual(render_tex("bad"), None) + + responses.post("http://localhost:9700/", status=403, body="") + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: (403) "]) + + responses.post("http://localhost:9700/", status=500, body="") + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: (500) "]) + + responses.post("http://localhost:9700/", body=requests.exceptions.Timeout()) + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual( + m.output, ["WARNING:root:KaTeX rendering service timed out with 3 byte long input"] + ) + + responses.post("http://localhost:9700/", body=requests.exceptions.ConnectionError()) + with self.assertLogs(level="WARNING") as m: + self.assertEqual(render_tex("bad"), None) + self.assertEqual(m.output, ["WARNING:root:KaTeX rendering service failed: ConnectionError"]) + + with override_settings(KATEX_SERVER_PORT=9701): + responses.post( + "http://localhost:9701/", + body="html", + content_type="text/html; charset=utf-8", + ) + self.assertEqual(render_tex("foo"), "html") + class MarkdownListPreprocessorTest(ZulipTestCase): # We test that the preprocessor inserts blank lines at correct places. diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 91035f42c9..94b6636172 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -520,6 +520,14 @@ INTERNAL_BOT_DOMAIN = "zulip.com" # This needs to be synced with the Camo installation CAMO_KEY = get_secret("camo_key") if CAMO_URI != "" else None +######################################################################## +# KATEX SERVER SETTINGS +######################################################################## + +KATEX_SERVER = get_config("application_server", "katex_server", False) +KATEX_SERVER_PORT = get_config("application_server", "katex_server_port", "9700") + + ######################################################################## # STATIC CONTENT AND MINIFICATION SETTINGS ########################################################################