From 1bd0ab506c7c389d33d25a6cab74fefa4e586b05 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 7 Oct 2024 16:09:59 -0400 Subject: [PATCH] nginx: Make uwsgi timeout shorter than nginx-to-uwsgi timeout. The nginx-to-uwsig-timeout defaults to 60s, which is exactly the same as the current "harakiri" timeout configured in uwsgi (which limits the length a request can run before the worker is terminated). This causes a race, where if nginx hits its 60s before uwsgi, then we return a 504; otherwise, we get a 502. Make the nginx-to-uwsgi timeout explicit, and shorten the "harakiri" timeout to be explicitly less than that. Document the 60s timeout, which all outer reverse proxies must be set to _longer than_ in order to have proper "onion" timeouts. --- docs/production/reverse-proxies.md | 12 ++++++------ puppet/zulip/files/nginx/uwsgi_params | 4 ++++ puppet/zulip/templates/uwsgi.ini.template.erb | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/production/reverse-proxies.md b/docs/production/reverse-proxies.md index 5717637e3f..45b8b65f08 100644 --- a/docs/production/reverse-proxies.md +++ b/docs/production/reverse-proxies.md @@ -255,13 +255,13 @@ things you need to be careful about when configuring it: browsers. This [nginx code snippet][nginx-proxy-longpolling-config] does this. - The key configuration options are, for the `/json/events` and - `/api/1/events` endpoints: + The key configuration options are: - - `proxy_read_timeout 1200;`. It's critical that this be - significantly above 60s, but the precise value isn't important. - - `proxy_buffering off`. If you don't do this, your `nginx` proxy may - return occasional 502 errors to clients using Zulip's events API. + - `proxy_read_timeout 1200;`. It's critical that this be significantly above + 60s, but the precise value isn't important. This is most important for the + events API, but must be applied to all endpoints. + - `proxy_buffering off`. If you don't do this, your `nginx` proxy may return + occasional 502 errors to clients using Zulip's events API. 1. The other tricky failure mode we've seen with `nginx` reverse proxies is that they can load-balance between the IPv4 and IPv6 diff --git a/puppet/zulip/files/nginx/uwsgi_params b/puppet/zulip/files/nginx/uwsgi_params index 09a1006932..47acef235e 100644 --- a/puppet/zulip/files/nginx/uwsgi_params +++ b/puppet/zulip/files/nginx/uwsgi_params @@ -18,4 +18,8 @@ uwsgi_param HTTP_X_FORWARDED_PROTO $trusted_x_forwarded_proto; uwsgi_param HTTP_X_FORWARDED_SSL ""; uwsgi_param HTTP_X_PROXY_MISCONFIGURATION $x_proxy_misconfiguration; +# This value is the default, and is provided for explicitness; it must +# be longer than the configured 55s "harakiri" timeout in uwsgi +uwsgi_read_timeout 60s; + uwsgi_pass django; diff --git a/puppet/zulip/templates/uwsgi.ini.template.erb b/puppet/zulip/templates/uwsgi.ini.template.erb index 825d578547..eab67bb121 100644 --- a/puppet/zulip/templates/uwsgi.ini.template.erb +++ b/puppet/zulip/templates/uwsgi.ini.template.erb @@ -47,7 +47,7 @@ auto-procname=true procname-prefix-spaced=zulip-django # Longest response allowed, in seconds, before killing the worker -harakiri=60 +harakiri=55