From bd217ad31b154bcca93e2c93af2a8652285ae86c Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Thu, 8 Jun 2023 19:30:41 +0000 Subject: [PATCH] puppet: Read resolver from /etc/resolv.conf. 04cf68b45ebb make nginx responsible for downloading (and caching) files from S3. As noted in that commit, nginx implements its own non-blocking DNS resolver, since the base syscall is blocking, so requires an explicit nameserver configuration. That commit used 127.0.0.53, which is provided by systemd-resolved, as the resolver. However, that service may not always be enabled and running, and may in fact not even be installed (e.g. on Docker). Switch to parsing `/etc/resolv.conf` and using the first-provided nameserver. In many deployments, this will still be `127.0.0.53`, but for others it will provide a working DNS server which is external to the host. In the event that a server is misconfigured and has no resolvers in `/etc/resolv.conf`, it will error out: ```console Error: Evaluation Error: Error while evaluating a Function Call, No nameservers found in /etc/resolv.conf! Configure one by setting application_server.nameserver in /etc/zulip/zulip.conf (file: /home/zulip/deployments/current/puppet/zulip/manifests/app_frontend_base.pp, line: 76, column: 70) on node example.zulipdev.org ``` --- docs/production/deployment.md | 8 ++++++++ docs/production/upload-backends.md | 12 ++++++++++++ .../zulip-include-frontend/uploads-internal.conf | 7 ------- puppet/zulip/lib/puppet/functions/resolver_ip.rb | 11 +++++++++++ puppet/zulip/manifests/app_frontend_base.pp | 13 ++++++++++++- puppet/zulip/templates/nginx/s3-cache.template.erb | 7 +++++++ 6 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 puppet/zulip/lib/puppet/functions/resolver_ip.rb diff --git a/docs/production/deployment.md b/docs/production/deployment.md index 708f8188c6..ca1af2caab 100644 --- a/docs/production/deployment.md +++ b/docs/production/deployment.md @@ -824,6 +824,14 @@ immutable, this serves only as a potential additional limit on the size of the contents on disk; `s3_disk_cache_size` is expected to be the primary control for cache sizing. +#### `nameserver` + +When the [S3 storage backend][s3-backend] is in use, downloads from S3 are +proxied from nginx, whose configuration requires an explicit value of a DNS +nameserver to resolve the S3 server's hostname. Zulip defaults to using the +resolver found in `/etc/resolv.conf`; this setting overrides any value found +there. + [s3-backend]: upload-backends.md #### `uwsgi_listen_backlog_limit` diff --git a/docs/production/upload-backends.md b/docs/production/upload-backends.md index 9b2f651afc..7fe1c65e1e 100644 --- a/docs/production/upload-backends.md +++ b/docs/production/upload-backends.md @@ -85,6 +85,18 @@ You may also wish to increase the cache sizes if the S3 storage (or S3-compatible equivalent) is not closely located to your Zulip server, as cache misses will be more expensive. +## nginx DNS nameserver configuration + +The S3 cache described above is maintained by nginx. nginx's configuration +requires an explicitly-set DNS nameserver to resolve the hostname of the S3 +servers; Zulip defaults this value to the first nameserver found in +`/etc/resolv.conf`, but this resolver can be [adjusted in +`/etc/zulip/zulip.conf`][s3-resolver] if needed. If you adjust this value, you +will need to run `/home/zulip/deployments/current/scripts/zulip-puppet-apply` to +update the nginx configuration for the new value. + +[s3-resolver]: deployment.md#nameserver + ## S3 bucket policy The best way to do the S3 integration with Amazon is to create a new IAM user diff --git a/puppet/zulip/files/nginx/zulip-include-frontend/uploads-internal.conf b/puppet/zulip/files/nginx/zulip-include-frontend/uploads-internal.conf index 9012561d01..015ab09a83 100644 --- a/puppet/zulip/files/nginx/zulip-include-frontend/uploads-internal.conf +++ b/puppet/zulip/files/nginx/zulip-include-frontend/uploads-internal.conf @@ -23,13 +23,6 @@ location ~ ^/internal/s3/(?[^/]+)/(?.*) { # the first response. Django explicitly unsets the first, and # does not set the latter two. - # nginx does its own DNS resolution, which is necessary here to - # resolve the IP of the S3 server. Point it at the local caching - # systemd resolved service. The validity duration is set to match - # S3's DNS validity. - resolver 127.0.0.53 valid=300s; - resolver_timeout 10s; - proxy_pass $download_url$is_args$args; proxy_cache uploads; # If the S3 response doesn't contain Cache-Control headers (which diff --git a/puppet/zulip/lib/puppet/functions/resolver_ip.rb b/puppet/zulip/lib/puppet/functions/resolver_ip.rb new file mode 100644 index 0000000000..c270234b58 --- /dev/null +++ b/puppet/zulip/lib/puppet/functions/resolver_ip.rb @@ -0,0 +1,11 @@ +require "resolv" + +Puppet::Functions.create_function(:resolver_ip) do + def resolver_ip() + parsed = Resolv::DNS::Config.default_config_hash() + if parsed[:nameserver].empty? + raise 'No nameservers found in /etc/resolv.conf! Configure one by setting application_server.nameserver in /etc/zulip/zulip.conf' + end + parsed[:nameserver][0] + end +end diff --git a/puppet/zulip/manifests/app_frontend_base.pp b/puppet/zulip/manifests/app_frontend_base.pp index 05e2a2854a..9a37f5de89 100644 --- a/puppet/zulip/manifests/app_frontend_base.pp +++ b/puppet/zulip/manifests/app_frontend_base.pp @@ -73,8 +73,19 @@ class zulip::app_frontend_base { $s3_memory_cache_size = zulipconf('application_server', 's3_memory_cache_size', '1M') $s3_disk_cache_size = zulipconf('application_server', 's3_disk_cache_size', '200M') $s3_cache_inactive_time = zulipconf('application_server', 's3_cache_inactive_time', '30d') + $configured_nginx_resolver = zulipconf('application_server', 'nameserver', '') + if $configured_nginx_resolver == '' { + # This may fail in the unlikely change that there is no configured + # resolver in /etc/resolv.conf, so only call it is unset in zulip.conf + $nginx_resolver_ip = resolver_ip() + } else { + $nginx_resolver_ip = $configured_nginx_resolver + } file { '/etc/nginx/zulip-include/s3-cache': - require => [Package[$zulip::common::nginx], File['/srv/zulip-uploaded-files-cache']], + require => [ + Package[$zulip::common::nginx], + File['/srv/zulip-uploaded-files-cache'], + ], owner => 'root', group => 'root', mode => '0644', diff --git a/puppet/zulip/templates/nginx/s3-cache.template.erb b/puppet/zulip/templates/nginx/s3-cache.template.erb index 23772e8432..ea93a38580 100644 --- a/puppet/zulip/templates/nginx/s3-cache.template.erb +++ b/puppet/zulip/templates/nginx/s3-cache.template.erb @@ -1,3 +1,10 @@ +# nginx does its own DNS resolution, which is necessary here to +# resolve the IP of the S3 server. Point it at whatever is configured +# first in /etc/resolv.conf. The validity duration is set to match +# S3's DNS validity. +resolver <%= @nginx_resolver_ip %> valid=300s; +resolver_timeout 10s; + # This cache is only used if S3 file storage is configured. proxy_cache_path /srv/zulip-uploaded-files-cache levels=1:2