diff --git a/puppet/kandra/manifests/prometheus/akamai.pp b/puppet/kandra/manifests/prometheus/akamai.pp index 2aad0f7ddd..b686926623 100644 --- a/puppet/kandra/manifests/prometheus/akamai.pp +++ b/puppet/kandra/manifests/prometheus/akamai.pp @@ -5,33 +5,19 @@ class kandra::prometheus::akamai { include kandra::vector include zulip::supervisor - $bin = $kandra::vector::bin - $conf = '/etc/vector.toml' $pipelines = { 'static' => zulipsecret('secrets', 'akamai_static_sqs_url', ''), 'realm' => zulipsecret('secrets', 'akamai_realm_sqs_url', ''), } - file { $conf: - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - content => template('kandra/vector.toml.template.erb'), + concat::fragment { 'vector_akamai': + target => $kandra::vector::conf, + order => '50', + content => template('kandra/vector_akamai.toml.template.erb'), } - file { "${zulip::common::supervisor_conf_dir}/prometheus_akamai_exporter.conf": - ensure => file, - require => [ - User[zulip], - Package[supervisor], - File['/etc/vector.toml'], - File[$bin], - ], - before => Exec['Cleanup vector'], - owner => 'root', - group => 'root', - mode => '0644', - content => template('kandra/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb'), - notify => Service[supervisor], + concat::fragment { 'vector_akamai_export': + target => $kandra::vector::conf, + content => ',"akamai_logs2metrics*"', + order => '90', } } diff --git a/puppet/kandra/manifests/vector.pp b/puppet/kandra/manifests/vector.pp index e44467d3c5..9126dd8800 100644 --- a/puppet/kandra/manifests/vector.pp +++ b/puppet/kandra/manifests/vector.pp @@ -4,6 +4,7 @@ class kandra::vector { $version = $zulip::common::versions['vector']['version'] $dir = "/srv/zulip-vector-${version}" $bin = "${dir}/bin/vector" + $conf = '/etc/vector.toml' $arch = $facts['os']['architecture'] ? { 'amd64' => 'x86_64', @@ -17,4 +18,56 @@ class kandra::vector { bin => [$bin], cleanup_after => [Service[supervisor]], } + file { "${zulip::common::supervisor_conf_dir}/vector.conf": + ensure => file, + require => [ + User[zulip], + Package[supervisor], + Concat[$conf], + File[$bin], + ], + before => Exec['Cleanup vector'], + owner => 'root', + group => 'root', + mode => '0644', + content => template('kandra/supervisor/conf.d/vector.conf.template.erb'), + notify => Service[supervisor], + } + + exec { 'reload vector': + command => 'supervisorctl signal HUP vector', + require => Service[supervisor], + refreshonly => true, + } + concat { $conf: + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + notify => Exec['reload vector'], + } + # All of the pipelines need to be included in the Prometheus + # exporter; they insert their strings at order 90, with a leading + # comma, in the middle of the "inputs" block + $vector_export = @(VECTOR) + [sources.vector_metrics] + type = "internal_metrics" + [sinks.prometheus_exporter] + type = "prometheus_exporter" + address = "0.0.0.0:9081" + flush_period_secs = 120 + suppress_timestamp = true + buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5] + inputs = ["vector_metrics" + |-VECTOR + concat::fragment { 'vector_export_prefix': + target => $conf, + content => $vector_export, + order => '89', + } + concat::fragment { 'vector_export_suffix': + target => $conf, + content => "]\n", + order => '99', + } } diff --git a/puppet/kandra/templates/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb b/puppet/kandra/templates/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb deleted file mode 100644 index 1bfc221229..0000000000 --- a/puppet/kandra/templates/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb +++ /dev/null @@ -1,8 +0,0 @@ -[program:prometheus_akamai_exporter] -command=<%= @bin %> --config <%= @conf %> -priority=10 -autostart=true -autorestart=true -user=zulip -redirect_stderr=true -stdout_logfile=/var/log/zulip/akamai_exporter.log diff --git a/puppet/kandra/templates/supervisor/conf.d/vector.conf.template.erb b/puppet/kandra/templates/supervisor/conf.d/vector.conf.template.erb new file mode 100644 index 0000000000..fd7d10647e --- /dev/null +++ b/puppet/kandra/templates/supervisor/conf.d/vector.conf.template.erb @@ -0,0 +1,8 @@ +[program:vector] +command=<%= @bin %> --config <%= @conf %> --require-healthy true +priority=10 +autostart=true +autorestart=true +user=zulip +redirect_stderr=true +stdout_logfile=/var/log/zulip/vector.log diff --git a/puppet/kandra/templates/vector.toml.template.erb b/puppet/kandra/templates/vector_akamai.toml.template.erb similarity index 67% rename from puppet/kandra/templates/vector.toml.template.erb rename to puppet/kandra/templates/vector_akamai.toml.template.erb index b1d9b79db1..73890b6bc8 100644 --- a/puppet/kandra/templates/vector.toml.template.erb +++ b/puppet/kandra/templates/vector_akamai.toml.template.erb @@ -1,6 +1,3 @@ -[sources.vector_metrics] - type = "internal_metrics" - # Akamai Datastream2 logs all accesses into AWS S3: # https://techdocs.akamai.com/datastream2/docs/stream-amazon-s3 # @@ -19,7 +16,7 @@ sqs.poll_secs = 15 sqs.queue_url = "<%= sqs_url %>" -[transforms.parse_<%= key %>] +[transforms.akamai_parse_<%= key %>] type = "remap" inputs = ["s3_akamai_<%= key %>"] source = ''' @@ -27,47 +24,39 @@ .turnAroundTimeSec = to_int!(.turnAroundTimeMSec) / 1000.0 ''' -[transforms.logs2metrics_<%= key %>] +[transforms.akamai_logs2metrics_<%= key %>] type = "log_to_metric" - inputs = ["parse_<%= key %>"] + inputs = ["akamai_parse_<%= key %>"] - [[transforms.logs2metrics_<%= key %>.metrics]] + [[transforms.akamai_logs2metrics_<%= key %>.metrics]] field = "cacheStatus" name = "requests_cache_count" namespace = "akamai_<%= key %>" type = "counter" - [transforms.logs2metrics_<%= key %>.metrics.tags] + [transforms.akamai_logs2metrics_<%= key %>.metrics.tags] status_code = "{{statusCode}}" cached = "{{cacheStatus}}" host = "{{reqHost}}" - [[transforms.logs2metrics_<%= key %>.metrics]] + [[transforms.akamai_logs2metrics_<%= key %>.metrics]] field = "bytes" name = "requests_bytes" namespace = "akamai_<%= key %>" type = "counter" increment_by_value = true - [transforms.logs2metrics_<%= key %>.metrics.tags] + [transforms.akamai_logs2metrics_<%= key %>.metrics.tags] status_code = "{{statusCode}}" cached = "{{cacheStatus}}" host = "{{reqHost}}" - [[transforms.logs2metrics_<%= key %>.metrics]] + [[transforms.akamai_logs2metrics_<%= key %>.metrics]] field = "turnAroundTimeSec" name = "turnaround_time_sec" namespace = "akamai_<%= key %>" type = "histogram" - [transforms.logs2metrics_<%= key %>.metrics.tags] + [transforms.akamai_logs2metrics_<%= key %>.metrics.tags] status_code = "{{statusCode}}" cached = "{{cacheStatus}}" host = "{{reqHost}}" <% end %> - -[sinks.prometheus_exporter] - type = "prometheus_exporter" - inputs = ["vector_metrics", "logs2metrics*"] - buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5] - address = "0.0.0.0:9081" - flush_period_secs = 120 - suppress_timestamp = true