zulip/puppet/kandra/templates/vector.toml.template.erb

74 lines
2.2 KiB
Plaintext

[sources.vector_metrics]
type = "internal_metrics"
# Akamai Datastream2 logs all accesses into AWS S3:
# https://techdocs.akamai.com/datastream2/docs/stream-amazon-s3
#
# The S3 bucket is configured to send event notifications to the SQS
# queue, which this host is allowed to read from. This consumer
# deletes the messages from the queue, and the S3 bucket is
# configured to purge old logs.
# https://vector.dev/docs/reference/configuration/sources/aws_s3/
<% @pipelines.each do |key,sqs_url| %>
[sources.s3_akamai_<%= key %>]
type = "aws_s3"
region = "us-east-1"
compression = "gzip"
sqs.delete_message = true
sqs.poll_secs = 15
sqs.queue_url = "<%= sqs_url %>"
[transforms.parse_<%= key %>]
type = "remap"
inputs = ["s3_akamai_<%= key %>"]
source = '''
. = parse_json!(string!(.message))
.turnAroundTimeSec = to_int!(.turnAroundTimeMSec) / 1000.0
'''
[transforms.logs2metrics_<%= key %>]
type = "log_to_metric"
inputs = ["parse_<%= key %>"]
[[transforms.logs2metrics_<%= key %>.metrics]]
field = "cacheStatus"
name = "requests_cache_count"
namespace = "akamai_<%= key %>"
type = "counter"
[transforms.logs2metrics_<%= key %>.metrics.tags]
status_code = "{{statusCode}}"
cached = "{{cacheStatus}}"
host = "{{reqHost}}"
[[transforms.logs2metrics_<%= key %>.metrics]]
field = "bytes"
name = "requests_bytes"
namespace = "akamai_<%= key %>"
type = "counter"
increment_by_value = true
[transforms.logs2metrics_<%= key %>.metrics.tags]
status_code = "{{statusCode}}"
cached = "{{cacheStatus}}"
host = "{{reqHost}}"
[[transforms.logs2metrics_<%= key %>.metrics]]
field = "turnAroundTimeSec"
name = "turnaround_time_sec"
namespace = "akamai_<%= key %>"
type = "histogram"
[transforms.logs2metrics_<%= key %>.metrics.tags]
status_code = "{{statusCode}}"
cached = "{{cacheStatus}}"
host = "{{reqHost}}"
<% end %>
[sinks.prometheus_exporter]
type = "prometheus_exporter"
inputs = ["vector_metrics", "logs2metrics*"]
buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
address = "0.0.0.0:9081"
flush_period_secs = 120
suppress_timestamp = true