mirror of https://github.com/zulip/zulip.git
grafana: Support notifications from Grafana Alerting.
This commit adds support for Grafana's new alerting system, Grafana Alerting. The existing Grafana integration has been modified to detect the version of the notification through the structure of the payload body, since the the structure varies by version. Support for legacy alerting is been continued. Example fixtures have been added for Grafana Alerting's webhooks. Tests updated.
This commit is contained in:
Hari Prashant Bhimaraju
Tim Abbott
parent
804eb15aa5
commit
977a043d03
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"receiver": "",
|
||||
"status": "resolved",
|
||||
"alerts": [
|
||||
{
|
||||
"status": "resolved",
|
||||
"labels": {
|
||||
"alertname": "TestAlert",
|
||||
"instance": "Grafana"
|
||||
},
|
||||
"annotations": {
|
||||
"summary": "Notification test"
|
||||
},
|
||||
"startsAt": "2022-08-31T05:54:04.52289368Z",
|
||||
"endsAt": "2022-08-31T10:30:00.52288431Z",
|
||||
"generatorURL": "",
|
||||
"fingerprint": "57c6d9296de2ad39",
|
||||
"silenceURL": "https://zuliptestingwh2.grafana.net/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTestAlert&matcher=instance%3DGrafana",
|
||||
"dashboardURL": "",
|
||||
"panelURL": "",
|
||||
"valueString": "[ metric='foo' labels={instance=bar} value=10 ]"
|
||||
}
|
||||
],
|
||||
"groupLabels": {},
|
||||
"commonLabels": {
|
||||
"alertname": "TestAlert",
|
||||
"instance": "Grafana"
|
||||
},
|
||||
"commonAnnotations": {
|
||||
"summary": "Notification test"
|
||||
},
|
||||
"externalURL": "https://zuliptestingwh2.grafana.net/",
|
||||
"version": "1",
|
||||
"groupKey": "{alertname=\"TestAlert\", instance=\"Grafana\"}2022-08-31 05:54:04.52289368 +0000 UTC m=+42208.256292221",
|
||||
"truncatedAlerts": 1,
|
||||
"orgId": 1,
|
||||
"title": "[RESOLVED:1] (TestAlert Grafana)",
|
||||
"state": "alerting",
|
||||
"message": "Webhook test message."
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
"receiver": "My Super Webhook",
|
||||
"status": "firing",
|
||||
"orgId": 1,
|
||||
"alerts": [
|
||||
{
|
||||
"status": "firing",
|
||||
"labels": {
|
||||
"alertname": "High memory usage",
|
||||
"team": "blue",
|
||||
"zone": "us-1"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "The system has high memory usage",
|
||||
"runbook_url": "https://myrunbook.com/runbook/1234",
|
||||
"summary": "This alert was triggered for zone us-1"
|
||||
},
|
||||
"startsAt": "2021-10-12T09:51:03.157076+02:00",
|
||||
"endsAt": "0001-01-01T00:00:00Z",
|
||||
"generatorURL": "https://play.grafana.org/alerting/1afz29v7z/edit",
|
||||
"fingerprint": "c6eadffa33fcdf37",
|
||||
"silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT2%2Cteam%3Dblue%2Czone%3Dus-1",
|
||||
"dashboardURL": "",
|
||||
"panelURL": "",
|
||||
"valueString": "[ metric='' labels={} value=14151.331895396988 ]"
|
||||
},
|
||||
{
|
||||
"status": "firing",
|
||||
"labels": {
|
||||
"alertname": "High CPU usage",
|
||||
"team": "blue",
|
||||
"zone": "eu-1"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "The system has high CPU usage",
|
||||
"runbook_url": "https://myrunbook.com/runbook/1234",
|
||||
"summary": "This alert was triggered for zone eu-1"
|
||||
},
|
||||
"startsAt": "2021-10-12T09:56:03.157076+02:00",
|
||||
"endsAt": "0001-01-01T00:00:00Z",
|
||||
"generatorURL": "https://play.grafana.org/alerting/d1rdpdv7k/edit",
|
||||
"fingerprint": "bc97ff14869b13e3",
|
||||
"silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT1%2Cteam%3Dblue%2Czone%3Deu-1",
|
||||
"dashboardURL": "",
|
||||
"panelURL": "",
|
||||
"valueString": "[ metric='' labels={} value=47043.702386305304 ]"
|
||||
}
|
||||
],
|
||||
"groupLabels": {},
|
||||
"commonLabels": {
|
||||
"team": "blue"
|
||||
},
|
||||
"commonAnnotations": {},
|
||||
"externalURL": "https://play.grafana.org/",
|
||||
"version": "1",
|
||||
"groupKey": "{}:{}",
|
||||
"truncatedAlerts": 0,
|
||||
"title": "[FIRING:2] (blue)",
|
||||
"state": "alerting",
|
||||
"message": "Webhook test message."
|
||||
}
|
||||
|
|
@ -135,3 +135,78 @@ Someone is testing the alert notification within grafana.
|
|||
expected_message,
|
||||
content_type="application/x-www-form-urlencoded",
|
||||
)
|
||||
|
||||
def test_alert_new(self) -> None:
|
||||
expected_topic = "[RESOLVED:1]"
|
||||
expected_message = """
|
||||
:checkbox: **RESOLVED**
|
||||
|
||||
Webhook test message.
|
||||
|
||||
---
|
||||
**Alert 1**: TestAlert.
|
||||
|
||||
This alert was fired at <time:2022-08-31T05:54:04.52289368Z>.
|
||||
|
||||
This alert was resolved at <time:2022-08-31T10:30:00.52288431Z>.
|
||||
|
||||
Labels:
|
||||
- alertname: TestAlert
|
||||
- instance: Grafana
|
||||
|
||||
Annotations:
|
||||
- summary: Notification test
|
||||
|
||||
1 alert(s) truncated.
|
||||
""".strip()
|
||||
|
||||
self.check_webhook(
|
||||
"alert_new",
|
||||
expected_topic,
|
||||
expected_message,
|
||||
content_type="application/x-www-form-urlencoded",
|
||||
)
|
||||
|
||||
def test_alert_new_multiple(self) -> None:
|
||||
expected_topic = "[FIRING:2]"
|
||||
expected_message = """
|
||||
:alert: **FIRING**
|
||||
|
||||
Webhook test message.
|
||||
|
||||
---
|
||||
**Alert 1**: High memory usage.
|
||||
|
||||
This alert was fired at <time:2021-10-12T09:51:03.157076+02:00>.
|
||||
Labels:
|
||||
- alertname: High memory usage
|
||||
- team: blue
|
||||
- zone: us-1
|
||||
|
||||
Annotations:
|
||||
- description: The system has high memory usage
|
||||
- runbook_url: https://myrunbook.com/runbook/1234
|
||||
- summary: This alert was triggered for zone us-1
|
||||
|
||||
|
||||
---
|
||||
**Alert 2**: High CPU usage.
|
||||
|
||||
This alert was fired at <time:2021-10-12T09:56:03.157076+02:00>.
|
||||
Labels:
|
||||
- alertname: High CPU usage
|
||||
- team: blue
|
||||
- zone: eu-1
|
||||
|
||||
Annotations:
|
||||
- description: The system has high CPU usage
|
||||
- runbook_url: https://myrunbook.com/runbook/1234
|
||||
- summary: This alert was triggered for zone eu-1
|
||||
""".strip()
|
||||
|
||||
self.check_webhook(
|
||||
"alert_new_multiple",
|
||||
expected_topic,
|
||||
expected_message,
|
||||
content_type="application/x-www-form-urlencoded",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,15 +16,32 @@ from zerver.lib.validator import (
|
|||
from zerver.lib.webhooks.common import check_send_webhook_message
|
||||
from zerver.models import UserProfile
|
||||
|
||||
GRAFANA_TOPIC_TEMPLATE = "{alert_title}"
|
||||
OLD_TOPIC_TEMPLATE = "{alert_title}"
|
||||
|
||||
GRAFANA_ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n"
|
||||
ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n"
|
||||
|
||||
GRAFANA_MESSAGE_TEMPLATE = (
|
||||
"{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}"
|
||||
)
|
||||
OLD_MESSAGE_TEMPLATE = "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}"
|
||||
|
||||
ALL_EVENT_TYPES = ["ok", "pending", "alerting", "paused"]
|
||||
NEW_TOPIC_TEMPLATE = "[{alert_status}:{alert_count}]"
|
||||
|
||||
ALERT_HEADER_TEMPLATE = """\n---
|
||||
**Alert {count}**"""
|
||||
|
||||
START_TIME_TEMPLATE = "\n\nThis alert was fired at <time:{start_time}>.\n"
|
||||
|
||||
END_TIME_TEMPLATE = "\nThis alert was resolved at <time:{end_time}>.\n\n"
|
||||
|
||||
MESSAGE_LABELS_TEMPLATE = "Labels:\n{label_information}\n"
|
||||
|
||||
MESSAGE_ANNOTATIONS_TEMPLATE = "Annotations:\n{annotation_information}\n"
|
||||
|
||||
TRUNCATED_ALERTS_TEMPLATE = "{count} alert(s) truncated.\n"
|
||||
|
||||
LEGACY_EVENT_TYPES = ["ok", "pending", "alerting", "paused"]
|
||||
|
||||
NEW_EVENT_TYPES = ["firing", "resolved"]
|
||||
|
||||
ALL_EVENT_TYPES = LEGACY_EVENT_TYPES + NEW_EVENT_TYPES
|
||||
|
||||
|
||||
@webhook_view("Grafana", all_event_types=ALL_EVENT_TYPES)
|
||||
|
|
@ -35,52 +52,104 @@ def api_grafana_webhook(
|
|||
payload: WildValue = REQ(argument_type="body", converter=to_wild_value),
|
||||
) -> HttpResponse:
|
||||
|
||||
topic = GRAFANA_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string))
|
||||
# Grafana alerting system.
|
||||
if "alerts" in payload:
|
||||
status = payload["status"].tame(check_string_in(["firing", "resolved"]))
|
||||
alert_count = len(payload["alerts"])
|
||||
|
||||
eval_matches_text = ""
|
||||
if "evalMatches" in payload and payload["evalMatches"] is not None:
|
||||
for match in payload["evalMatches"]:
|
||||
eval_matches_text += "**{}:** {}\n".format(
|
||||
match["metric"].tame(check_string),
|
||||
match["value"].tame(check_none_or(check_union([check_int, check_float]))),
|
||||
topic = NEW_TOPIC_TEMPLATE.format(alert_status=status.upper(), alert_count=alert_count)
|
||||
|
||||
if status == "firing":
|
||||
body = ALERT_STATUS_TEMPLATE.format(alert_icon=":alert:", alert_state=status.upper())
|
||||
else:
|
||||
body = ALERT_STATUS_TEMPLATE.format(alert_icon=":checkbox:", alert_state=status.upper())
|
||||
|
||||
if payload["message"]:
|
||||
body += payload["message"].tame(check_string) + "\n"
|
||||
|
||||
for index, alert in enumerate(payload["alerts"], 1):
|
||||
body += ALERT_HEADER_TEMPLATE.format(count=index)
|
||||
|
||||
if "alertname" in alert["labels"] and alert["labels"]["alertname"]:
|
||||
body += ": " + alert["labels"]["alertname"].tame(check_string) + "."
|
||||
|
||||
body += START_TIME_TEMPLATE.format(start_time=alert["startsAt"].tame(check_string))
|
||||
|
||||
end_time = alert["endsAt"].tame(check_string)
|
||||
if end_time != "0001-01-01T00:00:00Z":
|
||||
body += END_TIME_TEMPLATE.format(end_time=end_time)
|
||||
|
||||
if alert["labels"]:
|
||||
label_information = ""
|
||||
for key, value in alert["labels"].items():
|
||||
label_information += "- " + key + ": " + value.tame(check_string) + "\n"
|
||||
body += MESSAGE_LABELS_TEMPLATE.format(label_information=label_information)
|
||||
|
||||
if alert["annotations"]:
|
||||
annotation_information = ""
|
||||
for key, value in alert["annotations"].items():
|
||||
annotation_information += "- " + key + ": " + value.tame(check_string) + "\n"
|
||||
body += MESSAGE_ANNOTATIONS_TEMPLATE.format(
|
||||
annotation_information=annotation_information
|
||||
)
|
||||
|
||||
if payload["truncatedAlerts"]:
|
||||
body += TRUNCATED_ALERTS_TEMPLATE.format(
|
||||
count=payload["truncatedAlerts"].tame(check_int)
|
||||
)
|
||||
|
||||
message_text = ""
|
||||
if "message" in payload:
|
||||
message_text = payload["message"].tame(check_string) + "\n\n"
|
||||
check_send_webhook_message(request, user_profile, topic, body, status)
|
||||
|
||||
state = payload["state"].tame(
|
||||
check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"])
|
||||
)
|
||||
if state == "alerting":
|
||||
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":alert:", alert_state=state.upper()
|
||||
)
|
||||
elif state == "ok":
|
||||
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":squared_ok:", alert_state=state.upper()
|
||||
)
|
||||
return json_success(request)
|
||||
|
||||
# Legacy Grafana alerts.
|
||||
else:
|
||||
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":info:", alert_state=state.upper()
|
||||
topic = OLD_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string))
|
||||
|
||||
eval_matches_text = ""
|
||||
if "evalMatches" in payload and payload["evalMatches"] is not None:
|
||||
for match in payload["evalMatches"]:
|
||||
eval_matches_text += "**{}:** {}\n".format(
|
||||
match["metric"].tame(check_string),
|
||||
match["value"].tame(check_none_or(check_union([check_int, check_float]))),
|
||||
)
|
||||
|
||||
message_text = ""
|
||||
if "message" in payload:
|
||||
message_text = payload["message"].tame(check_string) + "\n\n"
|
||||
|
||||
state = payload["state"].tame(
|
||||
check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"])
|
||||
)
|
||||
if state == "alerting":
|
||||
alert_status = ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":alert:", alert_state=state.upper()
|
||||
)
|
||||
elif state == "ok":
|
||||
alert_status = ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":squared_ok:", alert_state=state.upper()
|
||||
)
|
||||
else:
|
||||
alert_status = ALERT_STATUS_TEMPLATE.format(
|
||||
alert_icon=":info:", alert_state=state.upper()
|
||||
)
|
||||
|
||||
body = OLD_MESSAGE_TEMPLATE.format(
|
||||
alert_message=message_text,
|
||||
alert_status=alert_status,
|
||||
rule_name=payload["ruleName"].tame(check_string),
|
||||
rule_url=payload["ruleUrl"].tame(check_string),
|
||||
eval_matches=eval_matches_text,
|
||||
)
|
||||
|
||||
body = GRAFANA_MESSAGE_TEMPLATE.format(
|
||||
alert_message=message_text,
|
||||
alert_status=alert_status,
|
||||
rule_name=payload["ruleName"].tame(check_string),
|
||||
rule_url=payload["ruleUrl"].tame(check_string),
|
||||
eval_matches=eval_matches_text,
|
||||
)
|
||||
if "imageUrl" in payload:
|
||||
body += "\n[Click to view visualization]({visualization})".format(
|
||||
visualization=payload["imageUrl"].tame(check_string)
|
||||
)
|
||||
|
||||
if "imageUrl" in payload:
|
||||
body += "\n[Click to view visualization]({visualization})".format(
|
||||
visualization=payload["imageUrl"].tame(check_string)
|
||||
)
|
||||
body = body.strip()
|
||||
|
||||
body = body.strip()
|
||||
# send the message
|
||||
check_send_webhook_message(request, user_profile, topic, body, state)
|
||||
|
||||
# send the message
|
||||
check_send_webhook_message(request, user_profile, topic, body, state)
|
||||
|
||||
return json_success(request)
|
||||
return json_success(request)
|
||||
|
|
|
|||
Loading…
Reference in New Issue