From 977a043d03d77d329e9a87ade27b0a4b901f397b Mon Sep 17 00:00:00 2001 From: Hari Prashant Bhimaraju Date: Sat, 3 Sep 2022 15:31:08 +0530 Subject: [PATCH] grafana: Support notifications from Grafana Alerting. This commit adds support for Grafana's new alerting system, Grafana Alerting. The existing Grafana integration has been modified to detect the version of the notification through the structure of the payload body, since the the structure varies by version. Support for legacy alerting is been continued. Example fixtures have been added for Grafana Alerting's webhooks. Tests updated. --- .../webhooks/grafana/fixtures/alert_new.json | 40 +++++ .../grafana/fixtures/alert_new_multiple.json | 61 +++++++ zerver/webhooks/grafana/tests.py | 75 +++++++++ zerver/webhooks/grafana/view.py | 159 +++++++++++++----- 4 files changed, 290 insertions(+), 45 deletions(-) create mode 100644 zerver/webhooks/grafana/fixtures/alert_new.json create mode 100644 zerver/webhooks/grafana/fixtures/alert_new_multiple.json diff --git a/zerver/webhooks/grafana/fixtures/alert_new.json b/zerver/webhooks/grafana/fixtures/alert_new.json new file mode 100644 index 0000000000..fbad31ed79 --- /dev/null +++ b/zerver/webhooks/grafana/fixtures/alert_new.json @@ -0,0 +1,40 @@ +{ + "receiver": "", + "status": "resolved", + "alerts": [ + { + "status": "resolved", + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" + }, + "annotations": { + "summary": "Notification test" + }, + "startsAt": "2022-08-31T05:54:04.52289368Z", + "endsAt": "2022-08-31T10:30:00.52288431Z", + "generatorURL": "", + "fingerprint": "57c6d9296de2ad39", + "silenceURL": "https://zuliptestingwh2.grafana.net/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTestAlert&matcher=instance%3DGrafana", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='foo' labels={instance=bar} value=10 ]" + } + ], + "groupLabels": {}, + "commonLabels": { + "alertname": "TestAlert", + "instance": "Grafana" + }, + "commonAnnotations": { + "summary": "Notification test" + }, + "externalURL": "https://zuliptestingwh2.grafana.net/", + "version": "1", + "groupKey": "{alertname=\"TestAlert\", instance=\"Grafana\"}2022-08-31 05:54:04.52289368 +0000 UTC m=+42208.256292221", + "truncatedAlerts": 1, + "orgId": 1, + "title": "[RESOLVED:1] (TestAlert Grafana)", + "state": "alerting", + "message": "Webhook test message." +} diff --git a/zerver/webhooks/grafana/fixtures/alert_new_multiple.json b/zerver/webhooks/grafana/fixtures/alert_new_multiple.json new file mode 100644 index 0000000000..e57dff2f85 --- /dev/null +++ b/zerver/webhooks/grafana/fixtures/alert_new_multiple.json @@ -0,0 +1,61 @@ +{ + "receiver": "My Super Webhook", + "status": "firing", + "orgId": 1, + "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "High memory usage", + "team": "blue", + "zone": "us-1" + }, + "annotations": { + "description": "The system has high memory usage", + "runbook_url": "https://myrunbook.com/runbook/1234", + "summary": "This alert was triggered for zone us-1" + }, + "startsAt": "2021-10-12T09:51:03.157076+02:00", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "https://play.grafana.org/alerting/1afz29v7z/edit", + "fingerprint": "c6eadffa33fcdf37", + "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT2%2Cteam%3Dblue%2Czone%3Dus-1", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='' labels={} value=14151.331895396988 ]" + }, + { + "status": "firing", + "labels": { + "alertname": "High CPU usage", + "team": "blue", + "zone": "eu-1" + }, + "annotations": { + "description": "The system has high CPU usage", + "runbook_url": "https://myrunbook.com/runbook/1234", + "summary": "This alert was triggered for zone eu-1" + }, + "startsAt": "2021-10-12T09:56:03.157076+02:00", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "https://play.grafana.org/alerting/d1rdpdv7k/edit", + "fingerprint": "bc97ff14869b13e3", + "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT1%2Cteam%3Dblue%2Czone%3Deu-1", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='' labels={} value=47043.702386305304 ]" + } + ], + "groupLabels": {}, + "commonLabels": { + "team": "blue" + }, + "commonAnnotations": {}, + "externalURL": "https://play.grafana.org/", + "version": "1", + "groupKey": "{}:{}", + "truncatedAlerts": 0, + "title": "[FIRING:2] (blue)", + "state": "alerting", + "message": "Webhook test message." +} diff --git a/zerver/webhooks/grafana/tests.py b/zerver/webhooks/grafana/tests.py index 92e074229e..63f9c79864 100644 --- a/zerver/webhooks/grafana/tests.py +++ b/zerver/webhooks/grafana/tests.py @@ -135,3 +135,78 @@ Someone is testing the alert notification within grafana. expected_message, content_type="application/x-www-form-urlencoded", ) + + def test_alert_new(self) -> None: + expected_topic = "[RESOLVED:1]" + expected_message = """ +:checkbox: **RESOLVED** + +Webhook test message. + +--- +**Alert 1**: TestAlert. + +This alert was fired at . + +This alert was resolved at . + +Labels: +- alertname: TestAlert +- instance: Grafana + +Annotations: +- summary: Notification test + +1 alert(s) truncated. +""".strip() + + self.check_webhook( + "alert_new", + expected_topic, + expected_message, + content_type="application/x-www-form-urlencoded", + ) + + def test_alert_new_multiple(self) -> None: + expected_topic = "[FIRING:2]" + expected_message = """ +:alert: **FIRING** + +Webhook test message. + +--- +**Alert 1**: High memory usage. + +This alert was fired at . +Labels: +- alertname: High memory usage +- team: blue +- zone: us-1 + +Annotations: +- description: The system has high memory usage +- runbook_url: https://myrunbook.com/runbook/1234 +- summary: This alert was triggered for zone us-1 + + +--- +**Alert 2**: High CPU usage. + +This alert was fired at . +Labels: +- alertname: High CPU usage +- team: blue +- zone: eu-1 + +Annotations: +- description: The system has high CPU usage +- runbook_url: https://myrunbook.com/runbook/1234 +- summary: This alert was triggered for zone eu-1 +""".strip() + + self.check_webhook( + "alert_new_multiple", + expected_topic, + expected_message, + content_type="application/x-www-form-urlencoded", + ) diff --git a/zerver/webhooks/grafana/view.py b/zerver/webhooks/grafana/view.py index a38e42012c..6bb279c501 100644 --- a/zerver/webhooks/grafana/view.py +++ b/zerver/webhooks/grafana/view.py @@ -16,15 +16,32 @@ from zerver.lib.validator import ( from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile -GRAFANA_TOPIC_TEMPLATE = "{alert_title}" +OLD_TOPIC_TEMPLATE = "{alert_title}" -GRAFANA_ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n" +ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n" -GRAFANA_MESSAGE_TEMPLATE = ( - "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}" -) +OLD_MESSAGE_TEMPLATE = "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}" -ALL_EVENT_TYPES = ["ok", "pending", "alerting", "paused"] +NEW_TOPIC_TEMPLATE = "[{alert_status}:{alert_count}]" + +ALERT_HEADER_TEMPLATE = """\n--- +**Alert {count}**""" + +START_TIME_TEMPLATE = "\n\nThis alert was fired at .\n" + +END_TIME_TEMPLATE = "\nThis alert was resolved at .\n\n" + +MESSAGE_LABELS_TEMPLATE = "Labels:\n{label_information}\n" + +MESSAGE_ANNOTATIONS_TEMPLATE = "Annotations:\n{annotation_information}\n" + +TRUNCATED_ALERTS_TEMPLATE = "{count} alert(s) truncated.\n" + +LEGACY_EVENT_TYPES = ["ok", "pending", "alerting", "paused"] + +NEW_EVENT_TYPES = ["firing", "resolved"] + +ALL_EVENT_TYPES = LEGACY_EVENT_TYPES + NEW_EVENT_TYPES @webhook_view("Grafana", all_event_types=ALL_EVENT_TYPES) @@ -35,52 +52,104 @@ def api_grafana_webhook( payload: WildValue = REQ(argument_type="body", converter=to_wild_value), ) -> HttpResponse: - topic = GRAFANA_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string)) + # Grafana alerting system. + if "alerts" in payload: + status = payload["status"].tame(check_string_in(["firing", "resolved"])) + alert_count = len(payload["alerts"]) - eval_matches_text = "" - if "evalMatches" in payload and payload["evalMatches"] is not None: - for match in payload["evalMatches"]: - eval_matches_text += "**{}:** {}\n".format( - match["metric"].tame(check_string), - match["value"].tame(check_none_or(check_union([check_int, check_float]))), + topic = NEW_TOPIC_TEMPLATE.format(alert_status=status.upper(), alert_count=alert_count) + + if status == "firing": + body = ALERT_STATUS_TEMPLATE.format(alert_icon=":alert:", alert_state=status.upper()) + else: + body = ALERT_STATUS_TEMPLATE.format(alert_icon=":checkbox:", alert_state=status.upper()) + + if payload["message"]: + body += payload["message"].tame(check_string) + "\n" + + for index, alert in enumerate(payload["alerts"], 1): + body += ALERT_HEADER_TEMPLATE.format(count=index) + + if "alertname" in alert["labels"] and alert["labels"]["alertname"]: + body += ": " + alert["labels"]["alertname"].tame(check_string) + "." + + body += START_TIME_TEMPLATE.format(start_time=alert["startsAt"].tame(check_string)) + + end_time = alert["endsAt"].tame(check_string) + if end_time != "0001-01-01T00:00:00Z": + body += END_TIME_TEMPLATE.format(end_time=end_time) + + if alert["labels"]: + label_information = "" + for key, value in alert["labels"].items(): + label_information += "- " + key + ": " + value.tame(check_string) + "\n" + body += MESSAGE_LABELS_TEMPLATE.format(label_information=label_information) + + if alert["annotations"]: + annotation_information = "" + for key, value in alert["annotations"].items(): + annotation_information += "- " + key + ": " + value.tame(check_string) + "\n" + body += MESSAGE_ANNOTATIONS_TEMPLATE.format( + annotation_information=annotation_information + ) + + if payload["truncatedAlerts"]: + body += TRUNCATED_ALERTS_TEMPLATE.format( + count=payload["truncatedAlerts"].tame(check_int) ) - message_text = "" - if "message" in payload: - message_text = payload["message"].tame(check_string) + "\n\n" + check_send_webhook_message(request, user_profile, topic, body, status) - state = payload["state"].tame( - check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"]) - ) - if state == "alerting": - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":alert:", alert_state=state.upper() - ) - elif state == "ok": - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":squared_ok:", alert_state=state.upper() - ) + return json_success(request) + + # Legacy Grafana alerts. else: - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":info:", alert_state=state.upper() + topic = OLD_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string)) + + eval_matches_text = "" + if "evalMatches" in payload and payload["evalMatches"] is not None: + for match in payload["evalMatches"]: + eval_matches_text += "**{}:** {}\n".format( + match["metric"].tame(check_string), + match["value"].tame(check_none_or(check_union([check_int, check_float]))), + ) + + message_text = "" + if "message" in payload: + message_text = payload["message"].tame(check_string) + "\n\n" + + state = payload["state"].tame( + check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"]) + ) + if state == "alerting": + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":alert:", alert_state=state.upper() + ) + elif state == "ok": + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":squared_ok:", alert_state=state.upper() + ) + else: + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":info:", alert_state=state.upper() + ) + + body = OLD_MESSAGE_TEMPLATE.format( + alert_message=message_text, + alert_status=alert_status, + rule_name=payload["ruleName"].tame(check_string), + rule_url=payload["ruleUrl"].tame(check_string), + eval_matches=eval_matches_text, ) - body = GRAFANA_MESSAGE_TEMPLATE.format( - alert_message=message_text, - alert_status=alert_status, - rule_name=payload["ruleName"].tame(check_string), - rule_url=payload["ruleUrl"].tame(check_string), - eval_matches=eval_matches_text, - ) + if "imageUrl" in payload: + body += "\n[Click to view visualization]({visualization})".format( + visualization=payload["imageUrl"].tame(check_string) + ) - if "imageUrl" in payload: - body += "\n[Click to view visualization]({visualization})".format( - visualization=payload["imageUrl"].tame(check_string) - ) + body = body.strip() - body = body.strip() + # send the message + check_send_webhook_message(request, user_profile, topic, body, state) - # send the message - check_send_webhook_message(request, user_profile, topic, body, state) - - return json_success(request) + return json_success(request)