diff --git a/zerver/webhooks/grafana/fixtures/alert_new.json b/zerver/webhooks/grafana/fixtures/alert_new.json new file mode 100644 index 0000000000..fbad31ed79 --- /dev/null +++ b/zerver/webhooks/grafana/fixtures/alert_new.json @@ -0,0 +1,40 @@ +{ + "receiver": "", + "status": "resolved", + "alerts": [ + { + "status": "resolved", + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" + }, + "annotations": { + "summary": "Notification test" + }, + "startsAt": "2022-08-31T05:54:04.52289368Z", + "endsAt": "2022-08-31T10:30:00.52288431Z", + "generatorURL": "", + "fingerprint": "57c6d9296de2ad39", + "silenceURL": "https://zuliptestingwh2.grafana.net/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTestAlert&matcher=instance%3DGrafana", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='foo' labels={instance=bar} value=10 ]" + } + ], + "groupLabels": {}, + "commonLabels": { + "alertname": "TestAlert", + "instance": "Grafana" + }, + "commonAnnotations": { + "summary": "Notification test" + }, + "externalURL": "https://zuliptestingwh2.grafana.net/", + "version": "1", + "groupKey": "{alertname=\"TestAlert\", instance=\"Grafana\"}2022-08-31 05:54:04.52289368 +0000 UTC m=+42208.256292221", + "truncatedAlerts": 1, + "orgId": 1, + "title": "[RESOLVED:1] (TestAlert Grafana)", + "state": "alerting", + "message": "Webhook test message." +} diff --git a/zerver/webhooks/grafana/fixtures/alert_new_multiple.json b/zerver/webhooks/grafana/fixtures/alert_new_multiple.json new file mode 100644 index 0000000000..e57dff2f85 --- /dev/null +++ b/zerver/webhooks/grafana/fixtures/alert_new_multiple.json @@ -0,0 +1,61 @@ +{ + "receiver": "My Super Webhook", + "status": "firing", + "orgId": 1, + "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "High memory usage", + "team": "blue", + "zone": "us-1" + }, + "annotations": { + "description": "The system has high memory usage", + "runbook_url": "https://myrunbook.com/runbook/1234", + "summary": "This alert was triggered for zone us-1" + }, + "startsAt": "2021-10-12T09:51:03.157076+02:00", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "https://play.grafana.org/alerting/1afz29v7z/edit", + "fingerprint": "c6eadffa33fcdf37", + "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT2%2Cteam%3Dblue%2Czone%3Dus-1", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='' labels={} value=14151.331895396988 ]" + }, + { + "status": "firing", + "labels": { + "alertname": "High CPU usage", + "team": "blue", + "zone": "eu-1" + }, + "annotations": { + "description": "The system has high CPU usage", + "runbook_url": "https://myrunbook.com/runbook/1234", + "summary": "This alert was triggered for zone eu-1" + }, + "startsAt": "2021-10-12T09:56:03.157076+02:00", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "https://play.grafana.org/alerting/d1rdpdv7k/edit", + "fingerprint": "bc97ff14869b13e3", + "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT1%2Cteam%3Dblue%2Czone%3Deu-1", + "dashboardURL": "", + "panelURL": "", + "valueString": "[ metric='' labels={} value=47043.702386305304 ]" + } + ], + "groupLabels": {}, + "commonLabels": { + "team": "blue" + }, + "commonAnnotations": {}, + "externalURL": "https://play.grafana.org/", + "version": "1", + "groupKey": "{}:{}", + "truncatedAlerts": 0, + "title": "[FIRING:2] (blue)", + "state": "alerting", + "message": "Webhook test message." +} diff --git a/zerver/webhooks/grafana/tests.py b/zerver/webhooks/grafana/tests.py index 92e074229e..63f9c79864 100644 --- a/zerver/webhooks/grafana/tests.py +++ b/zerver/webhooks/grafana/tests.py @@ -135,3 +135,78 @@ Someone is testing the alert notification within grafana. expected_message, content_type="application/x-www-form-urlencoded", ) + + def test_alert_new(self) -> None: + expected_topic = "[RESOLVED:1]" + expected_message = """ +:checkbox: **RESOLVED** + +Webhook test message. + +--- +**Alert 1**: TestAlert. + +This alert was fired at . + +This alert was resolved at . + +Labels: +- alertname: TestAlert +- instance: Grafana + +Annotations: +- summary: Notification test + +1 alert(s) truncated. +""".strip() + + self.check_webhook( + "alert_new", + expected_topic, + expected_message, + content_type="application/x-www-form-urlencoded", + ) + + def test_alert_new_multiple(self) -> None: + expected_topic = "[FIRING:2]" + expected_message = """ +:alert: **FIRING** + +Webhook test message. + +--- +**Alert 1**: High memory usage. + +This alert was fired at . +Labels: +- alertname: High memory usage +- team: blue +- zone: us-1 + +Annotations: +- description: The system has high memory usage +- runbook_url: https://myrunbook.com/runbook/1234 +- summary: This alert was triggered for zone us-1 + + +--- +**Alert 2**: High CPU usage. + +This alert was fired at . +Labels: +- alertname: High CPU usage +- team: blue +- zone: eu-1 + +Annotations: +- description: The system has high CPU usage +- runbook_url: https://myrunbook.com/runbook/1234 +- summary: This alert was triggered for zone eu-1 +""".strip() + + self.check_webhook( + "alert_new_multiple", + expected_topic, + expected_message, + content_type="application/x-www-form-urlencoded", + ) diff --git a/zerver/webhooks/grafana/view.py b/zerver/webhooks/grafana/view.py index a38e42012c..6bb279c501 100644 --- a/zerver/webhooks/grafana/view.py +++ b/zerver/webhooks/grafana/view.py @@ -16,15 +16,32 @@ from zerver.lib.validator import ( from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile -GRAFANA_TOPIC_TEMPLATE = "{alert_title}" +OLD_TOPIC_TEMPLATE = "{alert_title}" -GRAFANA_ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n" +ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n" -GRAFANA_MESSAGE_TEMPLATE = ( - "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}" -) +OLD_MESSAGE_TEMPLATE = "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}" -ALL_EVENT_TYPES = ["ok", "pending", "alerting", "paused"] +NEW_TOPIC_TEMPLATE = "[{alert_status}:{alert_count}]" + +ALERT_HEADER_TEMPLATE = """\n--- +**Alert {count}**""" + +START_TIME_TEMPLATE = "\n\nThis alert was fired at .\n" + +END_TIME_TEMPLATE = "\nThis alert was resolved at .\n\n" + +MESSAGE_LABELS_TEMPLATE = "Labels:\n{label_information}\n" + +MESSAGE_ANNOTATIONS_TEMPLATE = "Annotations:\n{annotation_information}\n" + +TRUNCATED_ALERTS_TEMPLATE = "{count} alert(s) truncated.\n" + +LEGACY_EVENT_TYPES = ["ok", "pending", "alerting", "paused"] + +NEW_EVENT_TYPES = ["firing", "resolved"] + +ALL_EVENT_TYPES = LEGACY_EVENT_TYPES + NEW_EVENT_TYPES @webhook_view("Grafana", all_event_types=ALL_EVENT_TYPES) @@ -35,52 +52,104 @@ def api_grafana_webhook( payload: WildValue = REQ(argument_type="body", converter=to_wild_value), ) -> HttpResponse: - topic = GRAFANA_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string)) + # Grafana alerting system. + if "alerts" in payload: + status = payload["status"].tame(check_string_in(["firing", "resolved"])) + alert_count = len(payload["alerts"]) - eval_matches_text = "" - if "evalMatches" in payload and payload["evalMatches"] is not None: - for match in payload["evalMatches"]: - eval_matches_text += "**{}:** {}\n".format( - match["metric"].tame(check_string), - match["value"].tame(check_none_or(check_union([check_int, check_float]))), + topic = NEW_TOPIC_TEMPLATE.format(alert_status=status.upper(), alert_count=alert_count) + + if status == "firing": + body = ALERT_STATUS_TEMPLATE.format(alert_icon=":alert:", alert_state=status.upper()) + else: + body = ALERT_STATUS_TEMPLATE.format(alert_icon=":checkbox:", alert_state=status.upper()) + + if payload["message"]: + body += payload["message"].tame(check_string) + "\n" + + for index, alert in enumerate(payload["alerts"], 1): + body += ALERT_HEADER_TEMPLATE.format(count=index) + + if "alertname" in alert["labels"] and alert["labels"]["alertname"]: + body += ": " + alert["labels"]["alertname"].tame(check_string) + "." + + body += START_TIME_TEMPLATE.format(start_time=alert["startsAt"].tame(check_string)) + + end_time = alert["endsAt"].tame(check_string) + if end_time != "0001-01-01T00:00:00Z": + body += END_TIME_TEMPLATE.format(end_time=end_time) + + if alert["labels"]: + label_information = "" + for key, value in alert["labels"].items(): + label_information += "- " + key + ": " + value.tame(check_string) + "\n" + body += MESSAGE_LABELS_TEMPLATE.format(label_information=label_information) + + if alert["annotations"]: + annotation_information = "" + for key, value in alert["annotations"].items(): + annotation_information += "- " + key + ": " + value.tame(check_string) + "\n" + body += MESSAGE_ANNOTATIONS_TEMPLATE.format( + annotation_information=annotation_information + ) + + if payload["truncatedAlerts"]: + body += TRUNCATED_ALERTS_TEMPLATE.format( + count=payload["truncatedAlerts"].tame(check_int) ) - message_text = "" - if "message" in payload: - message_text = payload["message"].tame(check_string) + "\n\n" + check_send_webhook_message(request, user_profile, topic, body, status) - state = payload["state"].tame( - check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"]) - ) - if state == "alerting": - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":alert:", alert_state=state.upper() - ) - elif state == "ok": - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":squared_ok:", alert_state=state.upper() - ) + return json_success(request) + + # Legacy Grafana alerts. else: - alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format( - alert_icon=":info:", alert_state=state.upper() + topic = OLD_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string)) + + eval_matches_text = "" + if "evalMatches" in payload and payload["evalMatches"] is not None: + for match in payload["evalMatches"]: + eval_matches_text += "**{}:** {}\n".format( + match["metric"].tame(check_string), + match["value"].tame(check_none_or(check_union([check_int, check_float]))), + ) + + message_text = "" + if "message" in payload: + message_text = payload["message"].tame(check_string) + "\n\n" + + state = payload["state"].tame( + check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"]) + ) + if state == "alerting": + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":alert:", alert_state=state.upper() + ) + elif state == "ok": + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":squared_ok:", alert_state=state.upper() + ) + else: + alert_status = ALERT_STATUS_TEMPLATE.format( + alert_icon=":info:", alert_state=state.upper() + ) + + body = OLD_MESSAGE_TEMPLATE.format( + alert_message=message_text, + alert_status=alert_status, + rule_name=payload["ruleName"].tame(check_string), + rule_url=payload["ruleUrl"].tame(check_string), + eval_matches=eval_matches_text, ) - body = GRAFANA_MESSAGE_TEMPLATE.format( - alert_message=message_text, - alert_status=alert_status, - rule_name=payload["ruleName"].tame(check_string), - rule_url=payload["ruleUrl"].tame(check_string), - eval_matches=eval_matches_text, - ) + if "imageUrl" in payload: + body += "\n[Click to view visualization]({visualization})".format( + visualization=payload["imageUrl"].tame(check_string) + ) - if "imageUrl" in payload: - body += "\n[Click to view visualization]({visualization})".format( - visualization=payload["imageUrl"].tame(check_string) - ) + body = body.strip() - body = body.strip() + # send the message + check_send_webhook_message(request, user_profile, topic, body, state) - # send the message - check_send_webhook_message(request, user_profile, topic, body, state) - - return json_success(request) + return json_success(request)