grafana: Support notifications from Grafana Alerting.

This commit adds support for Grafana's new alerting system, Grafana Alerting. The existing Grafana integration has been modified to detect the version of the notification through the structure of the payload body, since the the structure varies by version. Support for legacy alerting is been continued. Example fixtures have been added for Grafana Alerting's webhooks. Tests updated.
2022-09-03 15:31:08 +05:30 · 2022-09-03 15:31:08 +05:30 · 977a043d03
parent 804eb15aa5
commit 977a043d03
4 changed files with 290 additions and 45 deletions
--- a/zerver/webhooks/grafana/fixtures/alert_new.json
+++ b/zerver/webhooks/grafana/fixtures/alert_new.json
@ -0,0 +1,40 @@
+{
+    "receiver": "",
+    "status": "resolved",
+    "alerts": [
+      {
+        "status": "resolved",
+        "labels": {
+          "alertname": "TestAlert",
+          "instance": "Grafana"
+        },
+        "annotations": {
+          "summary": "Notification test"
+        },
+        "startsAt": "2022-08-31T05:54:04.52289368Z",
+        "endsAt": "2022-08-31T10:30:00.52288431Z",
+        "generatorURL": "",
+        "fingerprint": "57c6d9296de2ad39",
+        "silenceURL": "https://zuliptestingwh2.grafana.net/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DTestAlert&matcher=instance%3DGrafana",
+        "dashboardURL": "",
+        "panelURL": "",
+        "valueString": "[ metric='foo' labels={instance=bar} value=10 ]"
+      }
+    ],
+    "groupLabels": {},
+    "commonLabels": {
+      "alertname": "TestAlert",
+      "instance": "Grafana"
+    },
+    "commonAnnotations": {
+      "summary": "Notification test"
+    },
+    "externalURL": "https://zuliptestingwh2.grafana.net/",
+    "version": "1",
+    "groupKey": "{alertname=\"TestAlert\", instance=\"Grafana\"}2022-08-31 05:54:04.52289368 +0000 UTC m=+42208.256292221",
+    "truncatedAlerts": 1,
+    "orgId": 1,
+    "title": "[RESOLVED:1]  (TestAlert Grafana)",
+    "state": "alerting",
+    "message": "Webhook test message."
+}
--- a/zerver/webhooks/grafana/fixtures/alert_new_multiple.json
+++ b/zerver/webhooks/grafana/fixtures/alert_new_multiple.json
@ -0,0 +1,61 @@
+{
+    "receiver": "My Super Webhook",
+    "status": "firing",
+    "orgId": 1,
+    "alerts": [
+      {
+        "status": "firing",
+        "labels": {
+          "alertname": "High memory usage",
+          "team": "blue",
+          "zone": "us-1"
+        },
+        "annotations": {
+          "description": "The system has high memory usage",
+          "runbook_url": "https://myrunbook.com/runbook/1234",
+          "summary": "This alert was triggered for zone us-1"
+        },
+        "startsAt": "2021-10-12T09:51:03.157076+02:00",
+        "endsAt": "0001-01-01T00:00:00Z",
+        "generatorURL": "https://play.grafana.org/alerting/1afz29v7z/edit",
+        "fingerprint": "c6eadffa33fcdf37",
+        "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT2%2Cteam%3Dblue%2Czone%3Dus-1",
+        "dashboardURL": "",
+        "panelURL": "",
+        "valueString": "[ metric='' labels={} value=14151.331895396988 ]"
+      },
+      {
+        "status": "firing",
+        "labels": {
+          "alertname": "High CPU usage",
+          "team": "blue",
+          "zone": "eu-1"
+        },
+        "annotations": {
+          "description": "The system has high CPU usage",
+          "runbook_url": "https://myrunbook.com/runbook/1234",
+          "summary": "This alert was triggered for zone eu-1"
+        },
+        "startsAt": "2021-10-12T09:56:03.157076+02:00",
+        "endsAt": "0001-01-01T00:00:00Z",
+        "generatorURL": "https://play.grafana.org/alerting/d1rdpdv7k/edit",
+        "fingerprint": "bc97ff14869b13e3",
+        "silenceURL": "https://play.grafana.org/alerting/silence/new?alertmanager=grafana&matchers=alertname%3DT1%2Cteam%3Dblue%2Czone%3Deu-1",
+        "dashboardURL": "",
+        "panelURL": "",
+        "valueString": "[ metric='' labels={} value=47043.702386305304 ]"
+      }
+    ],
+    "groupLabels": {},
+    "commonLabels": {
+      "team": "blue"
+    },
+    "commonAnnotations": {},
+    "externalURL": "https://play.grafana.org/",
+    "version": "1",
+    "groupKey": "{}:{}",
+    "truncatedAlerts": 0,
+    "title": "[FIRING:2]  (blue)",
+    "state": "alerting",
+    "message": "Webhook test message."
+}
--- a/zerver/webhooks/grafana/tests.py
+++ b/zerver/webhooks/grafana/tests.py
@ -135,3 +135,78 @@ Someone is testing the alert notification within grafana.
            expected_message,
            content_type="application/x-www-form-urlencoded",
        )
+
+    def test_alert_new(self) -> None:
+        expected_topic = "[RESOLVED:1]"
+        expected_message = """
+:checkbox: **RESOLVED**
+
+Webhook test message.
+
+---
+**Alert 1**: TestAlert.
+
+This alert was fired at <time:2022-08-31T05:54:04.52289368Z>.
+
+This alert was resolved at <time:2022-08-31T10:30:00.52288431Z>.
+
+Labels:
+- alertname: TestAlert
+- instance: Grafana
+
+Annotations:
+- summary: Notification test
+
+1 alert(s) truncated.
+""".strip()
+
+        self.check_webhook(
+            "alert_new",
+            expected_topic,
+            expected_message,
+            content_type="application/x-www-form-urlencoded",
+        )
+
+    def test_alert_new_multiple(self) -> None:
+        expected_topic = "[FIRING:2]"
+        expected_message = """
+:alert: **FIRING**
+
+Webhook test message.
+
+---
+**Alert 1**: High memory usage.
+
+This alert was fired at <time:2021-10-12T09:51:03.157076+02:00>.
+Labels:
+- alertname: High memory usage
+- team: blue
+- zone: us-1
+
+Annotations:
+- description: The system has high memory usage
+- runbook_url: https://myrunbook.com/runbook/1234
+- summary: This alert was triggered for zone us-1
+
+
+---
+**Alert 2**: High CPU usage.
+
+This alert was fired at <time:2021-10-12T09:56:03.157076+02:00>.
+Labels:
+- alertname: High CPU usage
+- team: blue
+- zone: eu-1
+
+Annotations:
+- description: The system has high CPU usage
+- runbook_url: https://myrunbook.com/runbook/1234
+- summary: This alert was triggered for zone eu-1
+""".strip()
+
+        self.check_webhook(
+            "alert_new_multiple",
+            expected_topic,
+            expected_message,
+            content_type="application/x-www-form-urlencoded",
+        )
--- a/zerver/webhooks/grafana/view.py
+++ b/zerver/webhooks/grafana/view.py
@ -16,15 +16,32 @@ from zerver.lib.validator import (
 from zerver.lib.webhooks.common import check_send_webhook_message
 from zerver.models import UserProfile

-GRAFANA_TOPIC_TEMPLATE = "{alert_title}"
+OLD_TOPIC_TEMPLATE = "{alert_title}"

-GRAFANA_ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n"
+ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n"

-GRAFANA_MESSAGE_TEMPLATE = (
-    "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}"
-)
+OLD_MESSAGE_TEMPLATE = "{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}"

-ALL_EVENT_TYPES = ["ok", "pending", "alerting", "paused"]
+NEW_TOPIC_TEMPLATE = "[{alert_status}:{alert_count}]"
+
+ALERT_HEADER_TEMPLATE = """\n---
+**Alert {count}**"""
+
+START_TIME_TEMPLATE = "\n\nThis alert was fired at <time:{start_time}>.\n"
+
+END_TIME_TEMPLATE = "\nThis alert was resolved at <time:{end_time}>.\n\n"
+
+MESSAGE_LABELS_TEMPLATE = "Labels:\n{label_information}\n"
+
+MESSAGE_ANNOTATIONS_TEMPLATE = "Annotations:\n{annotation_information}\n"
+
+TRUNCATED_ALERTS_TEMPLATE = "{count} alert(s) truncated.\n"
+
+LEGACY_EVENT_TYPES = ["ok", "pending", "alerting", "paused"]
+
+NEW_EVENT_TYPES = ["firing", "resolved"]
+
+ALL_EVENT_TYPES = LEGACY_EVENT_TYPES + NEW_EVENT_TYPES


@webhook_view("Grafana", all_event_types=ALL_EVENT_TYPES)
@ -35,7 +52,59 @@ def api_grafana_webhook(
    payload: WildValue = REQ(argument_type="body", converter=to_wild_value),
 ) -> HttpResponse:

-    topic = GRAFANA_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string))
+    # Grafana alerting system.
+    if "alerts" in payload:
+        status = payload["status"].tame(check_string_in(["firing", "resolved"]))
+        alert_count = len(payload["alerts"])
+
+        topic = NEW_TOPIC_TEMPLATE.format(alert_status=status.upper(), alert_count=alert_count)
+
+        if status == "firing":
+            body = ALERT_STATUS_TEMPLATE.format(alert_icon=":alert:", alert_state=status.upper())
+        else:
+            body = ALERT_STATUS_TEMPLATE.format(alert_icon=":checkbox:", alert_state=status.upper())
+
+        if payload["message"]:
+            body += payload["message"].tame(check_string) + "\n"
+
+        for index, alert in enumerate(payload["alerts"], 1):
+            body += ALERT_HEADER_TEMPLATE.format(count=index)
+
+            if "alertname" in alert["labels"] and alert["labels"]["alertname"]:
+                body += ": " + alert["labels"]["alertname"].tame(check_string) + "."
+
+            body += START_TIME_TEMPLATE.format(start_time=alert["startsAt"].tame(check_string))
+
+            end_time = alert["endsAt"].tame(check_string)
+            if end_time != "0001-01-01T00:00:00Z":
+                body += END_TIME_TEMPLATE.format(end_time=end_time)
+
+            if alert["labels"]:
+                label_information = ""
+                for key, value in alert["labels"].items():
+                    label_information += "- " + key + ": " + value.tame(check_string) + "\n"
+                body += MESSAGE_LABELS_TEMPLATE.format(label_information=label_information)
+
+            if alert["annotations"]:
+                annotation_information = ""
+                for key, value in alert["annotations"].items():
+                    annotation_information += "- " + key + ": " + value.tame(check_string) + "\n"
+                body += MESSAGE_ANNOTATIONS_TEMPLATE.format(
+                    annotation_information=annotation_information
+                )
+
+        if payload["truncatedAlerts"]:
+            body += TRUNCATED_ALERTS_TEMPLATE.format(
+                count=payload["truncatedAlerts"].tame(check_int)
+            )
+
+        check_send_webhook_message(request, user_profile, topic, body, status)
+
+        return json_success(request)
+
+    # Legacy Grafana alerts.
+    else:
+        topic = OLD_TOPIC_TEMPLATE.format(alert_title=payload["title"].tame(check_string))

        eval_matches_text = ""
        if "evalMatches" in payload and payload["evalMatches"] is not None:
@ -53,19 +122,19 @@ def api_grafana_webhook(
            check_string_in(["no_data", "paused", "alerting", "ok", "pending", "unknown"])
        )
        if state == "alerting":
-        alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
+            alert_status = ALERT_STATUS_TEMPLATE.format(
                alert_icon=":alert:", alert_state=state.upper()
            )
        elif state == "ok":
-        alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
+            alert_status = ALERT_STATUS_TEMPLATE.format(
                alert_icon=":squared_ok:", alert_state=state.upper()
            )
        else:
-        alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
+            alert_status = ALERT_STATUS_TEMPLATE.format(
                alert_icon=":info:", alert_state=state.upper()
            )

-    body = GRAFANA_MESSAGE_TEMPLATE.format(
+        body = OLD_MESSAGE_TEMPLATE.format(
            alert_message=message_text,
            alert_status=alert_status,
            rule_name=payload["ruleName"].tame(check_string),