integrations: Enhance Grafana integration with alert state.

Having the alert state in the message body is useful when alert topics 
are not defined by alert description but encoded in the url.

E.g. in large environments having a topic for each alert [alerting] and [ok] would 
make it harder to properly track if an alert has been resolved.

When each alert is in a single topic, so far, the alert state has been missing.

This change will add the current alert state and a fitting icon in front
of the alert name.(Similar to the prometheus alertmanager integration)

The test cases have been amended to cover all possible alert states, even
though realistically grafana only fires the ok and alerting states via
webhook.
This commit is contained in:
AdamVB 2021-05-24 23:25:33 +02:00 committed by GitHub
parent 47689feb33
commit a9e73358bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 153 additions and 1 deletions

View File

@ -0,0 +1,20 @@
{
"dashboardId": 1,
"evalMatches": [
{
"value": 0,
"metric": "High value",
"tags": null
}
],
"imageUrl": "https://grafana.com/assets/img/blog/mixed_styles.png",
"message": "Someone is testing the alert notification within grafana.",
"orgId": 0,
"panelId": 1,
"ruleId": 0,
"ruleName": "Test rule",
"ruleUrl": "http://localhost:3000/",
"state": "ok",
"tags": {},
"title": "[Ok] Test notification"
}

View File

@ -0,0 +1,13 @@
{
"dashboardId": 1,
"imageUrl": "https://grafana.com/assets/img/blog/mixed_styles.png",
"message": "Someone is testing the alert notification within grafana.",
"orgId": 0,
"panelId": 1,
"ruleId": 0,
"ruleName": "Test rule",
"ruleUrl": "http://localhost:3000/",
"state": "paused",
"tags": {},
"title": "[Paused] Test notification"
}

View File

@ -0,0 +1,25 @@
{
"dashboardId": 1,
"evalMatches": [
{
"value": 100,
"metric": "High value",
"tags": null
},
{
"value": 200,
"metric": "Higher Value",
"tags": null
}
],
"imageUrl": "https://grafana.com/assets/img/blog/mixed_styles.png",
"message": "Someone is testing the alert notification within grafana.",
"orgId": 0,
"panelId": 1,
"ruleId": 0,
"ruleName": "Test rule",
"ruleUrl": "http://localhost:3000/",
"state": "pending",
"tags": {},
"title": "[Pending] Test notification"
}

View File

@ -10,6 +10,8 @@ class GrafanaHookTests(WebhookTestCase):
def test_alert(self) -> None: def test_alert(self) -> None:
expected_topic = "[Alerting] Test notification" expected_topic = "[Alerting] Test notification"
expected_message = """ expected_message = """
:alert: **ALERTING**
[Test rule](http://localhost:3000/) [Test rule](http://localhost:3000/)
Someone is testing the alert notification within grafana. Someone is testing the alert notification within grafana.
@ -31,6 +33,8 @@ Someone is testing the alert notification within grafana.
def test_no_data_alert(self) -> None: def test_no_data_alert(self) -> None:
expected_topic = "[Alerting] No Data alert" expected_topic = "[Alerting] No Data alert"
expected_message = """ expected_message = """
:alert: **ALERTING**
[No Data alert](http://localhost:3000/d/GG2qhR3Wz/alerttest?fullscreen&edit&tab=alert&panelId=6&orgId=1) [No Data alert](http://localhost:3000/d/GG2qhR3Wz/alerttest?fullscreen&edit&tab=alert&panelId=6&orgId=1)
The panel has no data. The panel has no data.
@ -48,6 +52,8 @@ The panel has no data.
def test_no_message_alert(self) -> None: def test_no_message_alert(self) -> None:
expected_topic = "[Alerting] No Message alert" expected_topic = "[Alerting] No Message alert"
expected_message = """ expected_message = """
:alert: **ALERTING**
[No Message alert](http://localhost:3000/d/GG2qhR3Wz/alerttest?fullscreen&edit&tab=alert&panelId=8&orgId=1) [No Message alert](http://localhost:3000/d/GG2qhR3Wz/alerttest?fullscreen&edit&tab=alert&panelId=8&orgId=1)
**A-series:** 21.573108436586445 **A-series:** 21.573108436586445
@ -60,3 +66,72 @@ The panel has no data.
expected_message, expected_message,
content_type="application/x-www-form-urlencoded", content_type="application/x-www-form-urlencoded",
) )
# Note: Include a test function per each distinct message condition your integration supports
def test_alert_ok(self) -> None:
expected_topic = "[Ok] Test notification"
expected_message = """
:squared_ok: **OK**
[Test rule](http://localhost:3000/)
Someone is testing the alert notification within grafana.
**High value:** 0
[Click to view visualization](https://grafana.com/assets/img/blog/mixed_styles.png)
""".strip()
# use fixture named helloworld_hello
self.check_webhook(
"alert_ok",
expected_topic,
expected_message,
content_type="application/x-www-form-urlencoded",
)
# Note: Include a test function per each distinct message condition your integration supports
def test_alert_paused(self) -> None:
expected_topic = "[Paused] Test notification"
expected_message = """
:info: **PAUSED**
[Test rule](http://localhost:3000/)
Someone is testing the alert notification within grafana.
[Click to view visualization](https://grafana.com/assets/img/blog/mixed_styles.png)
""".strip()
# use fixture named helloworld_hello
self.check_webhook(
"alert_paused",
expected_topic,
expected_message,
content_type="application/x-www-form-urlencoded",
)
# Note: Include a test function per each distinct message condition your integration supports
def test_alert_pending(self) -> None:
expected_topic = "[Pending] Test notification"
expected_message = """
:info: **PENDING**
[Test rule](http://localhost:3000/)
Someone is testing the alert notification within grafana.
**High value:** 100
**Higher Value:** 200
[Click to view visualization](https://grafana.com/assets/img/blog/mixed_styles.png)
""".strip()
# use fixture named helloworld_hello
self.check_webhook(
"alert_pending",
expected_topic,
expected_message,
content_type="application/x-www-form-urlencoded",
)

View File

@ -10,7 +10,11 @@ from zerver.models import UserProfile
GRAFANA_TOPIC_TEMPLATE = "{alert_title}" GRAFANA_TOPIC_TEMPLATE = "{alert_title}"
GRAFANA_MESSAGE_TEMPLATE = "[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}" GRAFANA_ALERT_STATUS_TEMPLATE = "{alert_icon} **{alert_state}**\n\n"
GRAFANA_MESSAGE_TEMPLATE = (
"{alert_status}[{rule_name}]({rule_url})\n\n{alert_message}{eval_matches}"
)
@webhook_view("Grafana") @webhook_view("Grafana")
@ -33,8 +37,23 @@ def api_grafana_webhook(
if payload.get("message") is not None: if payload.get("message") is not None:
message_text = payload["message"] + "\n\n" message_text = payload["message"] + "\n\n"
if payload.get("state") is not None:
if payload.get("state") == "alerting":
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
alert_icon=":alert:", alert_state=payload["state"].upper()
)
elif payload.get("state") == "ok":
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
alert_icon=":squared_ok:", alert_state=payload["state"].upper()
)
else:
alert_status = GRAFANA_ALERT_STATUS_TEMPLATE.format(
alert_icon=":info:", alert_state=payload["state"].upper()
)
body = GRAFANA_MESSAGE_TEMPLATE.format( body = GRAFANA_MESSAGE_TEMPLATE.format(
alert_message=message_text, alert_message=message_text,
alert_status=alert_status,
rule_name=payload["ruleName"], rule_name=payload["ruleName"],
rule_url=payload["ruleUrl"], rule_url=payload["ruleUrl"],
eval_matches=eval_matches_text, eval_matches=eval_matches_text,