webhooks: Add a webhook capable of parsing Slack payloads.

This adds a webhook that can be used to interpret standard Slack
payloads. Since there are a ton of existing Slack integrations out
there, having a webhook which can accept standard Slack payloads can
significantly ease transition pains. Obviously this can't do everything
that Slack payloads can (particularly WRT their widgets/interactions),
but we can ingest text and parse out multi-block payloads into a message
relatively reasonably.

zerver/lib/integrations.py

@@ -361,6 +361,12 @@ WEBHOOK_INTEGRATIONS: List[WebhookIntegration] = [
     WebhookIntegration('reviewboard', ['version-control'], display_name="ReviewBoard"),
     WebhookIntegration('semaphore', ['continuous-integration', 'deployment']),
     WebhookIntegration('sentry', ['monitoring']),
+    WebhookIntegration(
+        'slack_incoming',
+        ['communication'],
+        display_name="Slack-compatible webhook",
+        logo='images/integrations/logos/slack.svg',
+    ),
     WebhookIntegration('slack', ['communication']),
     WebhookIntegration('solano', ['continuous-integration'], display_name='Solano Labs'),
     WebhookIntegration('splunk', ['monitoring'], display_name='Splunk'),

zerver/webhooks/slack_incoming/doc.md

@@ -0,0 +1,9 @@
+Basic support for Slack-compatible webhooks.
+
+1. {!create-stream.md!}
+
+1. {!create-bot-construct-url-indented.md!}
+
+1. Use your new webhook URL any place that you would use a Slack webhook.
+
+{!congrats.md!}

zerver/webhooks/slack_incoming/fixtures/actions.json

@@ -0,0 +1,48 @@
+{
+  "channel": "C1H9RESGL",
+  "blocks": [
+    {
+      "type": "section",
+      "text": {
+        "type": "mrkdwn",
+        "text": "Danny Torrence left the following review for your property:"
+      }
+    },
+    {
+      "type": "section",
+      "block_id": "section567",
+      "text": {
+        "type": "mrkdwn",
+        "text": "<https://google.com|Overlook Hotel> \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s."
+      },
+      "accessory": {
+        "type": "image",
+        "image_url": "https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg",
+        "alt_text": "Haunted hotel image"
+      }
+    },
+    {
+      "type": "section",
+      "block_id": "section789",
+      "fields": [
+        {
+          "type": "mrkdwn",
+          "text": "*Average Rating*\n1.0"
+        }
+      ]
+    },
+    {
+      "type": "actions",
+      "elements": [
+        {
+          "type": "button",
+            "text": {
+                "type": "plain_text",
+                "text": "Reply to review",
+                "emoji": false
+            }
+        }
+      ]
+    }
+  ]
+}

zerver/webhooks/slack_incoming/fixtures/attachment.json

@@ -0,0 +1,20 @@
+{
+    "channel": "#prometheus-alerts",
+    "username": "Alertmanager",
+    "attachments": [
+        {
+            "title": "[FIRING:2] InstanceDown for api-server (env=\"prod\", severity=\"critical\")",
+            "title_link": "https://alertmanager.local//#/alerts?receiver=default",
+            "text": ":chart_with_upwards_trend: *<http://generator.local/1|Graph>*   :notebook: *<https://runbook.local/1|Runbook>*\n\n*Alert details*:\n*Alert:* api-server down - `critical`\n*Description:* api-server at 1.2.3.4:8080 couldn't be scraped *Details:*\n   • *alertname:* `InstanceDown`\n   • *env:* `prod`\n   • *instance:* `1.2.3.4:8080`\n   • *job:* `api-server`\n   • *severity:* `critical`\n\n*Alert:* api-server down - `critical`\n*Description:* api-server at 1.2.3.4:8081 couldn't be scraped *Details:*\n   • *alertname:* `InstanceDown`\n   • *env:* `prod`\n   • *instance:* `1.2.3.4:8081`\n   • *job:* `api-server`\n   • *severity:* `critical`\n  \n",
+            "fallback": "[FIRING:2] InstanceDown api-server (prod critical) | https://alertmanager.local//#/alerts?receiver=default",
+            "callback_id": "",
+            "footer": "",
+            "color": "danger",
+            "mrkdwn_in": [
+                "fallback",
+                "pretext",
+                "text"
+            ]
+        }
+    ]
+}

zerver/webhooks/slack_incoming/fixtures/blocks.json

@@ -0,0 +1,35 @@
+{
+    "text": "Danny Torrence left a 1 star review for your property.",
+    "blocks": [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "Danny Torrence left the following review for your property:"
+            }
+        },
+        {
+            "type": "section",
+            "block_id": "section567",
+            "text": {
+                "type": "mrkdwn",
+                "text": "<https://example.com|Overlook Hotel> \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s."
+            },
+            "accessory": {
+                "type": "image",
+                "image_url": "https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg",
+                "alt_text": "Haunted hotel image"
+            }
+        },
+        {
+            "type": "section",
+            "block_id": "section789",
+            "fields": [
+                {
+                    "type": "mrkdwn",
+                    "text": "*Average Rating*\n1.0"
+                }
+            ]
+        }
+    ]
+}

zerver/webhooks/slack_incoming/fixtures/text.json

@@ -0,0 +1,3 @@
+{
+    "text": "Hello, world."
+}

zerver/webhooks/slack_incoming/fixtures/urlencoded_text.txt

@@ -0,0 +1 @@
+payload=%7B%22username%22%3A%22DeployBot%22%2C%22icon_url%22%3A%22https%3A%2F%2Fraw.githubusercontent.com%2Fphallstrom%2Fslackistrano%2Fmaster%2Fimages%2Fslackistrano.png%22%2C%22icon_emoji%22%3A%22%3Azap%3A%22%2C%22text%22%3A%22chris+has+started+deploying+project+tag+v0.0.2rc10+to+staging%22%2C%22channel%22%3A%22%23devops%22%7D

zerver/webhooks/slack_incoming/tests.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+from zerver.lib.test_classes import WebhookTestCase
+
+
+class SlackIncomingHookTests(WebhookTestCase):
+    STREAM_NAME = 'slack_incoming'
+    URL_TEMPLATE = "/api/v1/external/slack_incoming?&api_key={api_key}&stream={stream}"
+    FIXTURE_DIR_NAME = 'slack_incoming'
+
+    def test_message(self) -> None:
+        expected_topic = "(no topic)"
+        expected_message = """
+Hello, world.
+""".strip()
+
+        self.send_and_test_stream_message(
+            'text',
+            expected_topic,
+            expected_message
+        )
+
+    def test_message_as_www_urlencoded(self) -> None:
+        expected_topic = "devops"
+        expected_message = """
+:zap: chris has started deploying project tag v0.0.2rc10 to staging
+""".strip()
+
+        self.send_and_test_stream_message(
+            'urlencoded_text',
+            expected_topic,
+            expected_message,
+            content_type="application/x-www-form-urlencoded"
+        )
+
+    def test_message_with_actions(self) -> None:
+        expected_topic = "C1H9RESGL"
+        expected_message = """
+Danny Torrence left the following review for your property:
+
+[Overlook Hotel](https://google.com) \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s.
+[Haunted hotel image](https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg)
+""".strip()
+
+        self.send_and_test_stream_message(
+            'actions',
+            expected_topic,
+            expected_message
+        )
+
+    def test_message_with_blocks(self) -> None:
+        expected_topic = "(no topic)"
+        expected_message = """
+Danny Torrence left the following review for your property:
+
+[Overlook Hotel](https://example.com) \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s.
+[Haunted hotel image](https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg)
+""".strip()
+
+        self.send_and_test_stream_message(
+            'blocks',
+            expected_topic,
+            expected_message
+        )
+
+    def test_message_with_attachment(self) -> None:
+        expected_topic = "prometheus-alerts"
+        expected_message = """
+[[FIRING:2] InstanceDown for api-server (env="prod", severity="critical")](https://alertmanager.local//#/alerts?receiver=default)
+:chart_with_upwards_trend: **[Graph](http://generator.local/1)**   :notebook: **[Runbook](https://runbook.local/1)**
+
+**Alert details**:
+**Alert:** api-server down - `critical`
+**Description:** api-server at 1.2.3.4:8080 couldn't be scraped **Details:**
+   • **alertname:** `InstanceDown`
+   • **env:** `prod`
+   • **instance:** `1.2.3.4:8080`
+   • **job:** `api-server`
+   • **severity:** `critical`
+
+**Alert:** api-server down - `critical`
+**Description:** api-server at 1.2.3.4:8081 couldn't be scraped **Details:**
+   • **alertname:** `InstanceDown`
+   • **env:** `prod`
+   • **instance:** `1.2.3.4:8081`
+   • **job:** `api-server`
+   • **severity:** `critical`
+""".strip()
+
+        self.send_and_test_stream_message(
+            'attachment',
+            expected_topic,
+            expected_message
+        )
+
+    def get_body(self, fixture_name: str) -> str:
+        if "urlencoded" in fixture_name:
+            file_type = "txt"
+        else:
+            file_type = "json"
+        return self.webhook_fixture_data("slack_incoming", fixture_name, file_type=file_type)

zerver/webhooks/slack_incoming/view.py

@@ -0,0 +1,99 @@
+# Webhooks for external integrations.
+from typing import Any, Dict, Optional
+
+from django.http import HttpRequest, HttpResponse
+
+from zerver.decorator import api_key_only_webhook_view
+from zerver.lib.request import REQ, has_request_variables
+from zerver.lib.response import json_success
+from zerver.lib.webhooks.common import check_send_webhook_message
+from zerver.lib.exceptions import InvalidJSONError
+from django.utils.translation import ugettext as _
+from zerver.models import UserProfile
+import re
+import ujson
+
+@api_key_only_webhook_view('SlackIncoming')
+@has_request_variables
+def api_slack_incoming_webhook(request: HttpRequest, user_profile: UserProfile,
+                               user_specified_topic: Optional[str]=REQ("topic", default=None),
+                               payload: Optional[Dict[str, Any]] = REQ(
+                                   'payload',
+                                   converter=ujson.loads,
+                                   default=None)) -> HttpResponse:
+
+    # Slack accepts webhook payloads as payload="encoded json" as
+    # application/x-www-form-urlencoded, as well as in the body as
+    # application/json. We use has_request_variables to try to get
+    # the form encoded version, and parse the body out ourselves if
+    # # we were given JSON.
+    if payload is None:
+        try:
+            payload = ujson.loads(request.body)
+        except ValueError:  # nocoverage
+            raise InvalidJSONError(_("Malformed JSON"))
+
+    if user_specified_topic is None and "channel" in payload:
+        user_specified_topic = re.sub("^[@#]", "", payload["channel"])
+
+    if user_specified_topic is None:
+        user_specified_topic = "(no topic)"
+
+    body = ""
+
+    if "blocks" in payload:
+        for block in payload["blocks"]:
+            body = add_block(block, body)
+
+    if "attachments" in payload:
+        for attachment in payload["attachments"]:
+            body = add_attachment(attachment, body)
+
+    if body == "" and "text" in payload:
+        body += payload["text"]
+        if "icon_emoji" in payload and payload["icon_emoji"] is not None:
+            body = "{} {}".format(payload["icon_emoji"], body)
+
+    if body != "":
+        body = replace_formatting(replace_links(body).strip())
+        check_send_webhook_message(request, user_profile, user_specified_topic, body)
+    return json_success()
+
+
+def add_block(block: Dict[str, Any], body: str) -> str:
+    block_type = block.get("type", None)
+    if block_type == "section":
+        if "text" in block:
+            text = block["text"]
+            while type(text) == dict:  # handle stuff like block["text"]["text"]
+                text = text["text"]
+            body += "\n\n{}".format(text)
+
+        if "accessory" in block:
+            accessory = block["accessory"]
+            accessory_type = accessory["type"]
+            if accessory_type == "image":
+                # This should become ![text](url) once proper Markdown images are supported
+                body += "\n[{alt_text}]({image_url})".format(**accessory)
+
+    return body
+
+def add_attachment(attachment: Dict[str, Any], body: str) -> str:
+    attachment_body = ""
+    if "title" in attachment and "title_link" in attachment:
+        attachment_body += "[{title}]({title_link})\n".format(**attachment)
+    if "text" in attachment:
+        attachment_body += attachment["text"]
+
+    return body + attachment_body
+
+def replace_links(text: str) -> str:
+    return re.sub(r"<(\w+?:\/\/.*?)\|(.*?)>", r"[\2](\1)", text)
+
+def replace_formatting(text: str) -> str:
+    # Slack uses *text* for bold, whereas Zulip interprets that as italics
+    text = re.sub(r'([^\w])\*(?!\s+)([^\*^\n]+)(?<!\s)\*([^\w])', r"\1**\2**\3", text)
+
+    # Slack uses _text_ for emphasis, whereas Zulip interprets that as nothing
+    text = re.sub(r"([^\w])[_](?!\s+)([^\_\^\n]+)(?<!\s)[_]([^\w])", r"\1**\2**\3", text)
+    return text