tools: Add script to trigger webhook notification using fixtures.

When creating a webhook integration or creating a new one, it is a pain to
create or update the screenshots in the documentation. This commit adds a
tool that can trigger a sample notification for the webhook using a fixture,
that is likely already written for the tests.

Currently, the developer needs to take a screenshot manually, but this could
be automated using puppeteer or something like that.

Also, the tool does not support webhooks with basic auth, and only supports
webhooks that use json fixtures. These can be fixed in subsequent commits.

requirements/dev.in

@@ -64,3 +64,6 @@ https://github.com/zulip/zulint/archive/aaed679f1ad38b230090eadd3870b7682500f60c
 importlib-metadata
 # built-in python > 3.6 needed by cfn-lint
 importlib-resources
+
+# Needed for using integration logo svg files as bot avatars
+cairosvg

requirements/dev.txt

@@ -81,6 +81,12 @@ cachetools==4.0.0 \
     --hash=sha256:9a52dd97a85f257f4e4127f15818e71a0c7899f121b34591fcc1173ea79a0198 \
     --hash=sha256:b304586d357c43221856be51d73387f93e2a961598a9b6b6670664746f3b6c6c \
     # via premailer
+cairocffi==1.1.0 \
+    --hash=sha256:f1c0c5878f74ac9ccb5d48b2601fcc75390c881ce476e79f4cfedd288b1b05db \
+    # via cairosvg
+cairosvg==2.4.2 \
+    --hash=sha256:4e668f96653326780036ebb0a9ff2bb59a8443d7bcfc51a14aab77b57a8e67ad \
+    --hash=sha256:9cb1df7e9bc60f75fb87f67940a8fb805aad544337a67a40b67c05cfe33711a2
 cchardet==2.1.5 \
     --hash=sha256:0f7ec49fcd28088c387d4afcc02c0549434d9e07deb2519365a6baa5b6c7ebb4 \
     --hash=sha256:1a6d00b7cbd8acfc5e3093cb5f983a667d0752dc328123c8dcb293e252bfb024 \
@@ -143,7 +149,7 @@ cffi==1.13.2 \
     --hash=sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25 \
     --hash=sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b \
     --hash=sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d \
-    # via argon2-cffi, cryptography
+    # via argon2-cffi, cairocffi, cryptography
 cfn-lint==0.27.4 \
     --hash=sha256:07aa493be259f90a77f590213d26df9e834d003843c4dafc026d730055ba54a9 \
     --hash=sha256:085ded355f11278c14a1c45335e27f81b2a31e6c3eb9ec288a2b39ec813829b4 \
@@ -219,6 +225,10 @@ cryptography==2.8 \
     --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \
     --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 \
     # via apns2, moto, pyopenssl, requests, scrapy, service-identity, social-auth-core, sshpubkeys
+cssselect2==0.3.0 \
+    --hash=sha256:5c2716f06b5de93f701d5755a9666f2ee22cbcd8b4da8adddfc30095ffea3abc \
+    --hash=sha256:97d7d4234f846f9996d838964d38e13b45541c18143bc55cf00e4bc1281ace76 \
+    # via cairosvg
 cssselect==1.1.0 \
     --hash=sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf \
     --hash=sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc \
@@ -822,6 +832,10 @@ https://github.com/zulip/talon/archive/7d8bdc4dbcfcc5a73298747293b99fe53da55315.
 tblib==1.6.0 \
     --hash=sha256:229bee3754cb5d98b4837dd5c4405e80cfab57cb9f93220410ad367f8b352344 \
     --hash=sha256:e222f44485d45ed13fada73b57775e2ff9bd8af62160120bbb6679f5ad80315b
+tinycss2==1.0.2 \
+    --hash=sha256:6427d0e3faa0a5e0e8c9f6437e2de26148a7a197a8b0992789f23d9a802788cf \
+    --hash=sha256:9fdacc0e22d344ddd2ca053837c133900fe820ae1222f63b79617490a498507a \
+    # via cairosvg, cssselect2
 tornado==4.5.3 \
     --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \
     --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \
@@ -898,6 +912,10 @@ wcwidth==0.1.8 \
     --hash=sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603 \
     --hash=sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8 \
     # via prompt-toolkit
+webencodings==0.5.1 \
+    --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+    --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
+    # via cssselect2, tinycss2
 websocket-client==0.57.0 \
     --hash=sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549 \
     --hash=sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010 \
@@ -980,4 +998,4 @@ pip==20.0.2 \
 setuptools==45.1.0 \
     --hash=sha256:68e7fd3508687f94367f1aa090a3ed921cd045a60b73d8b0aa1f305199a0ca28 \
     --hash=sha256:91f72d83602a6e5e4a9e4fe296e27185854038d7cbda49dcd7006c4d3b3b89d5 \
-    # via ipython, jsonschema, markdown, sphinx, zope.interface
+    # via cairocffi, cssselect2, ipython, jsonschema, markdown, sphinx, tinycss2, zope.interface

tools/generate-integration-docs-screenshot

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Create or update a webhook integration screenshot using a test fixture."""
+
+# check for the venv
+from lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+import os
+import sys
+
+TOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
+ROOT_DIR = os.path.dirname(TOOLS_DIR)
+
+sys.path.insert(0, ROOT_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+import django
+
+django.setup()
+
+import argparse
+from typing import Any, Dict
+
+import requests
+import ujson
+
+from zerver.models import UserProfile, get_user_by_delivery_email, get_realm
+from zerver.lib.actions import do_create_user, notify_created_bot
+from zerver.lib.upload import upload_avatar_image
+from zerver.lib.actions import do_change_avatar_fields
+from zerver.lib.integrations import WebhookIntegration, INTEGRATIONS, split_fixture_path
+from zerver.lib.webhooks.common import get_fixture_http_headers
+from setup.generate_zulip_bots_static_files import create_png_from_svg
+
+def create_integration_bot(integration_name: str) -> UserProfile:
+    realm = get_realm('zulip')
+    owner = get_user_by_delivery_email("iago@zulip.com", realm)
+    bot_email = "{}-bot@example.com".format(integration_name)
+    bot_name = "{} Bot".format(integration_name.capitalize())
+    try:
+        bot = UserProfile.objects.get(email=bot_email)
+    except UserProfile.DoesNotExist:
+        bot = do_create_user(
+            email=bot_email,
+            password="123",
+            realm=owner.realm,
+            full_name=bot_name,
+            short_name=bot_name,
+            bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
+            bot_owner=owner,
+        )
+        notify_created_bot(bot)
+
+        if integration.logo_url is None:
+            return bot
+        logo_relative_path = integration.logo_url[len(realm.uri) + 1:]
+        logo_path = os.path.join(ROOT_DIR, logo_relative_path)
+        if logo_path.endswith(".svg"):
+            logo_path = create_png_from_svg(logo_path)
+
+        with open(logo_path, "rb") as f:
+            upload_avatar_image(f, owner, bot)
+            do_change_avatar_fields(bot, UserProfile.AVATAR_FROM_USER)
+
+    return bot
+
+def get_integration(integration_name: str) -> WebhookIntegration:
+    integration = INTEGRATIONS[integration_name]
+    assert isinstance(integration, WebhookIntegration), "Not a WebhookIntegration"
+    return integration
+
+def get_requests_headers(integration_name: str, fixture_name: str) -> Dict[str, Any]:
+    headers = get_fixture_http_headers(integration_name, fixture_name)
+
+    def fix_name(header: str) -> str:
+        header = header if not header.startswith('HTTP_') else header[len('HTTP_'):]
+        return header.replace('_', '-')
+
+    return {fix_name(k): v for k, v in headers.items()}
+
+def webhook_json_fixture(path: str) -> str:
+    path = os.path.abspath(path)
+    if not (os.path.exists(path) and path.endswith('.json') and 'webhooks' in path):
+        raise ValueError('Not a valid webhook JSON fixture')
+    return path
+
+parser = argparse.ArgumentParser()
+parser.add_argument('fixture', type=webhook_json_fixture, help='Path to the fixture to use')
+options = parser.parse_args()
+
+integration_name, fixture_name = split_fixture_path(options.fixture)
+integration = get_integration(integration_name)
+bot = create_integration_bot(integration.name)
+assert isinstance(bot.bot_owner, UserProfile)
+
+url = "{}/{}?api_key={}&stream=devel".format(
+    bot.bot_owner.realm.uri, integration.url, bot.api_key
+)
+with open(options.fixture) as f:
+    data = ujson.load(f)
+headers = get_requests_headers(integration_name, fixture_name)
+response = requests.post(url, json=data, headers=headers)
+if response.status_code == 200:
+    print('Triggered {} webhook'.format(integration.name))
+else:
+    print(response.json())

tools/setup/generate_zulip_bots_static_files.py

@@ -4,7 +4,10 @@ import glob
 import os
 import sys
 import shutil
-from typing import List
+import tempfile
+from typing import List, Optional
+
+import cairosvg
 
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 if ZULIP_PATH not in sys.path:
@@ -42,5 +45,13 @@ def generate_zulip_bots_static_files() -> None:
     docs = glob.glob(doc_glob_pattern)
     copyfiles(docs)
 
+def create_png_from_svg(svg_path: str, destination_dir: Optional[str]=None) -> str:
+    png_name = os.path.splitext(os.path.basename(svg_path))[0] + '.png'
+    if destination_dir is None:
+        destination_dir = tempfile.gettempdir()
+    png_path = os.path.join(destination_dir, png_name)
+    cairosvg.svg2png(url=svg_path, write_to=png_path)
+    return png_path
+
 if __name__ == "__main__":
     generate_zulip_bots_static_files()

version.py

@@ -34,4 +34,4 @@ DESKTOP_WARNING_VERSION = "5.0.0"
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '75.6'
+PROVISION_VERSION = '75.7'

zerver/lib/integrations.py

@@ -181,6 +181,12 @@ class WebhookIntegration(Integration):
     def url_object(self) -> RegexPattern:
         return url(self.url, self.function)
 
+def split_fixture_path(path: str) -> Tuple[str, str]:
+    path, fixture_name = os.path.split(path)
+    fixture_name, _ = os.path.splitext(fixture_name)
+    integration_name = os.path.split(os.path.dirname(path))[-1]
+    return integration_name, fixture_name
+
 class HubotIntegration(Integration):
     GIT_URL_TEMPLATE = "https://github.com/hubot-scripts/hubot-{}"
 

zerver/tests/test_integrations.py

@@ -0,0 +1,10 @@
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.integrations import split_fixture_path
+
+class IntegrationsTestCase(ZulipTestCase):
+
+    def test_split_fixture_path(self) -> None:
+        path = 'zerver/webhooks/semaphore/fixtures/push.json'
+        integration_name, fixture_name = split_fixture_path(path)
+        self.assertEqual(integration_name, 'semaphore')
+        self.assertEqual(fixture_name, 'push')