From 4d2ce607c91f8fdd09be2c8eea31d6b04add448d Mon Sep 17 00:00:00 2001 From: Puneeth Chaganti Date: Thu, 9 Apr 2020 23:03:49 +0530 Subject: [PATCH] tools: Add script to trigger webhook notification using fixtures. When creating a webhook integration or creating a new one, it is a pain to create or update the screenshots in the documentation. This commit adds a tool that can trigger a sample notification for the webhook using a fixture, that is likely already written for the tests. Currently, the developer needs to take a screenshot manually, but this could be automated using puppeteer or something like that. Also, the tool does not support webhooks with basic auth, and only supports webhooks that use json fixtures. These can be fixed in subsequent commits. --- requirements/dev.in | 3 + requirements/dev.txt | 22 +++- tools/generate-integration-docs-screenshot | 110 ++++++++++++++++++ .../setup/generate_zulip_bots_static_files.py | 13 ++- version.py | 2 +- zerver/lib/integrations.py | 6 + zerver/tests/test_integrations.py | 10 ++ 7 files changed, 162 insertions(+), 4 deletions(-) create mode 100755 tools/generate-integration-docs-screenshot create mode 100644 zerver/tests/test_integrations.py diff --git a/requirements/dev.in b/requirements/dev.in index af04b318b5..f4c9d16f0f 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -64,3 +64,6 @@ https://github.com/zulip/zulint/archive/aaed679f1ad38b230090eadd3870b7682500f60c importlib-metadata # built-in python > 3.6 needed by cfn-lint importlib-resources + +# Needed for using integration logo svg files as bot avatars +cairosvg diff --git a/requirements/dev.txt b/requirements/dev.txt index 883e90583e..8a426edddd 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -81,6 +81,12 @@ cachetools==4.0.0 \ --hash=sha256:9a52dd97a85f257f4e4127f15818e71a0c7899f121b34591fcc1173ea79a0198 \ --hash=sha256:b304586d357c43221856be51d73387f93e2a961598a9b6b6670664746f3b6c6c \ # via premailer +cairocffi==1.1.0 \ + --hash=sha256:f1c0c5878f74ac9ccb5d48b2601fcc75390c881ce476e79f4cfedd288b1b05db \ + # via cairosvg +cairosvg==2.4.2 \ + --hash=sha256:4e668f96653326780036ebb0a9ff2bb59a8443d7bcfc51a14aab77b57a8e67ad \ + --hash=sha256:9cb1df7e9bc60f75fb87f67940a8fb805aad544337a67a40b67c05cfe33711a2 cchardet==2.1.5 \ --hash=sha256:0f7ec49fcd28088c387d4afcc02c0549434d9e07deb2519365a6baa5b6c7ebb4 \ --hash=sha256:1a6d00b7cbd8acfc5e3093cb5f983a667d0752dc328123c8dcb293e252bfb024 \ @@ -143,7 +149,7 @@ cffi==1.13.2 \ --hash=sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25 \ --hash=sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b \ --hash=sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d \ - # via argon2-cffi, cryptography + # via argon2-cffi, cairocffi, cryptography cfn-lint==0.27.4 \ --hash=sha256:07aa493be259f90a77f590213d26df9e834d003843c4dafc026d730055ba54a9 \ --hash=sha256:085ded355f11278c14a1c45335e27f81b2a31e6c3eb9ec288a2b39ec813829b4 \ @@ -219,6 +225,10 @@ cryptography==2.8 \ --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \ --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 \ # via apns2, moto, pyopenssl, requests, scrapy, service-identity, social-auth-core, sshpubkeys +cssselect2==0.3.0 \ + --hash=sha256:5c2716f06b5de93f701d5755a9666f2ee22cbcd8b4da8adddfc30095ffea3abc \ + --hash=sha256:97d7d4234f846f9996d838964d38e13b45541c18143bc55cf00e4bc1281ace76 \ + # via cairosvg cssselect==1.1.0 \ --hash=sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf \ --hash=sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc \ @@ -822,6 +832,10 @@ https://github.com/zulip/talon/archive/7d8bdc4dbcfcc5a73298747293b99fe53da55315. tblib==1.6.0 \ --hash=sha256:229bee3754cb5d98b4837dd5c4405e80cfab57cb9f93220410ad367f8b352344 \ --hash=sha256:e222f44485d45ed13fada73b57775e2ff9bd8af62160120bbb6679f5ad80315b +tinycss2==1.0.2 \ + --hash=sha256:6427d0e3faa0a5e0e8c9f6437e2de26148a7a197a8b0992789f23d9a802788cf \ + --hash=sha256:9fdacc0e22d344ddd2ca053837c133900fe820ae1222f63b79617490a498507a \ + # via cairosvg, cssselect2 tornado==4.5.3 \ --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \ --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \ @@ -898,6 +912,10 @@ wcwidth==0.1.8 \ --hash=sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603 \ --hash=sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8 \ # via prompt-toolkit +webencodings==0.5.1 \ + --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ + --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \ + # via cssselect2, tinycss2 websocket-client==0.57.0 \ --hash=sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549 \ --hash=sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010 \ @@ -980,4 +998,4 @@ pip==20.0.2 \ setuptools==45.1.0 \ --hash=sha256:68e7fd3508687f94367f1aa090a3ed921cd045a60b73d8b0aa1f305199a0ca28 \ --hash=sha256:91f72d83602a6e5e4a9e4fe296e27185854038d7cbda49dcd7006c4d3b3b89d5 \ - # via ipython, jsonschema, markdown, sphinx, zope.interface + # via cairocffi, cssselect2, ipython, jsonschema, markdown, sphinx, tinycss2, zope.interface diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot new file mode 100755 index 0000000000..f96405fa85 --- /dev/null +++ b/tools/generate-integration-docs-screenshot @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Create or update a webhook integration screenshot using a test fixture.""" + +# check for the venv +from lib import sanity_check + +sanity_check.check_venv(__file__) + +import os +import sys + +TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) +ROOT_DIR = os.path.dirname(TOOLS_DIR) + +sys.path.insert(0, ROOT_DIR) +from scripts.lib.setup_path import setup_path + +setup_path() + +os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" +import django + +django.setup() + +import argparse +from typing import Any, Dict + +import requests +import ujson + +from zerver.models import UserProfile, get_user_by_delivery_email, get_realm +from zerver.lib.actions import do_create_user, notify_created_bot +from zerver.lib.upload import upload_avatar_image +from zerver.lib.actions import do_change_avatar_fields +from zerver.lib.integrations import WebhookIntegration, INTEGRATIONS, split_fixture_path +from zerver.lib.webhooks.common import get_fixture_http_headers +from setup.generate_zulip_bots_static_files import create_png_from_svg + +def create_integration_bot(integration_name: str) -> UserProfile: + realm = get_realm('zulip') + owner = get_user_by_delivery_email("iago@zulip.com", realm) + bot_email = "{}-bot@example.com".format(integration_name) + bot_name = "{} Bot".format(integration_name.capitalize()) + try: + bot = UserProfile.objects.get(email=bot_email) + except UserProfile.DoesNotExist: + bot = do_create_user( + email=bot_email, + password="123", + realm=owner.realm, + full_name=bot_name, + short_name=bot_name, + bot_type=UserProfile.INCOMING_WEBHOOK_BOT, + bot_owner=owner, + ) + notify_created_bot(bot) + + if integration.logo_url is None: + return bot + logo_relative_path = integration.logo_url[len(realm.uri) + 1:] + logo_path = os.path.join(ROOT_DIR, logo_relative_path) + if logo_path.endswith(".svg"): + logo_path = create_png_from_svg(logo_path) + + with open(logo_path, "rb") as f: + upload_avatar_image(f, owner, bot) + do_change_avatar_fields(bot, UserProfile.AVATAR_FROM_USER) + + return bot + +def get_integration(integration_name: str) -> WebhookIntegration: + integration = INTEGRATIONS[integration_name] + assert isinstance(integration, WebhookIntegration), "Not a WebhookIntegration" + return integration + +def get_requests_headers(integration_name: str, fixture_name: str) -> Dict[str, Any]: + headers = get_fixture_http_headers(integration_name, fixture_name) + + def fix_name(header: str) -> str: + header = header if not header.startswith('HTTP_') else header[len('HTTP_'):] + return header.replace('_', '-') + + return {fix_name(k): v for k, v in headers.items()} + +def webhook_json_fixture(path: str) -> str: + path = os.path.abspath(path) + if not (os.path.exists(path) and path.endswith('.json') and 'webhooks' in path): + raise ValueError('Not a valid webhook JSON fixture') + return path + +parser = argparse.ArgumentParser() +parser.add_argument('fixture', type=webhook_json_fixture, help='Path to the fixture to use') +options = parser.parse_args() + +integration_name, fixture_name = split_fixture_path(options.fixture) +integration = get_integration(integration_name) +bot = create_integration_bot(integration.name) +assert isinstance(bot.bot_owner, UserProfile) + +url = "{}/{}?api_key={}&stream=devel".format( + bot.bot_owner.realm.uri, integration.url, bot.api_key +) +with open(options.fixture) as f: + data = ujson.load(f) +headers = get_requests_headers(integration_name, fixture_name) +response = requests.post(url, json=data, headers=headers) +if response.status_code == 200: + print('Triggered {} webhook'.format(integration.name)) +else: + print(response.json()) diff --git a/tools/setup/generate_zulip_bots_static_files.py b/tools/setup/generate_zulip_bots_static_files.py index a4412e22f2..536792b0f9 100755 --- a/tools/setup/generate_zulip_bots_static_files.py +++ b/tools/setup/generate_zulip_bots_static_files.py @@ -4,7 +4,10 @@ import glob import os import sys import shutil -from typing import List +import tempfile +from typing import List, Optional + +import cairosvg ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: @@ -42,5 +45,13 @@ def generate_zulip_bots_static_files() -> None: docs = glob.glob(doc_glob_pattern) copyfiles(docs) +def create_png_from_svg(svg_path: str, destination_dir: Optional[str]=None) -> str: + png_name = os.path.splitext(os.path.basename(svg_path))[0] + '.png' + if destination_dir is None: + destination_dir = tempfile.gettempdir() + png_path = os.path.join(destination_dir, png_name) + cairosvg.svg2png(url=svg_path, write_to=png_path) + return png_path + if __name__ == "__main__": generate_zulip_bots_static_files() diff --git a/version.py b/version.py index 1a5b41714a..64632e8530 100644 --- a/version.py +++ b/version.py @@ -34,4 +34,4 @@ DESKTOP_WARNING_VERSION = "5.0.0" # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '75.6' +PROVISION_VERSION = '75.7' diff --git a/zerver/lib/integrations.py b/zerver/lib/integrations.py index 3f0c8ca3b5..9262f4a657 100644 --- a/zerver/lib/integrations.py +++ b/zerver/lib/integrations.py @@ -181,6 +181,12 @@ class WebhookIntegration(Integration): def url_object(self) -> RegexPattern: return url(self.url, self.function) +def split_fixture_path(path: str) -> Tuple[str, str]: + path, fixture_name = os.path.split(path) + fixture_name, _ = os.path.splitext(fixture_name) + integration_name = os.path.split(os.path.dirname(path))[-1] + return integration_name, fixture_name + class HubotIntegration(Integration): GIT_URL_TEMPLATE = "https://github.com/hubot-scripts/hubot-{}" diff --git a/zerver/tests/test_integrations.py b/zerver/tests/test_integrations.py new file mode 100644 index 0000000000..387f119b68 --- /dev/null +++ b/zerver/tests/test_integrations.py @@ -0,0 +1,10 @@ +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.integrations import split_fixture_path + +class IntegrationsTestCase(ZulipTestCase): + + def test_split_fixture_path(self) -> None: + path = 'zerver/webhooks/semaphore/fixtures/push.json' + integration_name, fixture_name = split_fixture_path(path) + self.assertEqual(integration_name, 'semaphore') + self.assertEqual(fixture_name, 'push')