diff --git a/static/images/integrations/integrations_dev_panel.png b/static/images/integrations/integrations_dev_panel.png new file mode 100644 index 0000000000..462757ae5e Binary files /dev/null and b/static/images/integrations/integrations_dev_panel.png differ diff --git a/static/js/integrations_dev_panel.js b/static/js/integrations_dev_panel.js new file mode 100644 index 0000000000..dc307558b0 --- /dev/null +++ b/static/js/integrations_dev_panel.js @@ -0,0 +1,261 @@ +(function () { + + +// Data Segment: We lazy load the requested fixtures from the backend as and when required +// and then keep them here. +var loaded_fixtures = {}; +var url_base = "/api/v1/external/"; + + +// Resetting/Clearing: a method and a map for clearing certain UI elements. +var clear_handlers = { + stream_name: "#stream_name", + topic_name: "#topic_name", + URL: "#URL", + message: "#message", + bot_name: function () { $('#bot_name').children()[0].selected = true; }, + integration_name: function () { $('#integration_name').children()[0].selected = true; }, + fixture_name: function () { $('#fixture_name').empty(); }, + fixture_body: function () { $("#fixture_body")[0].value = ""; }, +}; + +function clear_elements(elements) { + /* Clear UI elements by specifying which ones to clear as an array of strings. */ + elements.forEach(function (element_name) { + var handler = clear_handlers[element_name]; + if (typeof handler === "string") { + // Handle clearing text input fields or the message field. + var element_object = $(handler)[0]; + element_object.value = ""; + element_object.innerHTML = ""; + } else { + // Use the function returned by the map directly. + handler(); + } + }); + return; +} + + +// Message handlers: The message is a small paragraph at the bottom of the page where +// we let the user know what happened - e.g. success, invalid JSON, etc. +var message_level_to_color_map = { + warning: "#be1931", + success: "#085d44", +}; + +function set_message(msg, level) { + var message_field = $("#message")[0]; + message_field.innerHTML = msg; + message_field.style.color = message_level_to_color_map[level]; + return; +} + + +// Helper methods +function get_api_key_from_selected_bot() { + return $("#bot_name").children("option:selected").val(); +} + +function get_selected_integration_name() { + return $("#integration_name").children("option:selected").val(); +} + +function load_fixture_body(fixture_name) { + /* Given a fixture name, use the loaded_fixtures dictionary to set the fixture body field. */ + var integration_name = get_selected_integration_name(); + var element = loaded_fixtures[integration_name][fixture_name]; + var fixture_body = JSON.stringify(element, null, 4); // 4 is for the pretty print indent factor. + + if (fixture_body === undefined) { + set_message("Fixture does not have a body.", "warning"); + return; + } + $("#fixture_body")[0].value = fixture_body; + + return; +} + +function load_fixture_options(integration_name) { + /* Using the integration name and loaded_fixtures object to set the fixture options for the + fixture_names dropdown and also set the fixture body to the first fixture by default. */ + var fixtures_options_dropdown = $("#fixture_name")[0]; + var fixtures_names = Object.keys(loaded_fixtures[integration_name]); + + fixtures_names.forEach(function (fixture_name) { + var new_dropdown_option = document.createElement("option"); + new_dropdown_option.value = fixture_name; + new_dropdown_option.innerHTML = fixture_name; + fixtures_options_dropdown.add(new_dropdown_option); + }); + load_fixture_body(fixtures_names[0]); + + return; +} + +function update_url() { + /* Automatically build the URL that the webhook should be targeting. To generate this URL, we + would need at least the bot's API Key and the integration name. The stream and topic are both + optional, and for the sake of completeness, it should be noted that the topic is irrelavent + without specifying the stream.*/ + var url_field = $("#URL")[0]; + + var integration_name = get_selected_integration_name(); + var api_key = get_api_key_from_selected_bot(); + + if (integration_name === "" || api_key === "") { + clear_elements(["URL"]); + } else { + var url = url_base + integration_name + "?api_key=" + api_key; + var stream_name = $("#stream_name").val(); + if (stream_name !== "") { + url += "&stream=" + stream_name; + var topic_name = $("#topic_name").val(); + if (topic_name !== "") { + url += "&topic=" + topic_name; + } + } + url_field.value = url; + url_field.innerHTML = url; + } + + return; +} + + +// API Callers: These methods handle communicating with the Python backend API. +function handle_unsuccessful_response(response) { + clear_elements(["fixture_body", "fixture_name", "integration_name", "URL"]); + try { + var status_code = response.statusCode().status; + response = JSON.parse(response.responseText); + set_message("Result: " + "(" + status_code + ") " + response.msg, "warning"); + } catch (err) { + // If the response is not a JSON response then it would be Django sending a HTML response + // containing a stack trace and useful debugging information regarding the backend code. + document.write(response.responseText); + } + return; +} + +function get_fixtures(integration_name) { + /* Request fixtures from the backend for any integrations that we don't already have fixtures + for (which would be stored in the JS variable called "loaded_fixtures"). */ + if (integration_name === "") { + clear_elements(["fixture_body", "fixture_name", "URL", "message"]); + return; + } + + if (loaded_fixtures[integration_name] !== undefined) { + load_fixture_options(integration_name); + return; + } + + // We don't have the fixutures for this integration; fetch them using Zulip's channel library. + // Relative url pattern: /devtools/integrations/(?P.+)/fixtures + channel.get({ + url: "/devtools/integrations/" + integration_name + "/fixtures", + idempotent: false, // Since the user may add or modify fixtures while testing. + success: function (response) { + loaded_fixtures[integration_name] = response.fixtures; + load_fixture_options(integration_name); + return; + }, + error: handle_unsuccessful_response, + }); + + return; +} + +function send_webhook_fixture_message() { + /* Make sure that the user is sending valid JSON in the fixture body and that the URL is not + empty. Then simply send the fixture body to the specified URL. */ + + // Note: If the user has just logged in using a seperate tab while the integrations dev panel is + // open, then the csrf token that we have stored in the hidden input element would be obsoleted + // leading to an error message when the user tries to send the fixture body. + var csrftoken = $("#csrftoken").val(); + + var url = $("#URL").val(); + if (url === "") { + set_message("URL can't be empty.", "warning"); + return; + } + + var body = $("#fixture_body").val(); + try { + // Let JavaScript validate the JSON for us. + body = JSON.stringify(JSON.parse(body)); + } catch (err) { + set_message("Invalid JSON in fixture body.", "warning"); + return; + } + + channel.post({ + url: "/devtools/integrations/check_send_webhook_fixture_message", + data: {url: url, body: body}, + beforeSend: function (xhr) {xhr.setRequestHeader('X-CSRFToken', csrftoken);}, + success: function () { + // If the previous fixture body was sent successfully, then we should change the success + // message up a bit to let the user easily know that this fixture body was also sent + // successfully. + if ($("#message")[0].innerHTML === "Success!") { + set_message("Success!!!", "success"); + } else { + set_message("Success!", "success"); + } + return; + }, + error: handle_unsuccessful_response, + }); + + return; +} + +// Initialization +$(function () { + clear_elements(["stream_name", "topic_name", "URL", "bot_name", "integration_name", + "fixture_name", "fixture_body", "message"]); + + var potential_default_bot = $("#bot_name")[0][1]; + if (potential_default_bot !== undefined) { + potential_default_bot.selected = true; + } + + $('#integration_name').change(function () { + clear_elements(["fixture_body", "fixture_name", "message"]); + var integration_name = $(this).children("option:selected").val(); + get_fixtures(integration_name); + update_url(); + return; + }); + + $('#fixture_name').change(function () { + clear_elements(["fixture_body", "message"]); + var fixture_name = $(this).children("option:selected").val(); + load_fixture_body(fixture_name); + return; + }); + + $('#send_fixture_button').click(function () { + send_webhook_fixture_message(); + return; + }); + + $("#bot_name").change(update_url); + + $("#stream_name").change(update_url); + + $("#topic_name").change(update_url); + +}); + +}()); + +/* +Development Notes: + - We currently don't support non-json fixtures. + +Possible Improvements: + - Add support for extra keys, headers, etc. +*/ diff --git a/static/styles/integrations_dev_panel.css b/static/styles/integrations_dev_panel.css new file mode 100644 index 0000000000..e38f68a074 --- /dev/null +++ b/static/styles/integrations_dev_panel.css @@ -0,0 +1,29 @@ +#panel_div { + margin: auto; + width: 720px; + margin-top: 50px; + border: 3px solid; + border-color: rgb(0, 128, 0); + padding: 10px; +} + +#fixture_body { + height: 500px; + width: 700px; +} + +#send_fixture_button { + float: right; +} + +#URL { + width: 90%; +} + +.center-text { + text-align: center; +} + +.pad-top { + padding-top: 30px; +} diff --git a/templates/zerver/api/incoming-webhooks-walkthrough.md b/templates/zerver/api/incoming-webhooks-walkthrough.md index b671bc03e4..f6ff4b1d31 100644 --- a/templates/zerver/api/incoming-webhooks-walkthrough.md +++ b/templates/zerver/api/incoming-webhooks-walkthrough.md @@ -40,7 +40,7 @@ When writing your own incoming webhook integration, you'll want to write a test for each distinct message condition your integration supports. You'll also need a corresponding fixture for each of these tests. Depending on the type of data the 3rd party service sends, your fixture may contain JSON, URL encoded text, or -some other kind of data. See [Step 4: Create tests](#step-4-create-tests) or +some other kind of data. See [Step 5: Create automated tests](#step-5-create-automated-tests) or [Testing](https://zulip.readthedocs.io/en/latest/testing/testing.html) for further details. ## Step 1: Initialize your webhook python package @@ -127,7 +127,7 @@ from the body of the http request, `stream` with a default of `test` (available by default in the Zulip development environment), and `topic` with a default of `Hello World`. If your webhook uses a custom stream, it must exist before a message can be created in it. (See -[Step 4: Create tests](#step-4-create-tests) for how to handle this in tests.) +[Step 4: Create automated tests](#step-5-create-automated-tests) for how to handle this in tests.) The line that begins `# type` is a mypy type annotation. See [this page](https://zulip.readthedocs.io/en/latest/testing/mypy.html) for details about @@ -179,14 +179,31 @@ icon. The second positional argument defines a list of categories for the integration. At this point, if you're following along and/or writing your own Hello World -webhook, you have written enough code to test your integration. +webhook, you have written enough code to test your integration. There are three +tools which you can use to test your webhook - 2 command line tools and a GUI. -First, get an API key from the Your bots section of your Zulip user's Settings -page. If you haven't created a bot already, you can do that there. Then copy -its API key and replace the placeholder `` in the examples with -your real key. This is how Zulip knows the request is from an authorized user. +## Step 4: Manually testing the webhook -Now you can test using Zulip itself, or curl on the command line. +For either one of the command line tools, first, you'll need to get an API key +from the **Your bots** section of your Zulip user's Settings page. To test the webhook, +you'll need to [create a bot](https://zulipchat.com/help/add-a-bot-or-integration) with +the **Incoming Webhook** type. Replace `` with your bot's API key in the examples +presented below! This is how Zulip knows that the request was made by an authorized user. + +### Curl + +Using curl: +``` +curl -X POST -H "Content-Type: application/json" -d '{ "featured_title":"Marilyn Monroe", "featured_url":"https://en.wikipedia.org/wiki/Marilyn_Monroe" }' http://localhost:9991/api/v1/external/helloworld\?api_key\= +``` + +After running the above command, you should see something similar to: + +``` +{"msg":"","result":"success"} +``` + +### Management Command: send_webhook_fixture_message Using `manage.py` from within the Zulip development environment: @@ -196,27 +213,13 @@ Using `manage.py` from within the Zulip development environment: --fixture=zerver/webhooks/helloworld/fixtures/hello.json \ '--url=http://localhost:9991/api/v1/external/helloworld?api_key=' ``` -After which you should see something similar to: + +After running the above command, you should see something similar to: ``` 2016-07-07 15:06:59,187 INFO 127.0.0.1 POST 200 143ms (mem: 6ms/13) (md: 43ms/1) (db: 20ms/9q) (+start: 147ms) /api/v1/external/helloworld (helloworld-bot@zulip.com via ZulipHelloWorldWebhook) ``` -Using curl: - -``` -curl -X POST -H "Content-Type: application/json" -d '{ "featured_title":"Marilyn Monroe", "featured_url":"https://en.wikipedia.org/wiki/Marilyn_Monroe" }' http://localhost:9991/api/v1/external/helloworld\?api_key\= -``` - -After which you should see: -``` -{"msg":"","result":"success"} -``` - -Using either method will create a message in Zulip: - -screenshot - Some webhooks require custom HTTP headers, which can be passed using `./manage.py send_webhook_fixture_message --custom-headers`. For example: @@ -227,7 +230,32 @@ The format is a JSON dictionary, so make sure that the header names do not contain any spaces in them and that you use the precise quoting approach shown above. -## Step 4: Create tests +### Integrations Dev Panel +This is the GUI tool. + + + +1. Run `./tools/run-dev.py` then go to http://localhost:9991/devtools/integrations/. + +2. Set the following mandatory fields: +- **Bot** - Any incoming webhook bot. +- **Integration** - One of the integrations. +- **Fixture** - Though not mandatory, it's recommended that you select one and then tweak it if necessary. +The remaining fields are optional, and the URL will automatically be generated. + +3. Click **Send**! + +By opening Zulip in one tab and this tool in another, you can quickly tweak +your code and send sample messages for many different test fixtures. + + +Your sample notification may look like: + + + + + +## Step 5: Create automated tests Every webhook integration should have a corresponding test file: `zerver/webhooks/mywebhook/tests.py`. @@ -329,7 +357,7 @@ Running zerver.webhooks.helloworld.tests.HelloWorldHookTests.test_hello_message DONE! ``` -## Step 5: Create documentation +## Step 6: Create documentation Next, we add end-user documentation for our integration. You can see the existing examples at @@ -389,7 +417,7 @@ screenshot. Mostly you should plan on templating off an existing guide, like [integration-docs-guide]: https://zulip.readthedocs.io/en/latest/subsystems/integration-docs.html -## Step 5: Preparing a pull request to zulip/zulip +## Step 7: Preparing a pull request to zulip/zulip When you have finished your webhook integration and are ready for it to be available in the Zulip product, follow these steps to prepare your pull diff --git a/templates/zerver/dev_tools.html b/templates/zerver/dev_tools.html index c6bfcc0952..7c36afc5ef 100644 --- a/templates/zerver/dev_tools.html +++ b/templates/zerver/dev_tools.html @@ -73,6 +73,11 @@ None needed Invalid confirmation link page + + /devtools/integrations + None needed + Test incoming webhook integrations +

Development-specific management commands live in zilencer/management/commands. Highlights include: diff --git a/templates/zerver/integrations/development/dev_panel.html b/templates/zerver/integrations/development/dev_panel.html new file mode 100644 index 0000000000..18084fca7c --- /dev/null +++ b/templates/zerver/integrations/development/dev_panel.html @@ -0,0 +1,90 @@ +{% extends "zerver/base.html" %} + +{% block customhead %} + + +{{ render_bundle('integrations-dev-panel') }} + +{% endblock %} + + +{% block content %} + + + +

+ +

Integrations Developer Panel

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
Bot + +
Integration + +
Fixture
+ + +
JSON Body
+ +
+

+ +
+ +
+ +


+ +{% endblock %} diff --git a/tools/webpack.assets.json b/tools/webpack.assets.json index a999126ce4..bd71259f7f 100644 --- a/tools/webpack.assets.json +++ b/tools/webpack.assets.json @@ -87,6 +87,11 @@ "./static/styles/app_components.scss" ], "dev-login": "./static/js/portico/dev-login.js", + "integrations-dev-panel": [ + "./static/js/integrations_dev_panel.js", + "./static/styles/integrations_dev_panel.css", + "./static/js/channel.js" + ], "email-log": "./static/js/portico/email_log.js", "stats": [ "./static/styles/stats.scss", diff --git a/zerver/tests/test_integrations_dev_panel.py b/zerver/tests/test_integrations_dev_panel.py new file mode 100644 index 0000000000..7f1d317932 --- /dev/null +++ b/zerver/tests/test_integrations_dev_panel.py @@ -0,0 +1,81 @@ +import ujson +from mock import MagicMock, patch +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import get_user, get_realm, Message, Stream + +class TestIntegrationsDevPanel(ZulipTestCase): + + zulip_realm = get_realm("zulip") + + def test_check_send_webhook_fixture_message_for_error(self) -> None: + bot = get_user('webhook-bot@zulip.com', self.zulip_realm) + url = "/api/v1/external/airbrake?api_key={key}".format(key=bot.api_key) + target_url = "/devtools/integrations/check_send_webhook_fixture_message" + body = "{}" # This empty body should generate a KeyError on the webhook code side. + + data = { + "url": url, + "body": body + } + + response = self.client_post(target_url, data) + + self.assertEqual(response.status_code, 500) # Since the response would be forwarded. + expected_response = {"result": "error", "msg": "Internal server error"} + self.assertEqual(ujson.loads(response.content), expected_response) + + def test_check_send_webhook_fixture_message_for_success(self) -> None: + bot = get_user('webhook-bot@zulip.com', self.zulip_realm) + url = "/api/v1/external/airbrake?api_key={key}&stream=Denmark&topic=Airbrake Notifications".format(key=bot.api_key) + target_url = "/devtools/integrations/check_send_webhook_fixture_message" + with open("zerver/webhooks/airbrake/fixtures/error_message.json", "r") as f: + body = f.read() + + data = { + "url": url, + "body": body, + } + + response = self.client_post(target_url, data) + self.assertEqual(response.status_code, 200) + + latest_msg = Message.objects.latest('id') + expected_message = "[ZeroDivisionError](https://zulip.airbrake.io/projects/125209/groups/1705190192091077626): \"Error message from logger\" occurred." + self.assertEqual(latest_msg.content, expected_message) + self.assertEqual(Stream.objects.get(id=latest_msg.recipient.type_id).name, "Denmark") + self.assertEqual(latest_msg.topic_name(), "Airbrake Notifications") + + def test_get_fixtures_for_nonexistant_integration(self) -> None: + target_url = "/devtools/integrations/somerandomnonexistantintegration/fixtures" + response = self.client_get(target_url) + expected_response = {'msg': '"somerandomnonexistantintegration" is not a valid webhook integration.', 'result': 'error'} + self.assertEqual(response.status_code, 404) + self.assertEqual(ujson.loads(response.content), expected_response) + + @patch("zerver.views.development.integrations.os.path.exists") + def test_get_fixtures_for_integration_without_fixtures(self, os_path_exists_mock: MagicMock) -> None: + os_path_exists_mock.return_value = False + target_url = "/devtools/integrations/airbrake/fixtures" + response = self.client_get(target_url) + expected_response = {'msg': 'The integration "airbrake" does not have fixtures.', 'result': 'error'} + self.assertEqual(response.status_code, 404) + self.assertEqual(ujson.loads(response.content), expected_response) + + def test_get_fixtures_for_integration_without_json_fixtures(self) -> None: + target_url = "/devtools/integrations/deskdotcom/fixtures" + response = self.client_get(target_url) + expected_response = {'msg': 'The integration "deskdotcom" has non-JSON fixtures.', 'result': 'error'} + self.assertEqual(response.status_code, 400) + self.assertEqual(ujson.loads(response.content), expected_response) + + def test_get_fixtures_for_success(self) -> None: + target_url = "/devtools/integrations/airbrake/fixtures" + response = self.client_get(target_url) + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(ujson.loads(response.content)["fixtures"]) + + def test_get_dev_panel_page(self) -> None: + # Just to satisfy the test suite. + target_url = "/devtools/integrations/" + response = self.client_get(target_url) + self.assertEqual(response.status_code, 200) diff --git a/zerver/views/development/integrations.py b/zerver/views/development/integrations.py new file mode 100644 index 0000000000..d0f28e019c --- /dev/null +++ b/zerver/views/development/integrations.py @@ -0,0 +1,68 @@ +import os +import ujson +from typing import List + +from django.http import HttpRequest, HttpResponse +from django.shortcuts import render +from django.test import Client + +from zerver.lib.integrations import WEBHOOK_INTEGRATIONS +from zerver.lib.request import has_request_variables, REQ +from zerver.lib.response import json_success, json_error +from zerver.models import UserProfile, get_realm + + +ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') + + +def get_webhook_integrations() -> List[str]: + return [integration.name for integration in WEBHOOK_INTEGRATIONS] + + +def dev_panel(request: HttpRequest) -> HttpResponse: + integrations = get_webhook_integrations() + bots = UserProfile.objects.filter(is_bot=True, bot_type=UserProfile.INCOMING_WEBHOOK_BOT) + context = {"integrations": integrations, "bots": bots} + return render(request, "zerver/integrations/development/dev_panel.html", context) + + +@has_request_variables +def get_fixtures(request: HttpResponse, + integration_name: str=REQ()) -> HttpResponse: + integrations = get_webhook_integrations() + if integration_name not in integrations: + return json_error("\"{integration_name}\" is not a valid webhook integration.".format( + integration_name=integration_name), status=404) + + fixtures = {} + fixtures_dir = os.path.join(ZULIP_PATH, "zerver/webhooks/{integration_name}/fixtures".format( + integration_name=integration_name)) + if not os.path.exists(fixtures_dir): + msg = ("The integration \"{integration_name}\" does not have fixtures.").format( + integration_name=integration_name) + return json_error(msg, status=404) + + for fixture in os.listdir(fixtures_dir): + fixture_path = os.path.join(fixtures_dir, fixture) + try: + json_data = ujson.loads(open(fixture_path).read()) + except ValueError: + msg = ("The integration \"{integration_name}\" has non-JSON fixtures.").format( + integration_name=integration_name) + return json_error(msg) + fixtures[fixture] = json_data + + return json_success({"fixtures": fixtures}) + + +@has_request_variables +def check_send_webhook_fixture_message(request: HttpRequest, + url: str=REQ(), + body: str=REQ()) -> HttpResponse: + client = Client() + realm = get_realm("zulip") + response = client.post(url, body, content_type="application/json", HTTP_HOST=realm.host) + if response.status_code == 200: + return json_success() + else: + return response diff --git a/zproject/dev_urls.py b/zproject/dev_urls.py index 18c8a98c21..95b260f6c8 100644 --- a/zproject/dev_urls.py +++ b/zproject/dev_urls.py @@ -6,6 +6,7 @@ from django.views.static import serve import zerver.views.development.registration import zerver.views.auth import zerver.views.development.email_log +import zerver.views.development.integrations # These URLs are available only in the development environment @@ -48,6 +49,13 @@ urls = [ # Have easy access for error pages url(r'^errors/404/$', TemplateView.as_view(template_name='404.html')), url(r'^errors/5xx/$', TemplateView.as_view(template_name='500.html')), + + # Add a convinient way to generate webhook messages from fixtures. + url(r'^devtools/integrations/$', zerver.views.development.integrations.dev_panel), + url(r'^devtools/integrations/check_send_webhook_fixture_message$', + zerver.views.development.integrations.check_send_webhook_fixture_message), + url(r'^devtools/integrations/(?P.+)/fixtures$', + zerver.views.development.integrations.get_fixtures), ] i18n_urls = [