import abc import json import logging from contextlib import suppress from time import perf_counter from typing import Any, AnyStr, Dict, Optional import requests from django.conf import settings from django.utils.translation import gettext as _ from requests import Response from typing_extensions import override from version import ZULIP_VERSION from zerver.actions.message_send import check_send_message from zerver.lib.exceptions import JsonableError, StreamDoesNotExistError from zerver.lib.message import MessageDict from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.queue import retry_event from zerver.lib.topic import get_topic_from_message_info from zerver.lib.url_encoding import near_message_url from zerver.models import ( GENERIC_INTERFACE, SLACK_INTERFACE, Realm, Service, UserProfile, get_client, get_user_profile_by_id, ) class OutgoingWebhookServiceInterface(metaclass=abc.ABCMeta): def __init__(self, token: str, user_profile: UserProfile, service_name: str) -> None: self.token: str = token self.user_profile: UserProfile = user_profile self.service_name: str = service_name self.session: requests.Session = OutgoingSession( role="webhook", timeout=settings.OUTGOING_WEBHOOK_TIMEOUT_SECONDS, headers={"User-Agent": "ZulipOutgoingWebhook/" + ZULIP_VERSION}, ) @abc.abstractmethod def make_request( self, base_url: str, event: Dict[str, Any], realm: Realm ) -> Optional[Response]: raise NotImplementedError @abc.abstractmethod def process_success(self, response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]: raise NotImplementedError class GenericOutgoingWebhookService(OutgoingWebhookServiceInterface): @override def make_request( self, base_url: str, event: Dict[str, Any], realm: Realm ) -> Optional[Response]: """ We send a simple version of the message to outgoing webhooks, since most of them really only need `content` and a few other fields. We may eventually allow certain bots to get more information, but that's not a high priority. We do send the gravatar info to the clients (so they don't have to compute it themselves). """ message_dict = MessageDict.finalize_payload( event["message"], apply_markdown=False, client_gravatar=False, keep_rendered_content=True, ) request_data = { "data": event["command"], "message": message_dict, "bot_email": self.user_profile.email, "bot_full_name": self.user_profile.full_name, "token": self.token, "trigger": event["trigger"], } return self.session.post(base_url, json=request_data) @override def process_success(self, response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]: if "response_not_required" in response_json and response_json["response_not_required"]: return None if "response_string" in response_json: # We are deprecating response_string. content = str(response_json["response_string"]) success_data = dict(content=content) return success_data if "content" in response_json: content = str(response_json["content"]) success_data = dict(content=content) if "widget_content" in response_json: success_data["widget_content"] = response_json["widget_content"] return success_data return None class SlackOutgoingWebhookService(OutgoingWebhookServiceInterface): @override def make_request( self, base_url: str, event: Dict[str, Any], realm: Realm ) -> Optional[Response]: if event["message"]["type"] == "private": failure_message = "Slack outgoing webhooks don't support direct messages." fail_with_message(event, failure_message) return None # https://api.slack.com/legacy/custom-integrations/outgoing-webhooks#legacy-info__post-data # documents the Slack outgoing webhook format: # # token=XXXXXXXXXXXXXXXXXX # team_id=T0001 # team_domain=example # channel_id=C2147483705 # channel_name=test # thread_ts=1504640714.003543 # timestamp=1504640775.000005 # user_id=U2147483697 # user_name=Steve # text=googlebot: What is the air-speed velocity of an unladen swallow? # trigger_word=googlebot: request_data = [ ("token", self.token), ("team_id", f"T{realm.id}"), ("team_domain", realm.host), ("channel_id", f"C{event['message']['stream_id']}"), ("channel_name", event["message"]["display_recipient"]), ("thread_ts", event["message"]["timestamp"]), ("timestamp", event["message"]["timestamp"]), ("user_id", f"U{event['message']['sender_id']}"), ("user_name", event["message"]["sender_full_name"]), ("text", event["command"]), ("trigger_word", event["trigger"]), ("service_id", event["user_profile_id"]), ] return self.session.post(base_url, data=request_data) @override def process_success(self, response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]: if "text" in response_json: content = response_json["text"] success_data = dict(content=content) return success_data return None AVAILABLE_OUTGOING_WEBHOOK_INTERFACES: Dict[str, Any] = { GENERIC_INTERFACE: GenericOutgoingWebhookService, SLACK_INTERFACE: SlackOutgoingWebhookService, } def get_service_interface_class(interface: str) -> Any: if interface not in AVAILABLE_OUTGOING_WEBHOOK_INTERFACES: return AVAILABLE_OUTGOING_WEBHOOK_INTERFACES[GENERIC_INTERFACE] else: return AVAILABLE_OUTGOING_WEBHOOK_INTERFACES[interface] def get_outgoing_webhook_service_handler(service: Service) -> Any: service_interface_class = get_service_interface_class(service.interface_name()) service_interface = service_interface_class( token=service.token, user_profile=service.user_profile, service_name=service.name ) return service_interface def send_response_message( bot_id: int, message_info: Dict[str, Any], response_data: Dict[str, Any] ) -> None: """ bot_id is the user_id of the bot sending the response message_info is used to address the message and should have these fields: type - "stream" or "private" display_recipient - like we have in other message events topic - see get_topic_from_message_info response_data is what the bot wants to send back and has these fields: content - raw Markdown content for Zulip to render WARNING: This function sends messages bypassing the stream access check for the bot - so use with caution to not call this in codepaths that might let someone send arbitrary messages to any stream through this. """ recipient_type_name = message_info["type"] display_recipient = message_info["display_recipient"] try: topic_name: Optional[str] = get_topic_from_message_info(message_info) except KeyError: topic_name = None bot_user = get_user_profile_by_id(bot_id) realm = bot_user.realm client = get_client("OutgoingWebhookResponse") content = response_data.get("content") assert content widget_content = response_data.get("widget_content") if recipient_type_name == "stream": message_to = [display_recipient] elif recipient_type_name == "private": message_to = [recipient["email"] for recipient in display_recipient] else: raise JsonableError(_("Invalid message type")) check_send_message( sender=bot_user, client=client, recipient_type_name=recipient_type_name, message_to=message_to, topic_name=topic_name, message_content=content, widget_content=widget_content, realm=realm, skip_stream_access_check=True, ) def fail_with_message(event: Dict[str, Any], failure_message: str) -> None: bot_id = event["user_profile_id"] message_info = event["message"] content = "Failure! " + failure_message response_data = dict(content=content) # If the stream has vanished while we were failing, there's no # reasonable place to report the error. with suppress(StreamDoesNotExistError): send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data) def get_message_url(event: Dict[str, Any]) -> str: bot_user = get_user_profile_by_id(event["user_profile_id"]) message = event["message"] realm = bot_user.realm return near_message_url( realm=realm, message=message, ) def notify_bot_owner( event: Dict[str, Any], status_code: Optional[int] = None, response_content: Optional[AnyStr] = None, failure_message: Optional[str] = None, exception: Optional[Exception] = None, ) -> None: message_url = get_message_url(event) bot_id = event["user_profile_id"] bot = get_user_profile_by_id(bot_id) bot_owner = bot.bot_owner assert bot_owner is not None notification_message = f"[A message]({message_url}) to your bot @_**{bot.full_name}** triggered an outgoing webhook." if exception: notification_message += ( "\nWhen trying to send a request to the webhook service, an exception " f"of type {type(exception).__name__} occurred:\n```\n{exception}\n```" ) elif failure_message: notification_message += "\n" + failure_message elif status_code == 407: notification_message += ( "\nThe URL configured for the webhook is for a private or disallowed network." ) elif status_code: notification_message += f"\nThe webhook got a response with status code *{status_code}*." if response_content: notification_message += ( f"\nThe response contains the following payload:\n```\n{response_content!r}\n```" ) message_info = dict( type="private", display_recipient=[dict(email=bot_owner.email)], ) response_data = dict(content=notification_message) send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data) def request_retry(event: Dict[str, Any], failure_message: Optional[str] = None) -> None: def failure_processor(event: Dict[str, Any]) -> None: """ The name of the argument is 'event' on purpose. This argument will hide the 'event' argument of the request_retry function. Keeping the same name results in a smaller diff. """ bot_user = get_user_profile_by_id(event["user_profile_id"]) fail_with_message(event, "Bot is unavailable") notify_bot_owner(event, failure_message=failure_message) logging.warning( "Maximum retries exceeded for trigger:%s event:%s", bot_user.email, event["command"], ) retry_event("outgoing_webhooks", event, failure_processor) def process_success_response( event: Dict[str, Any], service_handler: Any, response: Response ) -> None: try: response_json = json.loads(response.text) except json.JSONDecodeError: raise JsonableError(_("Invalid JSON in response")) if response_json == "": # Versions of zulip_botserver before 2021-05 used # json.dumps("") as their "no response required" success # response; handle that for backwards-compatibility. return if not isinstance(response_json, dict): raise JsonableError(_("Invalid response format")) success_data = service_handler.process_success(response_json) if success_data is None: return content = success_data.get("content") if content is None or content.strip() == "": return widget_content = success_data.get("widget_content") bot_id = event["user_profile_id"] message_info = event["message"] response_data = dict(content=content, widget_content=widget_content) send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data) def do_rest_call( base_url: str, event: Dict[str, Any], service_handler: OutgoingWebhookServiceInterface, ) -> Optional[Response]: """Returns response of call if no exception occurs.""" try: start_time = perf_counter() bot_profile = service_handler.user_profile response = service_handler.make_request( base_url, event, bot_profile.realm, ) logging.info( "Outgoing webhook request from %s@%s took %f seconds", bot_profile.id, bot_profile.realm.string_id, perf_counter() - start_time, ) if response is None: return None if str(response.status_code).startswith("2"): try: process_success_response(event, service_handler, response) except JsonableError as e: response_message = e.msg logging.info("Outhook trigger failed:", stack_info=True) fail_with_message(event, response_message) response_message = f"The outgoing webhook server attempted to send a message in Zulip, but that request resulted in the following error:\n> {e}" notify_bot_owner( event, response_content=response.text, failure_message=response_message ) return None else: logging.warning( "Message %(message_url)s triggered an outgoing webhook, returning status " 'code %(status_code)s.\n Content of response (in quotes): "' '%(response)s"', { "message_url": get_message_url(event), "status_code": response.status_code, "response": response.text, }, ) failure_message = f"Third party responded with {response.status_code}" fail_with_message(event, failure_message) notify_bot_owner(event, response.status_code, response.content) return response except requests.exceptions.Timeout: logging.info( "Trigger event %s on %s timed out. Retrying", event["command"], event["service_name"], ) failure_message = ( f"Request timed out after {settings.OUTGOING_WEBHOOK_TIMEOUT_SECONDS} seconds." ) request_retry(event, failure_message=failure_message) return None except requests.exceptions.ConnectionError: logging.info( "Trigger event %s on %s resulted in a connection error. Retrying", event["command"], event["service_name"], ) failure_message = "A connection error occurred. Is my bot server down?" request_retry(event, failure_message=failure_message) return None except requests.exceptions.RequestException as e: response_message = ( f"An exception of type *{type(e).__name__}* occurred for message `{event['command']}`! " "See the Zulip server logs for more information." ) logging.exception("Outhook trigger failed:", stack_info=True) fail_with_message(event, response_message) notify_bot_owner(event, exception=e) return None