From e46cbaffa2cdd055e2a52f971f7e28d58d89c1f7 Mon Sep 17 00:00:00 2001 From: Rishi Gupta Date: Wed, 3 May 2017 17:06:31 -0700 Subject: [PATCH] email: Remove Mandrill pathways and dependency. Everything it was doing (send_future_email) can now be done using ScheduledJob. --- requirements/common.txt | 3 - version.py | 2 +- zerver/decorator.py | 14 -- zerver/lib/digest.py | 2 +- zerver/lib/mandrill_client.py | 17 --- zerver/lib/notifications.py | 124 +++--------------- zerver/management/commands/deliver_email.py | 2 +- .../commands/print_email_delivery_backlog.py | 1 - zerver/models.py | 3 - zproject/local_settings.py | 1 - zproject/settings.py | 3 - 11 files changed, 23 insertions(+), 149 deletions(-) delete mode 100644 zerver/lib/mandrill_client.py diff --git a/requirements/common.txt b/requirements/common.txt index a7fbc5e226..3032234ca7 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -77,9 +77,6 @@ httplib2==0.10.3 # Needed for JWT-based auth PyJWT==1.5.0 -# Needed for USING_MANDRILL option for outgoing email -mandrill==1.0.57 - # Needed for including other markdown files for user docs markdown-include==0.5.1 diff --git a/version.py b/version.py index 1174526478..9099a73eb0 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.5.1+git" -PROVISION_VERSION = '4.24' +PROVISION_VERSION = '5.0' diff --git a/zerver/decorator.py b/zerver/decorator.py index a456a13a4a..6816ddbb3d 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -27,7 +27,6 @@ import datetime import logging import cProfile from io import BytesIO -from zerver.lib.mandrill_client import get_mandrill_client from six.moves import zip, urllib from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text, cast @@ -694,19 +693,6 @@ def profiled(func): return retval return wrapped_func # type: ignore # https://github.com/python/mypy/issues/1927 -def uses_mandrill(func): - # type: (FuncT) -> FuncT - """ - This decorator takes a function with keyword argument "mail_client" and - fills it in with the mail_client for the Mandrill account. - """ - @wraps(func) - def wrapped_func(*args, **kwargs): - # type: (*Any, **Any) -> Any - kwargs['mail_client'] = get_mandrill_client() - return func(*args, **kwargs) - return wrapped_func # type: ignore # https://github.com/python/mypy/issues/1927 - def return_success_on_head_request(view_func): # type: (Callable) -> Callable @wraps(view_func) diff --git a/zerver/lib/digest.py b/zerver/lib/digest.py index 5c0ef4a16a..704190e85a 100644 --- a/zerver/lib/digest.py +++ b/zerver/lib/digest.py @@ -211,6 +211,6 @@ def handle_digest_email(user_profile_id, cutoff): template_payload["hot_conversations"], new_streams_count, new_users_count): logger.info("Sending digest email for %s" % (user_profile.email,)) - # Send now, through Mandrill + # Send now, as a ScheduledJob send_future_email('zerver/emails/digest', recipients, sender=sender, context=template_payload, tags=["digest-emails"]) diff --git a/zerver/lib/mandrill_client.py b/zerver/lib/mandrill_client.py deleted file mode 100644 index 0b3e8f4716..0000000000 --- a/zerver/lib/mandrill_client.py +++ /dev/null @@ -1,17 +0,0 @@ -import mandrill -from django.conf import settings - -MAIL_CLIENT = None - -from typing import Optional - -def get_mandrill_client(): - # type: () -> Optional[mandrill.Mandrill] - if settings.MANDRILL_API_KEY is None or settings.DEVELOPMENT: - return None - - global MAIL_CLIENT - if not MAIL_CLIENT: - MAIL_CLIENT = mandrill.Mandrill(settings.MANDRILL_API_KEY) - - return MAIL_CLIENT diff --git a/zerver/lib/notifications.py b/zerver/lib/notifications.py index 3f4b6a0430..d6d5af000b 100644 --- a/zerver/lib/notifications.py +++ b/zerver/lib/notifications.py @@ -2,13 +2,12 @@ from __future__ import print_function from typing import cast, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Text -import mandrill from confirmation.models import Confirmation from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.template import loader from django.utils.timezone import now as timezone_now -from zerver.decorator import statsd_increment, uses_mandrill +from zerver.decorator import statsd_increment from zerver.lib.queue import queue_json_publish from zerver.models import ( Recipient, @@ -365,31 +364,13 @@ def handle_missedmessage_emails(user_profile_id, missed_email_events): message_count_by_recipient_subject[recipient_subject], ) -@uses_mandrill -def clear_followup_emails_queue(email, mail_client=None): - # type: (Text, Optional[mandrill.Mandrill]) -> None +def clear_followup_emails_queue(email): + # type: (Text) -> None """ - Clear out queued emails (from Mandrill's queue) that would otherwise - be sent to a specific email address. Optionally specify which sender - to filter by (useful when there are more Zulip subsystems using our - mandrill account). - - `email` is a string representing the recipient email - `from_email` is a string representing the email account used - to send the email (E.g. support@example.com). + Clear out queued emails that would otherwise be sent to a specific email address. """ - # SMTP mail delivery implementation - if not mail_client: - items = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, filter_string__iexact = email) - items.delete() - return - - # Mandrill implementation - for email_message in mail_client.messages.list_scheduled(to=email): - result = mail_client.messages.cancel_scheduled(id=email_message["_id"]) - if result.get("status") == "error": - print(result.get("name"), result.get("error")) - return + items = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, filter_string__iexact = email) + items.delete() def log_digest_event(msg): # type: (Text) -> None @@ -397,93 +378,28 @@ def log_digest_event(msg): logging.basicConfig(filename=settings.DIGEST_LOG_PATH, level=logging.INFO) logging.info(msg) -@uses_mandrill def send_future_email(template_prefix, recipients, sender=None, context={}, - delay=datetime.timedelta(0), tags=[], mail_client=None): - # type: (str, List[Dict[str, Any]], Optional[Dict[str, Text]], Dict[str, Any], datetime.timedelta, Iterable[Text], Optional[mandrill.Mandrill]) -> None - """ - Sends email via Mandrill, with optional delay - - 'mail_client' is filled in by the decorator - """ - # When sending real emails while testing locally, don't accidentally send - # emails to non-zulip.com users. - if settings.DEVELOPMENT and \ - settings.EMAIL_BACKEND != 'django.core.mail.backends.console.EmailBackend': - for recipient in recipients: - email = recipient.get("email") - if get_user_profile_by_email(email).realm.string_id != "zulip": - raise ValueError("digest: refusing to send emails to non-zulip.com users.") - + delay=datetime.timedelta(0), tags=[]): + # type: (str, List[Dict[str, Any]], Optional[Dict[str, Text]], Dict[str, Any], datetime.timedelta, Iterable[Text]) -> None subject = loader.render_to_string(template_prefix + '.subject', context).strip() email_text = loader.render_to_string(template_prefix + '.txt', context) email_html = loader.render_to_string(template_prefix + '.html', context) # SMTP mail delivery implementation - if not mail_client: - if sender is None: - # This may likely overridden by settings.DEFAULT_FROM_EMAIL - sender = {'email': settings.NOREPLY_EMAIL_ADDRESS, 'name': 'Zulip'} - for recipient in recipients: - email_fields = {'email_html': email_html, - 'email_subject': subject, - 'email_text': email_text, - 'recipient_email': recipient.get('email'), - 'recipient_name': recipient.get('name'), - 'sender_email': sender['email'], - 'sender_name': sender['name']} - ScheduledJob.objects.create(type=ScheduledJob.EMAIL, filter_string=recipient.get('email'), - data=ujson.dumps(email_fields), - scheduled_timestamp=timezone_now() + delay) - return - - # Mandrill implementation if sender is None: + # This may likely overridden by settings.DEFAULT_FROM_EMAIL sender = {'email': settings.NOREPLY_EMAIL_ADDRESS, 'name': 'Zulip'} - - message = {'from_email': sender['email'], - 'from_name': sender['name'], - 'to': recipients, - 'subject': subject, - 'html': email_html, - 'text': email_text, - 'tags': tags, - } - # ignore any delays smaller than 1-minute because it's cheaper just to sent them immediately - if not isinstance(delay, datetime.timedelta): - raise TypeError("specified delay is of the wrong type: %s" % (type(delay),)) - # Note: In the next section we hackishly use **{"async": False} to - # work around https://github.com/python/mypy/issues/2959 "# type: ignore" doesn't work - if delay < datetime.timedelta(minutes=1): - results = mail_client.messages.send(message=message, ip_pool="Main Pool", **{"async": False}) - else: - send_time = (timezone_now() + delay).__format__("%Y-%m-%d %H:%M:%S") - results = mail_client.messages.send(message=message, ip_pool="Main Pool", - send_at=send_time, **{"async": False}) - problems = [result for result in results if (result['status'] in ('rejected', 'invalid'))] - - if problems: - for problem in problems: - if problem["status"] == "rejected": - if problem["reject_reason"] == "hard-bounce": - # A hard bounce means the address doesn't exist or the - # recipient mail server is completely blocking - # delivery. Don't try to send further emails. - if "digest-emails" in tags: - from zerver.lib.actions import do_change_enable_digest_emails - bounce_email = problem["email"] - user_profile = get_user_profile_by_email(bounce_email) - do_change_enable_digest_emails(user_profile, False) - log_digest_event("%s\nTurned off digest emails for %s" % ( - str(problems), bounce_email)) - continue - elif problem["reject_reason"] == "soft-bounce": - # A soft bounce is temporary; let it try to resolve itself. - continue - raise Exception( - "While sending email (%s), encountered problems with these recipients: %r" - % (subject, problems)) - return + for recipient in recipients: + email_fields = {'email_html': email_html, + 'email_subject': subject, + 'email_text': email_text, + 'recipient_email': recipient.get('email'), + 'recipient_name': recipient.get('name'), + 'sender_email': sender['email'], + 'sender_name': sender['name']} + ScheduledJob.objects.create(type=ScheduledJob.EMAIL, filter_string=recipient.get('email'), + data=ujson.dumps(email_fields), + scheduled_timestamp=timezone_now() + delay) def enqueue_welcome_emails(email, name): # type: (Text, Text) -> None diff --git a/zerver/management/commands/deliver_email.py b/zerver/management/commands/deliver_email.py index 6b703be71b..6b7410f433 100755 --- a/zerver/management/commands/deliver_email.py +++ b/zerver/management/commands/deliver_email.py @@ -69,7 +69,7 @@ class Command(BaseCommand): help = """Deliver emails queued by various parts of Zulip (either for immediate sending or sending at a specified time). -Run this command under supervisor. We use Mandrill for zulip.com; this is for SMTP email delivery. +Run this command under supervisor. This is for SMTP email delivery. Usage: ./manage.py deliver_email """ diff --git a/zerver/management/commands/print_email_delivery_backlog.py b/zerver/management/commands/print_email_delivery_backlog.py index ab01931559..6321978a8d 100755 --- a/zerver/management/commands/print_email_delivery_backlog.py +++ b/zerver/management/commands/print_email_delivery_backlog.py @@ -21,7 +21,6 @@ class Command(BaseCommand): (The number of currently overdue (by at least a minute) email jobs) This is run as part of the nagios health check for the deliver_email command. -Please note that this is only relevant to the SMTP-based email delivery (no Mandrill). Usage: ./manage.py print_email_delivery_backlog """ diff --git a/zerver/models.py b/zerver/models.py index be6b3c501b..607afb3c51 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -1621,9 +1621,6 @@ class Referral(models.Model): email = models.EmailField(blank=False, null=False) # type: Text timestamp = models.DateTimeField(auto_now_add=True, null=False) # type: datetime.datetime -# This table only gets used on Zulip Voyager instances -# For reasons of deliverability (and sending from multiple email addresses), -# we will still send from mandrill when we send things from the (staging.)zulip.com install class ScheduledJob(models.Model): scheduled_timestamp = models.DateTimeField(auto_now_add=False, null=False) # type: datetime.datetime type = models.PositiveSmallIntegerField() # type: int diff --git a/zproject/local_settings.py b/zproject/local_settings.py index b7d7ff2b00..841001377a 100644 --- a/zproject/local_settings.py +++ b/zproject/local_settings.py @@ -47,7 +47,6 @@ EMAIL_HOST_USER = 'zulip@zulip.com' EMAIL_PORT = 587 EMAIL_USE_TLS = True -# We use mandrill, so this doesn't actually get used on our hosted deployment DEFAULT_FROM_EMAIL = "Zulip " # The noreply address to be used as Reply-To for certain generated emails. NOREPLY_EMAIL_ADDRESS = "noreply@zulip.com" diff --git a/zproject/settings.py b/zproject/settings.py index f89cbc6744..46253aa632 100644 --- a/zproject/settings.py +++ b/zproject/settings.py @@ -537,9 +537,6 @@ DROPBOX_APP_KEY = get_secret("dropbox_app_key") MAILCHIMP_API_KEY = get_secret("mailchimp_api_key") -# This comes from our mandrill accounts page -MANDRILL_API_KEY = get_secret("mandrill_api_key") - # Twitter API credentials # Secrecy not required because its only used for R/O requests. # Please don't make us go over our rate limit.