email: Remove Mandrill pathways and dependency.

Everything it was doing (send_future_email) can now be done using
ScheduledJob.

requirements/common.txt

@@ -77,9 +77,6 @@ httplib2==0.10.3
 # Needed for JWT-based auth
 PyJWT==1.5.0
 
-# Needed for USING_MANDRILL option for outgoing email
-mandrill==1.0.57
-
 # Needed for including other markdown files for user docs
 markdown-include==0.5.1
 

version.py

@@ -1,2 +1,2 @@
 ZULIP_VERSION = "1.5.1+git"
-PROVISION_VERSION = '4.24'
+PROVISION_VERSION = '5.0'

zerver/decorator.py

@@ -27,7 +27,6 @@ import datetime
 import logging
 import cProfile
 from io import BytesIO
-from zerver.lib.mandrill_client import get_mandrill_client
 from six.moves import zip, urllib
 
 from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Text, cast
@@ -694,19 +693,6 @@ def profiled(func):
         return retval
     return wrapped_func # type: ignore # https://github.com/python/mypy/issues/1927
 
-def uses_mandrill(func):
-    # type: (FuncT) -> FuncT
-    """
-    This decorator takes a function with keyword argument "mail_client" and
-    fills it in with the mail_client for the Mandrill account.
-    """
-    @wraps(func)
-    def wrapped_func(*args, **kwargs):
-        # type: (*Any, **Any) -> Any
-        kwargs['mail_client'] = get_mandrill_client()
-        return func(*args, **kwargs)
-    return wrapped_func # type: ignore # https://github.com/python/mypy/issues/1927
-
 def return_success_on_head_request(view_func):
     # type: (Callable) -> Callable
     @wraps(view_func)

zerver/lib/digest.py

@@ -211,6 +211,6 @@ def handle_digest_email(user_profile_id, cutoff):
                       template_payload["hot_conversations"],
                       new_streams_count, new_users_count):
         logger.info("Sending digest email for %s" % (user_profile.email,))
-        # Send now, through Mandrill
+        # Send now, as a ScheduledJob
         send_future_email('zerver/emails/digest', recipients, sender=sender,
                           context=template_payload, tags=["digest-emails"])

zerver/lib/mandrill_client.py

@@ -1,17 +0,0 @@
-import mandrill
-from django.conf import settings
-
-MAIL_CLIENT = None
-
-from typing import Optional
-
-def get_mandrill_client():
-    # type: () -> Optional[mandrill.Mandrill]
-    if settings.MANDRILL_API_KEY is None or settings.DEVELOPMENT:
-        return None
-
-    global MAIL_CLIENT
-    if not MAIL_CLIENT:
-        MAIL_CLIENT = mandrill.Mandrill(settings.MANDRILL_API_KEY)
-
-    return MAIL_CLIENT

zerver/lib/notifications.py

@@ -2,13 +2,12 @@ from __future__ import print_function
 
 from typing import cast, Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Text
 
-import mandrill
 from confirmation.models import Confirmation
 from django.conf import settings
 from django.core.mail import EmailMultiAlternatives
 from django.template import loader
 from django.utils.timezone import now as timezone_now
-from zerver.decorator import statsd_increment, uses_mandrill
+from zerver.decorator import statsd_increment
 from zerver.lib.queue import queue_json_publish
 from zerver.models import (
     Recipient,
@@ -365,31 +364,13 @@ def handle_missedmessage_emails(user_profile_id, missed_email_events):
             message_count_by_recipient_subject[recipient_subject],
         )
 
-@uses_mandrill
+def clear_followup_emails_queue(email):
-def clear_followup_emails_queue(email, mail_client=None):
+    # type: (Text) -> None
-    # type: (Text, Optional[mandrill.Mandrill]) -> None
     """
-    Clear out queued emails (from Mandrill's queue) that would otherwise
+    Clear out queued emails that would otherwise be sent to a specific email address.
-    be sent to a specific email address. Optionally specify which sender
-    to filter by (useful when there are more Zulip subsystems using our
-    mandrill account).
-
-    `email` is a string representing the recipient email
-    `from_email` is a string representing the email account used
-    to send the email (E.g. support@example.com).
     """
-    # SMTP mail delivery implementation
+    items = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, filter_string__iexact = email)
-    if not mail_client:
+    items.delete()
-        items = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, filter_string__iexact = email)
-        items.delete()
-        return
-
-    # Mandrill implementation
-    for email_message in mail_client.messages.list_scheduled(to=email):
-        result = mail_client.messages.cancel_scheduled(id=email_message["_id"])
-        if result.get("status") == "error":
-            print(result.get("name"), result.get("error"))
-    return
 
 def log_digest_event(msg):
     # type: (Text) -> None
@@ -397,93 +378,28 @@ def log_digest_event(msg):
     logging.basicConfig(filename=settings.DIGEST_LOG_PATH, level=logging.INFO)
     logging.info(msg)
 
-@uses_mandrill
 def send_future_email(template_prefix, recipients, sender=None, context={},
-                      delay=datetime.timedelta(0), tags=[], mail_client=None):
+                      delay=datetime.timedelta(0), tags=[]):
-    # type: (str, List[Dict[str, Any]], Optional[Dict[str, Text]], Dict[str, Any], datetime.timedelta, Iterable[Text], Optional[mandrill.Mandrill]) -> None
+    # type: (str, List[Dict[str, Any]], Optional[Dict[str, Text]], Dict[str, Any], datetime.timedelta, Iterable[Text]) -> None
-    """
-    Sends email via Mandrill, with optional delay
-
-    'mail_client' is filled in by the decorator
-    """
-    # When sending real emails while testing locally, don't accidentally send
-    # emails to non-zulip.com users.
-    if settings.DEVELOPMENT and \
-       settings.EMAIL_BACKEND != 'django.core.mail.backends.console.EmailBackend':
-        for recipient in recipients:
-            email = recipient.get("email")
-            if get_user_profile_by_email(email).realm.string_id != "zulip":
-                raise ValueError("digest: refusing to send emails to non-zulip.com users.")
-
     subject = loader.render_to_string(template_prefix + '.subject', context).strip()
     email_text = loader.render_to_string(template_prefix + '.txt', context)
     email_html = loader.render_to_string(template_prefix + '.html', context)
 
     # SMTP mail delivery implementation
-    if not mail_client:
-        if sender is None:
-            # This may likely overridden by settings.DEFAULT_FROM_EMAIL
-            sender = {'email': settings.NOREPLY_EMAIL_ADDRESS, 'name': 'Zulip'}
-        for recipient in recipients:
-            email_fields = {'email_html': email_html,
-                            'email_subject': subject,
-                            'email_text': email_text,
-                            'recipient_email': recipient.get('email'),
-                            'recipient_name': recipient.get('name'),
-                            'sender_email': sender['email'],
-                            'sender_name': sender['name']}
-            ScheduledJob.objects.create(type=ScheduledJob.EMAIL, filter_string=recipient.get('email'),
-                                        data=ujson.dumps(email_fields),
-                                        scheduled_timestamp=timezone_now() + delay)
-        return
-
-    # Mandrill implementation
     if sender is None:
+        # This may likely overridden by settings.DEFAULT_FROM_EMAIL
         sender = {'email': settings.NOREPLY_EMAIL_ADDRESS, 'name': 'Zulip'}
-
+    for recipient in recipients:
-    message = {'from_email': sender['email'],
+        email_fields = {'email_html': email_html,
-               'from_name': sender['name'],
+                        'email_subject': subject,
-               'to': recipients,
+                        'email_text': email_text,
-               'subject': subject,
+                        'recipient_email': recipient.get('email'),
-               'html': email_html,
+                        'recipient_name': recipient.get('name'),
-               'text': email_text,
+                        'sender_email': sender['email'],
-               'tags': tags,
+                        'sender_name': sender['name']}
-               }
+        ScheduledJob.objects.create(type=ScheduledJob.EMAIL, filter_string=recipient.get('email'),
-    # ignore any delays smaller than 1-minute because it's cheaper just to sent them immediately
+                                    data=ujson.dumps(email_fields),
-    if not isinstance(delay, datetime.timedelta):
+                                    scheduled_timestamp=timezone_now() + delay)
-        raise TypeError("specified delay is of the wrong type: %s" % (type(delay),))
-    # Note: In the next section we hackishly use **{"async": False} to
-    # work around https://github.com/python/mypy/issues/2959 "# type: ignore" doesn't work
-    if delay < datetime.timedelta(minutes=1):
-        results = mail_client.messages.send(message=message, ip_pool="Main Pool", **{"async": False})
-    else:
-        send_time = (timezone_now() + delay).__format__("%Y-%m-%d %H:%M:%S")
-        results = mail_client.messages.send(message=message, ip_pool="Main Pool",
-                                            send_at=send_time, **{"async": False})
-    problems = [result for result in results if (result['status'] in ('rejected', 'invalid'))]
-
-    if problems:
-        for problem in problems:
-            if problem["status"] == "rejected":
-                if problem["reject_reason"] == "hard-bounce":
-                    # A hard bounce means the address doesn't exist or the
-                    # recipient mail server is completely blocking
-                    # delivery. Don't try to send further emails.
-                    if "digest-emails" in tags:
-                        from zerver.lib.actions import do_change_enable_digest_emails
-                        bounce_email = problem["email"]
-                        user_profile = get_user_profile_by_email(bounce_email)
-                        do_change_enable_digest_emails(user_profile, False)
-                        log_digest_event("%s\nTurned off digest emails for %s" % (
-                            str(problems), bounce_email))
-                        continue
-                elif problem["reject_reason"] == "soft-bounce":
-                    # A soft bounce is temporary; let it try to resolve itself.
-                    continue
-            raise Exception(
-                "While sending email (%s), encountered problems with these recipients: %r"
-                % (subject, problems))
-    return
 
 def enqueue_welcome_emails(email, name):
     # type: (Text, Text) -> None

zerver/management/commands/deliver_email.py

@@ -69,7 +69,7 @@ class Command(BaseCommand):
     help = """Deliver emails queued by various parts of Zulip
 (either for immediate sending or sending at a specified time).
 
-Run this command under supervisor. We use Mandrill for zulip.com; this is for SMTP email delivery.
+Run this command under supervisor. This is for SMTP email delivery.
 
 Usage: ./manage.py deliver_email
 """

zerver/management/commands/print_email_delivery_backlog.py

@@ -21,7 +21,6 @@ class Command(BaseCommand):
 (The number of currently overdue (by at least a minute) email jobs)
 
 This is run as part of the nagios health check for the deliver_email command.
-Please note that this is only relevant to the SMTP-based email delivery (no Mandrill).
 
 Usage: ./manage.py print_email_delivery_backlog
 """

zerver/models.py

@@ -1621,9 +1621,6 @@ class Referral(models.Model):
     email = models.EmailField(blank=False, null=False) # type: Text
     timestamp = models.DateTimeField(auto_now_add=True, null=False) # type: datetime.datetime
 
-# This table only gets used on Zulip Voyager instances
-# For reasons of deliverability (and sending from multiple email addresses),
-# we will still send from mandrill when we send things from the (staging.)zulip.com install
 class ScheduledJob(models.Model):
     scheduled_timestamp = models.DateTimeField(auto_now_add=False, null=False) # type: datetime.datetime
     type = models.PositiveSmallIntegerField() # type: int

zproject/local_settings.py

@@ -47,7 +47,6 @@ EMAIL_HOST_USER = 'zulip@zulip.com'
 EMAIL_PORT = 587
 EMAIL_USE_TLS = True
 
-# We use mandrill, so this doesn't actually get used on our hosted deployment
 DEFAULT_FROM_EMAIL = "Zulip <zulip@zulip.com>"
 # The noreply address to be used as Reply-To for certain generated emails.
 NOREPLY_EMAIL_ADDRESS = "noreply@zulip.com"

zproject/settings.py

@@ -537,9 +537,6 @@ DROPBOX_APP_KEY = get_secret("dropbox_app_key")
 
 MAILCHIMP_API_KEY = get_secret("mailchimp_api_key")
 
-# This comes from our mandrill accounts page
-MANDRILL_API_KEY = get_secret("mandrill_api_key")
-
 # Twitter API credentials
 # Secrecy not required because its only used for R/O requests.
 # Please don't make us go over our rate limit.