emails: Add option to forward mails send in dev env to external email.

Fixes #7085.

docs/email.md

@@ -56,6 +56,23 @@ our custom backend, `EmailLogBackEnd`.  It does the following:
 * Print a friendly message on console advertising `/emails` to make
   this nice and discoverable.
 
+You can also forward all the emails sent in the development environment
+to an email id of your choice by clicking on **Forward emails to a mail
+account** in `/emails` page. This feature can be used for testing how
+emails gets rendered by different email clients. Before enabling this
+you have to first configure the following SMTP settings.
+
+* The hostname `EMAIL_HOST` in `zproject/dev_settings.py`
+* The username `EMAIL_HOST_USER` in `zproject/dev_settings.py`.
+* The password `email_password` in `zproject/dev-secrets.conf`.
+
+See [this](prod-email.html#free-outgoing-email-services)
+section for instructions on obtaining SMTP details.
+
+**Note: The base_image_uri of the images in forwarded emails would be replaced
+with `https://chat.zulip.org/static/images/emails` inorder for the email clients
+to render the images. See `zproject/email_backends.py` for more details.**
+
 While running the backend test suite, we use
 `django.core.mail.backends.locmem.EmailBackend` as the email
 backend. The `locmem` backend stores messages in a special attribute

templates/zerver/config_error.html

@@ -19,20 +19,29 @@
                     {% endif %}
 
                     {% if smtp_error %}
-                    <p>
-                        It appears there are problems with the
-                        email configuration.
-                    </p>
-                    <p>
-                        See <code>/var/log/zulip/errors.log</code> for more
-                        details on the error.
-                    </p>
-                    <p>
-                        You may also want to test your email configuration,
-                        as described in the
-                        <a href="https://zulip.readthedocs.io/en/latest/prod-email.html">
-                        Production installation docs</a>.
-                    </p>
+                        <p>
+                            It appears there are problems with the
+                            email configuration.
+                        </p>
+                        {% if not development_environment %}
+                        <p>
+                            See <code>/var/log/zulip/errors.log</code> for more
+                            details on the error.
+                        </p>
+                        <p>
+                            You may also want to test your email configuration,
+                            as described in the
+                            <a href="https://zulip.readthedocs.io/en/latest/prod-email.html">
+                            Production installation docs</a>.
+                        </p>
+                        {% else %}
+                        <p>
+                            Please have a look at our
+                            <a target="_blank" href="https://zulip.readthedocs.io/en/latest/email.html#development-and-testing">
+                            setup guide</a> for forwarding emails sent in development
+                            environment to an email account.
+                        </p>
+                        {% endif %}
                     {% endif %}
 
                     {% if google_error %}
@@ -43,22 +52,24 @@
                     {{ render_markdown_path('zerver/github-error.md', {"root_domain_uri": root_domain_uri, "settings_path": settings_path, "secrets_path": secrets_path}) }}
                     {% endif %}
 
+                    {% if google_error or github_error %}
+                        {% if development_environment %}
+                        <p>
+                            For more information, have a look at
+                            the <a href="http://zulip.readthedocs.io/en/latest/settings.html#testing-google-github-authentication">authentication
+                            setup guide</a> for the development environment.
+                        </p>
+                        {% else %}
+                        <p>
+                            For more information, have a look at
+                            our <a href="http://zulip.readthedocs.io/en/latest/prod-authentication-methods.html">authentication
+                            setup guide</a> and the comments in <code>{{ settings_comments_path }}</code>.
+                        </p>
+                        {% endif %}
+                    {% endif %}
+
                     <p>After making your changes, remember to restart
                     the Zulip server.</p>
-
-                    {% if development %}
-                    <p>
-                        For more information, have a look at
-                        the <a href="http://zulip.readthedocs.io/en/latest/settings.html#testing-google-github-authentication">authentication
-                        setup guide</a> for the development environment.
-                    </p>
-                    {% else %}
-                    <p>
-                        For more information, have a look at
-                        our <a href="http://zulip.readthedocs.io/en/latest/prod-authentication-methods.html">authentication
-                        setup guide</a> and the comments in <code>{{ settings_comments_path }}</code>.
-                    </p>
-                    {% endif %}
                 </div>
 
             </div>

templates/zerver/email_log.html

@@ -12,6 +12,51 @@
                 <input type="checkbox" id="toggle"/>
                 <strong>Show text only version</strong>
             </label>
+            <a href="#" data-toggle="modal" data-target="#forward_email_modal">
+                <strong>Forward emails to a mail account</strong>
+            </a>
+        </div>
+    </div>
+    <div style="padding-top:100px">
+        {{ log |safe }}
+    </div>
+    <div id="forward_email_modal" class="modal hide" tabindex="-1" role="dialog" aria-hidden="true">
+        <div class="modal-body">
+            <div class="input-group">
+                <form id="smtp_form">
+                    {{ csrf_input }}
+                    <div class="alert alert-info"
+                        id="smtp_form_status" style="display:none;">
+                        Updated successfully.
+                    </div>
+                    <label for="forward">
+                        <strong>Forwards all emails sent in the
+                            development environment to an external
+                            mail account.
+                        </strong>
+                    </label>
+                    <label class="radio">
+                        <input name="forward" type="radio" value="enabled" {% if forward_address %}checked{% endif %}/>Yes
+                    </label>
+                    <label class="radio">
+                        <input name="forward" type="radio" value="disabled" {% if not forward_address %}checked{% endif %}/>No
+                    </label>
+                    <div id="forward_address_sections" {% if not forward_address %}style="display:none;"{% endif %}>
+                        <label for="forward_address"><strong>Address to which emails should be forwarded</strong></label>
+                        <input type="text" id="address" name="forward_address" placeholder="eg: your-email@example.com" value="{{forward_address}}"/>
+                    </div>
+                    <br/>
+                    <div class="alert alert-info">
+                        You must setup SMTP as described
+                        <a target="_blank" href="https://zulip.readthedocs.io/en/latest/email.html#development-and-testing">
+                        here</a> first before enabling this.
+                    </div>
+                </form>
+            </div>
+        </div>
+        <div class="modal-footer">
+            <button id='save_smptp_details'>Update</button>
+            <button data-dismiss="modal">Close</button>
         </div>
     </div>
     <script type="text/javascript">
@@ -33,9 +78,24 @@
             });
         }
     });
+    $('input[type=radio][name=forward]').on('change', function() {
+        if ($(this).val() == "enabled") {
+            $("#forward_address_sections").show();
+        } else {
+            $("#forward_address_sections").hide();
+        }
+    });
+    $("#save_smptp_details").on("click", function() {
+        var address = $('input[name=forward]:checked').val() == "enabled" ? $("#address").val(): "";
+        var csrf_token = $('input[name="csrfmiddlewaretoken"]').attr('value');
+        var data = {"forward_address": address, "csrfmiddlewaretoken": csrf_token};
+        $.post("/emails/", data, function() {
+            $("#smtp_form_status").show();
+            setTimeout(function() {
+                $("#smtp_form_status").hide();
+            }, 3000);
+        });
+    });
     </script>
-    <div style="padding-top:100px">
-        {{ log |safe }}
-    </div>
 </div>
 {% endblock %}

tools/linter_lib/custom_check.py

@@ -468,7 +468,8 @@ def build_custom_checkers(by_lang):
                            '<td><input type="text" class="new-realm-domain" placeholder="acme.com"></input></td>')],
          'exclude': set(["static/templates/settings/emoji-settings-admin.handlebars",
                          "static/templates/settings/realm-filter-settings-admin.handlebars",
-                         "static/templates/settings/bot-settings.handlebars"])},
+                         "static/templates/settings/bot-settings.handlebars",
+                         "templates/zerver/email_log.html"])},
         {'pattern': "placeholder='[^{]",
          'description': "`placeholder` value should be translatable."},
         {'pattern': "aria-label='[^{]",

zerver/tests/test_docs.py

@@ -73,21 +73,8 @@ class DocPageTest(ZulipTestCase):
         self._test('/devtools/', 'Useful development URLs')
         self._test('/errors/404/', 'Page not found')
         self._test('/errors/5xx/', 'Internal server error')
-
-        with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \
-                mock.patch('logging.info', return_value=None):
-            # For reaching full coverage for clear_emails function
-            result = self.client_get('/emails/clear/')
-            self.assertEqual(result.status_code, 302)
-            result = self.client_get(result['Location'])
-            self.assertIn('manually generate most of the emails by clicking', str(result.content))
-
-            result = self.client_get('/emails/generate/')
-            self.assertEqual(result.status_code, 302)
-            self.assertIn('emails', result['Location'])
-
-            self._test('/emails/', 'manually generate most of the emails by clicking')
-            self._test('/register/', 'Sign up for Zulip')
+        self._test('/emails/', 'manually generate most of the emails by clicking')
+        self._test('/register/', 'Sign up for Zulip')
 
         result = self.client_get('/integrations/doc-html/nonexistent_integration', follow=True)
         self.assertEqual(result.status_code, 404)
@@ -239,4 +226,4 @@ class ConfigErrorTest(ZulipTestCase):
         # type: () -> None
         result = self.client_get("/config-error/smtp")
         self.assertEqual(result.status_code, 200)
-        self.assert_in_success_response(["/var/log/zulip"], result)
+        self.assert_in_success_response(["email configuration"], result)

zerver/tests/test_email_log.py

@@ -0,0 +1,46 @@
+import os
+import mock
+from django.conf import settings
+
+from zerver.lib.test_classes import ZulipTestCase
+from zproject.email_backends import get_forward_address
+
+class EmailLogTest(ZulipTestCase):
+    def test_get_email_log_page(self):
+        # type: () -> None
+        result = self.client_get("/emails/")
+        self.assert_in_success_response(["All the emails sent in the Zulip"], result)
+
+    def test_clear_email_logs(self):
+        # type: () -> None
+        result = self.client_get('/emails/clear/')
+        self.assertEqual(result.status_code, 302)
+        result = self.client_get(result['Location'])
+        self.assertIn('manually generate most of the emails by clicking', str(result.content))
+
+    def test_generate_emails(self):
+        # type: () -> None
+        with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \
+                mock.patch('logging.info', return_value=None):
+            with mock.patch('zproject.email_backends.EmailLogBackEnd.send_email_smtp'):
+                result = self.client_get('/emails/generate/')
+                self.assertEqual(result.status_code, 302)
+                self.assertIn('emails', result['Location'])
+
+    def test_forward_address_details(self):
+        # type: () -> None
+        forward_address = "forward-to@example.com"
+        result = self.client_post("/emails/", {"forward_address": forward_address})
+        self.assert_json_success(result)
+
+        self.assertEqual(get_forward_address(), forward_address)
+
+        with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \
+                mock.patch('logging.info', return_value=None):
+            with mock.patch('zproject.email_backends.EmailLogBackEnd.send_email_smtp'):
+                result = self.client_get('/emails/generate/')
+                self.assertEqual(result.status_code, 302)
+                self.assertIn('emails', result['Location'])
+                result = self.client_get(result['Location'])
+                self.assert_in_success_response([forward_address], result)
+        os.remove(settings.FORWARD_ADDRESS_CONFIG_FILE)

zerver/views/email_log.py

@@ -3,10 +3,16 @@ from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render, redirect
 from django.test import Client
 from django.views.decorators.http import require_GET
+from django.views.decorators.csrf import csrf_exempt
 
 from zerver.models import get_realm, get_user
 from zerver.lib.notifications import enqueue_welcome_emails
-import urllib
+from zerver.lib.response import json_success
+from zproject.email_backends import (
+    get_forward_address,
+    set_forward_address,
+)
+from six.moves import urllib
 from confirmation.models import Confirmation, confirmation_url
 
 import os
@@ -17,12 +23,17 @@ client = Client()
 
 def email_page(request):
     # type: (HttpRequest) -> HttpResponse
+    if request.method == 'POST':
+        set_forward_address(request.POST["forward_address"])
+        return json_success()
     try:
         with open(settings.EMAIL_CONTENT_LOG_PATH, "r+") as f:
             content = f.read()
     except FileNotFoundError:
         content = ""
-    return render(request, 'zerver/email_log.html', {'log': content})
+    return render(request, 'zerver/email_log.html',
+                  {'log': content,
+                   'forward_address': get_forward_address()})
 
 def clear_emails(request):
     # type: (HttpRequest) -> HttpResponse

zproject/dev_settings.py

@@ -8,6 +8,7 @@ from typing import Set
 
 LOCAL_UPLOADS_DIR = 'var/uploads'
 EMAIL_LOG_DIR = "/var/log/zulip/email.log"
+FORWARD_ADDRESS_CONFIG_FILE = "var/forward_address.ini"
 # Check if test_settings.py set EXTERNAL_HOST.
 EXTERNAL_HOST = os.getenv('EXTERNAL_HOST')
 if EXTERNAL_HOST is None:
@@ -58,3 +59,8 @@ INLINE_URL_EMBED_PREVIEW = True
 # Don't require anything about password strength in development
 PASSWORD_MIN_LENGTH = 0
 PASSWORD_MIN_GUESSES = 0
+
+# SMTP settings for forwarding emails sent in development
+# environment to an email account.
+EMAIL_HOST = ""
+EMAIL_HOST_USER = ""

zproject/email_backends.py

@@ -1,13 +1,68 @@
 import logging
 
 from typing import List
+from six.moves import configparser
+
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 
 from django.conf import settings
 from django.core.mail.backends.base import BaseEmailBackend
 from django.core.mail import EmailMultiAlternatives
 from django.template import loader
 
+def get_forward_address():
+    # type: () -> str
+    config = configparser.ConfigParser()
+    config.read(settings.FORWARD_ADDRESS_CONFIG_FILE)
+    try:
+        return config.get("DEV_EMAIL", "forward_address")
+    except (configparser.NoSectionError, configparser.NoOptionError) as e:
+        return ""
+
+def set_forward_address(forward_address):
+    # type: (str) -> None
+    config = configparser.ConfigParser()
+    config.read(settings.FORWARD_ADDRESS_CONFIG_FILE)
+
+    if not config.has_section("DEV_EMAIL"):
+        config.add_section("DEV_EMAIL")
+    config.set("DEV_EMAIL", "forward_address", forward_address)
+
+    with open(settings.FORWARD_ADDRESS_CONFIG_FILE, "w") as cfgfile:
+            config.write(cfgfile)
+
 class EmailLogBackEnd(BaseEmailBackend):
+    def send_email_smtp(self, email):
+        # type: (EmailMultiAlternatives) -> None
+        from_email = email.from_email
+        to = get_forward_address()
+
+        msg = MIMEMultipart('alternative')
+        msg['Subject'] = email.subject
+        msg['From'] = from_email
+        msg['To'] = to
+
+        text = email.body
+        html = email.alternatives[0][0]
+
+        # Here, we replace the email addresses used in development
+        # with chat.zulip.org, so that web email providers like Gmail
+        # will be able to fetch the illustrations used in the emails.
+        localhost_email_images_base_uri = settings.ROOT_DOMAIN_URI + '/static/images/emails'
+        czo_email_images_base_uri = 'https://chat.zulip.org/static/images/emails'
+        html = html.replace(localhost_email_images_base_uri, czo_email_images_base_uri)
+
+        msg.attach(MIMEText(text, 'plain'))
+        msg.attach(MIMEText(html, 'html'))
+
+        smtp = smtplib.SMTP(settings.EMAIL_HOST)
+        smtp.starttls()
+        smtp.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
+        smtp.sendmail(from_email, to, msg.as_string())
+        smtp.quit()
+
     def log_email(self, email: EmailMultiAlternatives) -> None:
         """Used in development to record sent emails in a nice HTML log"""
         html_message = 'Missing HTML message'
@@ -38,6 +93,8 @@ class EmailLogBackEnd(BaseEmailBackend):
     def send_messages(self, email_messages: List[EmailMultiAlternatives]) -> int:
         for email in email_messages:
             self.log_email(email)
+            if get_forward_address():
+                self.send_email_smtp(email)
             email_log_url = settings.ROOT_DOMAIN_URI + "/emails"
             logging.info("Emails sent in development are available at %s" % (email_log_url,))
         return len(email_messages)