email_mirror: Add support for "+include-quotations" in address.

We add an option to disable the stripping of quotations from the email
body, if "+include-quotations" token is included in the email address.

zerver/lib/email_mirror.py

@@ -153,8 +153,8 @@ def mark_missed_message_address_as_used(address: str) -> None:
         raise ZulipEmailForwardError('Missed message address has already been used')
 
 def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False,
-                         remove_quotations: bool=True, include_footers: bool=False) -> str:
-    body = extract_body(message, remove_quotations)
+                         include_quotations: bool=False, include_footers: bool=False) -> str:
+    body = extract_body(message, include_quotations)
     # Remove null characters, since Zulip will reject
     body = body.replace("\x00", "")
     if not include_footers:
@@ -243,7 +243,7 @@ def get_message_part_by_type(message: message.Message, content_type: str) -> Opt
     return None
 
 talon_initialized = False
-def extract_body(message: message.Message, remove_quotations: bool=True) -> str:
+def extract_body(message: message.Message, include_quotations: bool=False) -> str:
     import talon
     global talon_initialized
     if not talon_initialized:
@@ -254,18 +254,18 @@ def extract_body(message: message.Message, remove_quotations: bool=True) -> str:
     # that.
     plaintext_content = get_message_part_by_type(message, "text/plain")
     if plaintext_content:
-        if remove_quotations:
-            return talon.quotations.extract_from_plain(plaintext_content)
-        else:
+        if include_quotations:
             return plaintext_content
+        else:
+            return talon.quotations.extract_from_plain(plaintext_content)
 
     # If we only have an HTML version, try to make that look nice.
     html_content = get_message_part_by_type(message, "text/html")
     if html_content:
-        if remove_quotations:
-            return convert_html_to_markdown(talon.quotations.extract_from_html(html_content))
-        else:
+        if include_quotations:
             return convert_html_to_markdown(html_content)
+        else:
+            return convert_html_to_markdown(talon.quotations.extract_from_html(html_content))
 
     if plaintext_content is not None or html_content is not None:
         raise ZulipEmailForwardUserError("Email has no nonempty body sections; ignoring.")
@@ -358,8 +358,10 @@ def process_stream_message(to: str, message: message.Message) -> None:
     subject = strip_from_subject(subject_header) or "(no topic)"
 
     stream, options = extract_and_validate(to)
-    # Don't remove quotations if message is forwarded:
-    options['remove_quotations'] = not is_forwarded(subject_header)
+    # Don't remove quotations if message is forwarded, unless otherwise specified:
+    if 'include_quotations' not in options:
+        options['include_quotations'] = is_forwarded(subject_header)
+
     body = construct_zulip_body(message, stream.realm, **options)
     send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body)
     logger.info("Successfully processed email to %s (%s)" % (

zerver/lib/email_mirror_helpers.py

@@ -7,7 +7,7 @@ from zerver.models import Stream
 
 from typing import Dict, Tuple
 
-optional_address_tokens = ["show-sender", "include-footers"]
+optional_address_tokens = ["show-sender", "include-footers", "include-quotations"]
 
 class ZulipEmailForwardError(Exception):
     pass

zerver/tests/test_email_mirror.py

@@ -59,9 +59,10 @@ from typing import Any, Callable, Dict, Mapping, Union, Optional
 
 class TestEncodeDecode(ZulipTestCase):
     def _assert_options(self, options: Dict[str, bool], show_sender: bool=False,
-                        include_footers: bool=False) -> None:
+                        include_footers: bool=False, include_quotations: bool=False) -> None:
         self.assertEqual(show_sender, ('show_sender' in options) and options['show_sender'])
         self.assertEqual(include_footers, ('include_footers' in options) and options['include_footers'])
+        self.assertEqual(include_quotations, ('include_quotations' in options) and options['include_quotations'])
 
     def test_encode_decode(self) -> None:
         realm = get_realm('zulip')
@@ -75,10 +76,10 @@ class TestEncodeDecode(ZulipTestCase):
         self.assertEqual(token, stream.email_token)
 
         parts = email_address.split('@')
-        parts[0] += "+include-footers+show-sender"
+        parts[0] += "+include-footers+show-sender+include-quotations"
         email_address_all_options = '@'.join(parts)
         token, options = decode_email_address(email_address_all_options)
-        self._assert_options(options, show_sender=True, include_footers=True)
+        self._assert_options(options, show_sender=True, include_footers=True, include_quotations=True)
         self.assertEqual(token, stream.email_token)
 
         email_address_dots = email_address.replace('+', '.')
@@ -88,7 +89,7 @@ class TestEncodeDecode(ZulipTestCase):
 
         email_address_dots_all_options = email_address_all_options.replace('+', '.')
         token, options = decode_email_address(email_address_dots_all_options)
-        self._assert_options(options, show_sender=True, include_footers=True)
+        self._assert_options(options, show_sender=True, include_footers=True, include_quotations=True)
         self.assertEqual(token, stream.email_token)
 
         email_address = email_address.replace('@testserver', '@zulip.org')
@@ -105,7 +106,7 @@ class TestEncodeDecode(ZulipTestCase):
             self.assertEqual(token, stream.email_token)
 
             token, options = decode_email_address(email_address_all_options)
-            self._assert_options(options, show_sender=True, include_footers=True)
+            self._assert_options(options, show_sender=True, include_footers=True, include_quotations=True)
             self.assertEqual(token, stream.email_token)
 
         with self.assertRaises(ZulipEmailForwardError):
@@ -343,6 +344,36 @@ class TestStreamEmailMessagesSuccess(ZulipTestCase):
         self.assertEqual(get_display_recipient(message.recipient), stream.name)
         self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
 
+    def test_receive_stream_email_include_quotations_success(self) -> None:
+        user_profile = self.example_user('hamlet')
+        self.login(user_profile.email)
+        self.subscribe(user_profile, "Denmark")
+        stream = get_stream("Denmark", user_profile.realm)
+
+        stream_to_address = encode_email_address(stream)
+        parts = stream_to_address.split('@')
+        parts[0] += "+include-quotations"
+        stream_to_address = '@'.join(parts)
+
+        text = """Reply
+
+        -----Original Message-----
+
+        Quote"""
+
+        incoming_valid_message = MIMEText(text)
+        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
+        incoming_valid_message['From'] = self.example_email('hamlet')
+        incoming_valid_message['To'] = stream_to_address
+        incoming_valid_message['Reply-to'] = self.example_email('othello')
+
+        process_message(incoming_valid_message)
+        message = most_recent_message(user_profile)
+
+        self.assertEqual(message.content, text)
+        self.assertEqual(get_display_recipient(message.recipient), stream.name)
+        self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
+
 class TestEmailMirrorMessagesWithAttachments(ZulipTestCase):
     def test_message_with_valid_attachment(self) -> None:
         user_profile = self.example_user('hamlet')