markdown: Reduce mentions inside blockquotes to silent-mentions.

On the backend, we extend the BlockQuoteProcessor's clean function that
just removes '>' from the start of each line to convert each mention to
have the silent mention syntax, before UserMentionPattern is invoked.

The frontend, however, has an edge case where if you are mentioned in
some message and you quote it while having mentioned yourself above
the quoted message, you wouldn't see the red highlight till we get the
final rendered message from the backend.

This is such a subtle glitch that it's likely not worth worrying about.

Fixes #8025.

frontend_tests/node_tests/markdown.js

@@ -304,6 +304,8 @@ run_test('marked', () => {
          expected: '<p>\u{1f6b2}</p>' },
         {input: 'Silent mention: _@**Cordelia Lear**',
          expected: '<p>Silent mention: <span class="user-mention silent" data-user-id="101">@Cordelia Lear</span></p>'},
+        {input: '> Mention in quote: @**Cordelia Lear**\n\nMention outside quote: @**Cordelia Lear**',
+         expected: '<blockquote>\n<p>Mention in quote: <span class="user-mention silent" data-user-id="101">@Cordelia Lear</span></p>\n</blockquote>\n<p>Mention outside quote: <span class="user-mention" data-user-id="101">@Cordelia Lear</span></p>'},
         // Test only those realm filters which don't return True for
         // `contains_backend_only_syntax()`. Those which return True
         // are tested separately.

static/js/markdown.js

@@ -105,6 +105,20 @@ exports.apply_markdown = function (message) {
             }
             return;
         },
+        silencedMentionHandler: function (quote) {
+            // Silence quoted mentions.
+            var user_mention_re = /<span.*user-mention.*data-user-id="(\d+|\*)"[^>]*>/gm;
+            quote = quote.replace(user_mention_re, function (match) {
+                return match.replace(/"user-mention"/g, '"user-mention silent"');
+            });
+            // In most cases, if you are being mentioned in the message you're quoting, you wouldn't
+            // mention yourself outside of the blockquote (and, above it). If that you do that, the
+            // following mentioned status is false; the backend rendering is authoritative and the
+            // only side effect is the lack red flash on immediately sending the message.
+            message.mentioned = false;
+            message.mentioned_me_directly = false;
+            return quote;
+        },
     };
     message.content = marked(message.raw_content + '\n\n', options).trim();
     message.is_me_message = exports.is_status_message(message.raw_content, message.content);

static/third/marked/lib/marked.js

@@ -988,6 +988,7 @@ Renderer.prototype.code = function(code, lang, escaped) {
 };
 
 Renderer.prototype.blockquote = function(quote) {
+  quote = this.options.silencedMentionHandler(quote);
   return '<blockquote>\n' + quote + '</blockquote>\n';
 };
 

zerver/lib/bugdown/__init__.py

@@ -1320,6 +1320,24 @@ class ListIndentProcessor(markdown.blockprocessors.ListIndentProcessor):
         super().__init__(parser)
         parser.markdown.tab_length = 4
 
+class BlockQuoteProcessor(markdown.blockprocessors.BlockQuoteProcessor):
+    """ Process BlockQuotes.
+
+        Based on markdown.blockprocessors.BlockQuoteProcessor, but with 2-space indent
+    """
+
+    # Original regex for blockquote is RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)')
+    RE = re.compile(r'(^|\n)(?!(?:[ ]{0,3}>\s*(?:$|\n))*(?:$|\n))'
+                    r'[ ]{0,3}>[ ]?(.*)')
+    mention_re = re.compile(mention.find_mentions)
+
+    def clean(self, line: str) -> str:
+        # Silence all the mentions inside blockquotes
+        line = re.sub(self.mention_re, lambda m: "_@{}".format(m.group('match')), line)
+
+        # And then run the upstream processor's code for removing the '>'
+        return super().clean(line)
+
 class BugdownUListPreprocessor(markdown.preprocessors.Preprocessor):
     """ Allows unordered list blocks that come directly after a
         paragraph to be rendered as an unordered list
@@ -1715,16 +1733,12 @@ class Bugdown(markdown.Extension):
             '>strong')
 
     def extend_block_formatting(self, md: markdown.Markdown) -> None:
-        for k in ('hashheader', 'setextheader', 'olist', 'ulist', 'indent'):
+        for k in ('hashheader', 'setextheader', 'olist', 'ulist', 'indent', 'quote'):
             del md.parser.blockprocessors[k]
 
         md.parser.blockprocessors.add('ulist', UListProcessor(md.parser), '>hr')
         md.parser.blockprocessors.add('indent', ListIndentProcessor(md.parser), '<ulist')
-
-        # Original regex for blockquote is RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)')
-        md.parser.blockprocessors['quote'].RE = re.compile(
-            r'(^|\n)(?!(?:[ ]{0,3}>\s*(?:$|\n))*(?:$|\n))'
-            r'[ ]{0,3}>[ ]?(.*)')
+        md.parser.blockprocessors.add('quote', BlockQuoteProcessor(md.parser), '<ulist')
 
     def extend_avatars(self, md: markdown.Markdown) -> None:
         # Note that !gravatar syntax should be deprecated long term.

zerver/tests/test_bugdown.py

@@ -996,6 +996,37 @@ class BugdownTest(ZulipTestCase):
                          'check this out</p>' % (hamlet.id, cordelia.id))
         self.assertEqual(msg.mentions_user_ids, set([hamlet.id, cordelia.id]))
 
+    def test_mention_in_quotes(self) -> None:
+        othello = self.example_user('othello')
+        hamlet = self.example_user('hamlet')
+        cordelia = self.example_user('cordelia')
+        msg = Message(sender=othello, sending_client=get_client("test"))
+
+        content = "> @**King Hamlet** and @**Othello, the Moor of Venice**\n\n @**King Hamlet** and @**Cordelia Lear**"
+        self.assertEqual(render_markdown(msg, content),
+                         '<blockquote>\n<p>'
+                         '<span class="user-mention silent" data-user-id="%s">@King Hamlet</span>'
+                         ' and '
+                         '<span class="user-mention silent" data-user-id="%s">@Othello, the Moor of Venice</span>'
+                         '</p>\n</blockquote>\n'
+                         '<p>'
+                         '<span class="user-mention" data-user-id="%s">@King Hamlet</span>'
+                         ' and '
+                         '<span class="user-mention" data-user-id="%s">@Cordelia Lear</span>'
+                         '</p>' % (hamlet.id, othello.id, hamlet.id, cordelia.id))
+        self.assertEqual(msg.mentions_user_ids, set([hamlet.id, cordelia.id]))
+
+        # Both fenced quote and > quote should be identical
+        expected = ('<blockquote>\n<p>'
+                    '<span class="user-mention silent" data-user-id="%s">@King Hamlet</span>'
+                    '</p>\n</blockquote>' % (hamlet.id))
+        content = "```quote\n@**King Hamlet**\n```"
+        self.assertEqual(render_markdown(msg, content), expected)
+        self.assertEqual(msg.mentions_user_ids, set())
+        content = "> @**King Hamlet**"
+        self.assertEqual(render_markdown(msg, content), expected)
+        self.assertEqual(msg.mentions_user_ids, set())
+
     def test_mention_duplicate_full_name(self) -> None:
         realm = get_realm('zulip')