markdown: Update characters allowed before @ and stream mentions.

Now the following characters are allowed before @-mentions and stream
references (starting with #) for proper rendering - {, [, /.

This commit makes the markdown rendering consistent with autocomplete
(anything that is autocompleted is also rendered properly).

zerver/lib/markdown/__init__.py

@@ -54,7 +54,12 @@ from zerver.lib.emoji import EMOTICON_RE, codepoint_to_name, name_to_codepoint,
 from zerver.lib.exceptions import MarkdownRenderingException
 from zerver.lib.markdown import fenced_code
 from zerver.lib.markdown.fenced_code import FENCE_RE
-from zerver.lib.mention import FullNameInfo, MentionBackend, MentionData
+from zerver.lib.mention import (
+    BEFORE_MENTION_ALLOWED_REGEX,
+    FullNameInfo,
+    MentionBackend,
+    MentionData,
+)
 from zerver.lib.outgoing_http import OutgoingSession
 from zerver.lib.subdomains import is_static_or_current_realm_url
 from zerver.lib.tex import render_tex
@@ -162,8 +167,8 @@ def verbose_compile(pattern: str) -> Pattern[str]:
     )
 
 
-STREAM_LINK_REGEX = r"""
-                     (?<![^\s'"\(,:<])            # Start after whitespace or specified chars
+STREAM_LINK_REGEX = rf"""
+                     {BEFORE_MENTION_ALLOWED_REGEX} # Start after whitespace or specified chars
                      \#\*\*                         # and after hash sign followed by double asterisks
                          (?P<stream_name>[^\*]+)    # stream name can contain anything
                      \*\*                           # ends by double asterisks
@@ -183,8 +188,8 @@ def get_compiled_stream_link_regex() -> Pattern[str]:
     )
 
 
-STREAM_TOPIC_LINK_REGEX = r"""
-                     (?<![^\s'"\(,:<])             # Start after whitespace or specified chars
+STREAM_TOPIC_LINK_REGEX = rf"""
+                     {BEFORE_MENTION_ALLOWED_REGEX}  # Start after whitespace or specified chars
                      \#\*\*                          # and after hash sign followed by double asterisks
                          (?P<stream_name>[^\*>]+)    # stream name can contain anything except >
                          >                           # > acts as separator

zerver/lib/mention.py

@@ -7,10 +7,16 @@ from django.db.models import Q
 
 from zerver.models import UserGroup, UserProfile, get_linkable_streams
 
+BEFORE_MENTION_ALLOWED_REGEX = r"(?<![^\s\'\"\(\{\[\/<])"
+
 # Match multi-word string between @** ** or match any one-word
 # sequences after @
-MENTIONS_RE = re.compile(r"(?<![^\s\'\"\(,:<])@(?P<silent>_?)(\*\*(?P<match>[^\*]+)\*\*)")
-USER_GROUP_MENTIONS_RE = re.compile(r"(?<![^\s\'\"\(,:<])@(?P<silent>_?)(\*(?P<match>[^\*]+)\*)")
+MENTIONS_RE = re.compile(
+    rf"{BEFORE_MENTION_ALLOWED_REGEX}@(?P<silent>_?)(\*\*(?P<match>[^\*]+)\*\*)"
+)
+USER_GROUP_MENTIONS_RE = re.compile(
+    rf"{BEFORE_MENTION_ALLOWED_REGEX}@(?P<silent>_?)(\*(?P<match>[^\*]+)\*)"
+)
 
 wildcards = ["all", "everyone", "stream"]
 

zerver/tests/test_markdown.py

@@ -1,6 +1,7 @@
 import copy
 import os
 import re
+from html import escape
 from textwrap import dedent
 from typing import Any, Dict, List, Optional, Set, Tuple, cast
 from unittest import mock
@@ -1946,6 +1947,38 @@ class MarkdownTest(ZulipTestCase):
         )
         self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})
 
+    def test_mention_with_valid_special_characters_before(self) -> None:
+        sender_user_profile = self.example_user("othello")
+        user_profile = self.example_user("hamlet")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+        user_id = user_profile.id
+
+        valid_characters_before_mention = ["(", "{", "[", "/", "<"]
+        for character in valid_characters_before_mention:
+            content = f"{character}@**King Hamlet**"
+            rendering_result = render_markdown(msg, content)
+            self.assertEqual(
+                rendering_result.rendered_content,
+                f'<p>{escape(character)}<span class="user-mention" '
+                f'data-user-id="{user_id}">'
+                "@King Hamlet</span></p>",
+            )
+        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})
+
+    def test_mention_with_invalid_special_characters_before(self) -> None:
+        sender_user_profile = self.example_user("othello")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+
+        invalid_characters_before_mention = [".", ",", ";", ":", "#"]
+        for character in invalid_characters_before_mention:
+            content = f"{character}@**King Hamlet**"
+            rendering_result = render_markdown(msg, content)
+            unicode_character = escape(character)
+            self.assertEqual(
+                rendering_result.rendered_content,
+                f"<p>{unicode_character}@<strong>King Hamlet</strong></p>",
+            )
+
     def test_mention_silent(self) -> None:
         sender_user_profile = self.example_user("othello")
         user_profile = self.example_user("hamlet")