import copy import os import re from textwrap import dedent from typing import Any, Dict, List, Optional, Set, Tuple, cast from unittest import mock import orjson from django.conf import settings from django.test import override_settings from markdown import Markdown from zerver.lib.actions import ( change_user_is_active, do_add_alert_words, do_create_realm, do_remove_realm_emoji, do_set_realm_property, do_set_user_display_setting, ) from zerver.lib.alert_words import get_alert_word_automaton from zerver.lib.camo import get_camo_url from zerver.lib.create_user import create_user from zerver.lib.emoji import get_emoji_url from zerver.lib.exceptions import JsonableError, MarkdownRenderingException from zerver.lib.markdown import ( MarkdownListPreprocessor, MessageRenderingResult, clear_state_for_testing, content_has_emoji_syntax, fetch_tweet_data, get_tweet_id, image_preview_enabled, markdown_convert, maybe_update_markdown_engines, possible_linked_stream_names, topic_links, url_embed_preview_enabled, url_to_a, ) from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor from zerver.lib.mdiff import diff_strings from zerver.lib.mention import ( MentionData, get_possible_mentions_info, possible_mentions, possible_user_group_mentions, ) from zerver.lib.message import render_markdown from zerver.lib.test_classes import ZulipTestCase from zerver.lib.tex import render_tex from zerver.lib.user_groups import create_user_group from zerver.models import ( Message, RealmEmoji, RealmFilter, Stream, UserGroup, UserMessage, UserProfile, flush_linkifiers, flush_per_request_caches, get_client, get_realm, get_stream, linkifiers_for_realm, realm_in_local_linkifiers_cache, ) class SimulatedFencedBlockPreprocessor(FencedBlockPreprocessor): # Simulate code formatting. def format_code(self, lang: str, code: str) -> str: return lang + ":" + code def placeholder(self, s: str) -> str: return "**" + s.strip("\n") + "**" class FencedBlockPreprocessorTest(ZulipTestCase): def test_simple_quoting(self) -> None: processor = FencedBlockPreprocessor(Markdown()) markdown_input = [ "~~~ quote", "hi", "bye", "", "", ] expected = [ "", "> hi", "> bye", "> ", "> ", "", "", ] lines = processor.run(markdown_input) self.assertEqual(lines, expected) def test_serial_quoting(self) -> None: processor = FencedBlockPreprocessor(Markdown()) markdown_input = [ "~~~ quote", "hi", "~~~", "", "~~~ quote", "bye", "", "", ] expected = [ "", "> hi", "", "", "", "> bye", "> ", "> ", "", "", ] lines = processor.run(markdown_input) self.assertEqual(lines, expected) def test_serial_code(self) -> None: processor = SimulatedFencedBlockPreprocessor(Markdown()) markdown_input = [ "``` .py", "hello()", "```", "", "```vb.net", "goodbye()", "```", "", "```c#", "weirdchar()", "```", "", "```", "no-highlight()", "```", "", ] expected = [ "", "**py:hello()**", "", "", "", "**vb.net:goodbye()**", "", "", "", "**c#:weirdchar()**", "", "", "", "**:no-highlight()**", "", "", ] lines = processor.run(markdown_input) self.assertEqual(lines, expected) def test_nested_code(self) -> None: processor = SimulatedFencedBlockPreprocessor(Markdown()) markdown_input = [ "~~~ quote", "hi", "``` .py", "hello()", "```", "", "", ] expected = [ "", "> hi", "> ", "> **py:hello()**", "> ", "> ", "> ", "", "", ] lines = processor.run(markdown_input) self.assertEqual(lines, expected) def markdown_convert_wrapper(content: str) -> str: return markdown_convert( content=content, message_realm=get_realm("zulip"), ).rendered_content class MarkdownMiscTest(ZulipTestCase): def test_diffs_work_as_expected(self) -> None: str1 = "

The quick brown fox jumps over the lazy dog. Animal stories are fun, yeah

" str2 = "

The fast fox jumps over the lazy dogs and cats. Animal stories are fun

" expected_diff = "\u001b[34m-\u001b[0m

The \u001b[33mquick brown\u001b[0m fox jumps over the lazy dog. Animal stories are fun\u001b[31m, yeah\u001b[0m

\n\u001b[34m+\u001b[0m

The \u001b[33mfast\u001b[0m fox jumps over the lazy dog\u001b[32ms and cats\u001b[0m. Animal stories are fun

\n" self.assertEqual(diff_strings(str1, str2), expected_diff) def test_get_possible_mentions_info(self) -> None: realm = get_realm("zulip") def make_user(email: str, full_name: str) -> UserProfile: return create_user( email=email, password="whatever", realm=realm, full_name=full_name, ) fred1 = make_user("fred1@example.com", "Fred Flintstone") change_user_is_active(fred1, False) fred2 = make_user("fred2@example.com", "Fred Flintstone") fred3 = make_user("fred3@example.com", "Fred Flintstone") change_user_is_active(fred3, False) fred4 = make_user("fred4@example.com", "Fred Flintstone") lst = get_possible_mentions_info( realm.id, {"Fred Flintstone", "Cordelia, LEAR's daughter", "Not A User"} ) set_of_names = set(map(lambda x: x["full_name"].lower(), lst)) self.assertEqual(set_of_names, {"fred flintstone", "cordelia, lear's daughter"}) by_id = {row["id"]: row for row in lst} self.assertEqual( by_id.get(fred2.id), dict( email=fred2.email, full_name="Fred Flintstone", id=fred2.id, ), ) self.assertEqual( by_id.get(fred4.id), dict( email=fred4.email, full_name="Fred Flintstone", id=fred4.id, ), ) def test_mention_data(self) -> None: realm = get_realm("zulip") hamlet = self.example_user("hamlet") cordelia = self.example_user("cordelia") content = "@**King Hamlet** @**Cordelia, lear's daughter**" mention_data = MentionData(realm.id, content) self.assertEqual(mention_data.get_user_ids(), {hamlet.id, cordelia.id}) self.assertEqual( mention_data.get_user_by_id(hamlet.id), dict( email=hamlet.email, full_name=hamlet.full_name, id=hamlet.id, ), ) user = mention_data.get_user_by_name("king hamLET") assert user is not None self.assertEqual(user["email"], hamlet.email) self.assertFalse(mention_data.message_has_wildcards()) content = "@**King Hamlet** @**Cordelia, lear's daughter** @**all**" mention_data = MentionData(realm.id, content) self.assertTrue(mention_data.message_has_wildcards()) def test_invalid_katex_path(self) -> None: with self.settings(DEPLOY_ROOT="/nonexistent"): with self.assertLogs(level="ERROR") as m: render_tex("random text") self.assertEqual(m.output, ["ERROR:root:Cannot find KaTeX for latex rendering!"]) class MarkdownListPreprocessorTest(ZulipTestCase): # We test that the preprocessor inserts blank lines at correct places. # We use <> to indicate that we need to insert a blank line here. def split_message(self, msg: str) -> Tuple[List[str], List[str]]: original = msg.replace("<>", "").split("\n") expected = re.split(r"\n|<>", msg) return original, expected def test_basic_list(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message("List without a gap\n<>* One\n* Two") self.assertEqual(preprocessor.run(original), expected) def test_list_after_quotes(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message( "```quote\nSomething\n```\n\nList without a gap\n<>* One\n* Two" ) self.assertEqual(preprocessor.run(original), expected) def test_list_in_code(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message("```\nList without a gap\n* One\n* Two\n```") self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_different_fences(self) -> None: preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one * two ~~~ This is a nested code fence, do not make changes here: * one * two ````quote Quote in code fence. Should not convert: * one * two ```` ~~~ Back in the quote. We should convert:<> * one * two ``` Outside. Should convert:<> * one * two """ original, expected = self.split_message(msg) self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_same_fence(self) -> None: preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one * two ```python This is a nested code fence, do not make changes here: * one * two ```quote Quote in code fence. Should not convert: * one * two ``` ``` Back in the quote. We should convert:<> * one * two ``` Outside. Should convert:<> * one * two """ original, expected = self.split_message(msg) self.assertEqual(preprocessor.run(original), expected) class MarkdownTest(ZulipTestCase): def setUp(self) -> None: super().setUp() clear_state_for_testing() def assertEqual(self, first: Any, second: Any, msg: str = "") -> None: if isinstance(first, str) and isinstance(second, str): if first != second: raise AssertionError( "Actual and expected outputs do not match; showing diff.\n" + diff_strings(first, second) + msg ) else: super().assertEqual(first, second) def load_markdown_tests(self) -> Tuple[Dict[str, Any], List[List[str]]]: test_fixtures = {} with open( os.path.join(os.path.dirname(__file__), "fixtures/markdown_test_cases.json"), "rb" ) as f: data = orjson.loads(f.read()) for test in data["regular_tests"]: test_fixtures[test["name"]] = test return test_fixtures, data["linkify_tests"] def test_markdown_no_ignores(self) -> None: # We do not want any ignored tests to be committed and merged. format_tests, linkify_tests = self.load_markdown_tests() for name, test in format_tests.items(): message = f'Test "{name}" shouldn\'t be ignored.' is_ignored = test.get("ignore", False) self.assertFalse(is_ignored, message) def test_markdown_fixtures(self) -> None: format_tests, linkify_tests = self.load_markdown_tests() valid_keys = { "name", "input", "expected_output", "backend_only_rendering", "marked_expected_output", "text_content", "translate_emoticons", "ignore", } for name, test in format_tests.items(): with self.subTest(markdown_test_case=name): # Check that there aren't any unexpected keys as those are often typos self.assert_length(set(test.keys()) - valid_keys, 0) # Ignore tests if specified if test.get("ignore", False): continue # nocoverage if test.get("translate_emoticons", False): # Create a userprofile and send message with it. user_profile = self.example_user("othello") do_set_user_display_setting(user_profile, "translate_emoticons", True) msg = Message(sender=user_profile, sending_client=get_client("test")) rendering_result = render_markdown(msg, test["input"]) converted = rendering_result.rendered_content else: converted = markdown_convert_wrapper(test["input"]) self.assertEqual(converted, test["expected_output"]) def replaced(payload: str, url: str, phrase: str = "") -> str: if url[:4] == "http": href = url elif "@" in url: href = "mailto:" + url else: href = "http://" + url return payload % (f'{url}',) with mock.patch( "zerver.lib.url_preview.preview.link_embed_data_from_cache", return_value=None ): for inline_url, reference, url in linkify_tests: try: match = replaced(reference, url, phrase=inline_url) except TypeError: match = reference converted = markdown_convert_wrapper(inline_url) self.assertEqual(match, converted) def test_inline_file(self) -> None: msg = "Check out this file file:///Volumes/myserver/Users/Shared/pi.py" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

', ) clear_state_for_testing() with self.settings(ENABLE_FILE_LINKS=False): realm = do_create_realm(string_id="file_links_test", name="file_links_test") maybe_update_markdown_engines(realm.id, False) self.assertEqual( markdown_convert(msg, message_realm=realm).rendered_content, "

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

", ) def test_inline_bitcoin(self) -> None: msg = "To bitcoin:1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa or not to bitcoin" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

To bitcoin:1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa or not to bitcoin

', ) def test_inline_youtube(self) -> None: msg = "Check out the debate: http://www.youtube.com/watch?v=hx1mjT73xYE" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

Check out the debate: http://www.youtube.com/watch?v=hx1mjT73xYE

\n
""", ) msg = "http://www.youtube.com/watch?v=hx1mjT73xYE" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

http://www.youtube.com/watch?v=hx1mjT73xYE

\n
""", ) msg = "https://youtu.be/hx1mjT73xYE" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

https://youtu.be/hx1mjT73xYE

\n
""", ) msg = "https://www.youtube.com/playlist?list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo" not_converted = markdown_convert_wrapper(msg) self.assertEqual( not_converted, '

https://www.youtube.com/playlist?list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo

', ) msg = ( "https://www.youtube.com/playlist?v=O5nskjZ_GoI&list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

https://www.youtube.com/playlist?v=O5nskjZ_GoI&list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo

\n
""", ) msg = "http://www.youtube.com/watch_videos?video_ids=nOJgD4fcZhI,i96UO8-GFvw" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

http://www.youtube.com/watch_videos?video_ids=nOJgD4fcZhI,i96UO8-GFvw

\n
""", ) @override_settings(INLINE_URL_EMBED_PREVIEW=False) def test_inline_vimeo(self) -> None: msg = "Check out the debate: https://vimeo.com/246979354" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

Check out the debate: https://vimeo.com/246979354

', ) msg = "https://vimeo.com/246979354" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

https://vimeo.com/246979354

', ) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_thumbnail_url(self) -> None: realm = get_realm("zephyr") msg = "[foobar](/user_uploads/{realm_id}/50/w2G6ok9kr8AMCQCTNAUOFMln/IMG_0677.JPG)" msg = msg.format(realm_id=realm.id) thumbnail_img = '<' thumbnail_img = thumbnail_img.format(realm_id=realm.id) converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) msg = "https://www.google.com/images/srpr/logo4w.png" thumbnail_img = '' converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) msg = "www.google.com/images/srpr/logo4w.png" thumbnail_img = '' converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) msg = "https://www.google.com/images/srpr/logo4w.png" thumbnail_img = f"""
""" with self.settings(THUMBNAIL_IMAGES=False): converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) # Any URL which is not an external link and doesn't start with # /user_uploads/ is not thumbnailed msg = "[foobar](/static/images/cute/turtle.png)" thumbnail_img = '
' converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) msg = "[foobar](/user_avatars/{realm_id}/emoji/images/50.png)" msg = msg.format(realm_id=realm.id) thumbnail_img = '
' thumbnail_img = thumbnail_img.format(realm_id=realm.id) converted = markdown_convert_wrapper(msg) self.assertIn(thumbnail_img, converted) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_preview(self) -> None: with_preview = '
' without_preview = '

http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg

' content = "http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg" sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, with_preview) realm = msg.get_realm() setattr(realm, "inline_image_preview", False) realm.save() sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, without_preview) @override_settings(THUMBNAIL_IMAGES=False, EXTERNAL_URI_SCHEME="https://") def test_external_image_preview_use_camo(self) -> None: content = "https://example.com/thing.jpeg" thumbnail_img = f"""
""" converted = markdown_convert_wrapper(content) self.assertIn(converted, thumbnail_img) @override_settings(THUMBNAIL_IMAGES=False, EXTERNAL_URI_SCHEME="https://") def test_static_image_preview_skip_camo(self) -> None: content = f"{ settings.STATIC_URL }/thing.jpeg" thumbnail_img = f"""
""" converted = markdown_convert_wrapper(content) self.assertIn(converted, thumbnail_img) @override_settings(THUMBNAIL_IMAGES=False, EXTERNAL_URI_SCHEME="https://") def test_realm_image_preview_skip_camo(self) -> None: content = f"https://zulip.{ settings.EXTERNAL_HOST }/thing.jpeg" converted = markdown_convert_wrapper(content) self.assertNotIn(converted, get_camo_url(content)) @override_settings(THUMBNAIL_IMAGES=False, EXTERNAL_URI_SCHEME="https://") def test_cross_realm_image_preview_use_camo(self) -> None: content = f"https://otherrealm.{ settings.EXTERNAL_HOST }/thing.jpeg" thumbnail_img = f"""
""" converted = markdown_convert_wrapper(content) self.assertIn(converted, thumbnail_img) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_quoted_blocks(self) -> None: content = "http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg" expected = '
' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) content = ">http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg\n\nAwesome!" expected = '
\n

http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg

\n
\n

Awesome!

' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) content = ">* http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg\n\nAwesome!" expected = '
\n\n
\n

Awesome!

' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_preview_order(self) -> None: realm = get_realm("zulip") content = "http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg" expected = '

http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg
\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg
\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg

\n
' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) content = "http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg\n\n>http://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg\n\n* http://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg\n* https://www.google.com/images/srpr/logo4w.png" expected = '
\n

http://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg

\n
\n' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) content = "Test 1\n[21136101110_1dde1c1a7e_o.jpg](/user_uploads/{realm_id}/6d/F1PX6u16JA2P-nK45PyxHIYZ/21136101110_1dde1c1a7e_o.jpg) \n\nNext image\n[IMG_20161116_023910.jpg](/user_uploads/{realm_id}/69/sh7L06e7uH7NaX6d5WFfVYQp/IMG_20161116_023910.jpg) \n\nAnother screenshot\n[Screenshot-from-2016-06-01-16-22-42.png](/user_uploads/{realm_id}/70/_aZmIEWaN1iUaxwkDjkO7bpj/Screenshot-from-2016-06-01-16-22-42.png)" content = content.format(realm_id=realm.id) expected = '

Test 1
\n21136101110_1dde1c1a7e_o.jpg

\n

Next image
\nIMG_20161116_023910.jpg

\n

Another screenshot
\nScreenshot-from-2016-06-01-16-22-42.png

\n
' expected = expected.format(realm_id=realm.id) msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_corrected_image_source(self) -> None: # testing only Wikipedia because linx.li URLs can be expected to expire content = "https://en.wikipedia.org/wiki/File:Wright_of_Derby,_The_Orrery.jpg" expected = '
' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=False) def test_image_preview_enabled(self) -> None: ret = image_preview_enabled() self.assertFalse(ret) settings.INLINE_IMAGE_PREVIEW = True sender_user_profile = self.example_user("othello") message = Message(sender=sender_user_profile, sending_client=get_client("test")) realm = message.get_realm() ret = image_preview_enabled() self.assertTrue(ret) ret = image_preview_enabled(no_previews=True) self.assertFalse(ret) ret = image_preview_enabled(message, realm) self.assertTrue(ret) ret = image_preview_enabled(message) self.assertTrue(ret) ret = image_preview_enabled(message, realm, no_previews=True) self.assertFalse(ret) ret = image_preview_enabled(message, no_previews=True) self.assertFalse(ret) @override_settings(INLINE_URL_EMBED_PREVIEW=False) def test_url_embed_preview_enabled(self) -> None: sender_user_profile = self.example_user("othello") message = copy.deepcopy( Message(sender=sender_user_profile, sending_client=get_client("test")) ) realm = message.get_realm() realm.inline_url_embed_preview = True # off by default realm.save(update_fields=["inline_url_embed_preview"]) ret = url_embed_preview_enabled() self.assertFalse(ret) settings.INLINE_URL_EMBED_PREVIEW = True ret = url_embed_preview_enabled() self.assertTrue(ret) ret = image_preview_enabled(no_previews=True) self.assertFalse(ret) ret = url_embed_preview_enabled(message, realm) self.assertTrue(ret) ret = url_embed_preview_enabled(message) self.assertTrue(ret) ret = url_embed_preview_enabled(message, no_previews=True) self.assertFalse(ret) def test_inline_dropbox(self) -> None: msg = "Look at how hilarious our old office was: https://www.dropbox.com/s/ymdijjcg67hv2ta/IMG_0923.JPG" image_info = { "image": "https://photos-4.dropbox.com/t/2/AABIre1oReJgPYuc_53iv0IHq1vUzRaDg2rrCfTpiWMccQ/12/129/jpeg/1024x1024/2/_/0/4/IMG_0923.JPG/CIEBIAEgAiAHKAIoBw/ymdijjcg67hv2ta/AABz2uuED1ox3vpWWvMpBxu6a/IMG_0923.JPG", "desc": "Shared with Dropbox", "title": "IMG_0923.JPG", } with mock.patch("zerver.lib.markdown.fetch_open_graph_image", return_value=image_info): converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

Look at how hilarious our old office was: https://www.dropbox.com/s/ymdijjcg67hv2ta/IMG_0923.JPG

\n
""", ) msg = "Look at my hilarious drawing folder: https://www.dropbox.com/sh/cm39k9e04z7fhim/AAAII5NK-9daee3FcF41anEua?dl=" image_info = { "image": "https://cf.dropboxstatic.com/static/images/icons128/folder_dropbox.png", "desc": "Shared with Dropbox", "title": "Saves", } with mock.patch("zerver.lib.markdown.fetch_open_graph_image", return_value=image_info): converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

Look at my hilarious drawing folder: https://www.dropbox.com/sh/cm39k9e04z7fhim/AAAII5NK-9daee3FcF41anEua?dl=

\n
Saves
""", ) def test_inline_dropbox_preview(self) -> None: # Test photo album previews msg = "https://www.dropbox.com/sc/tditp9nitko60n5/03rEiZldy5" image_info = { "image": "https://photos-6.dropbox.com/t/2/AAAlawaeD61TyNewO5vVi-DGf2ZeuayfyHFdNTNzpGq-QA/12/271544745/jpeg/1024x1024/2/_/0/5/baby-piglet.jpg/CKnjvYEBIAIgBygCKAc/tditp9nitko60n5/AADX03VAIrQlTl28CtujDcMla/0", "desc": "Shared with Dropbox", "title": "1 photo", } with mock.patch("zerver.lib.markdown.fetch_open_graph_image", return_value=image_info): converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

https://www.dropbox.com/sc/tditp9nitko60n5/03rEiZldy5

\n
""", ) def test_inline_dropbox_negative(self) -> None: # Make sure we're not overzealous in our conversion: msg = "Look at the new dropbox logo: https://www.dropbox.com/static/images/home_logo.png" with mock.patch("zerver.lib.markdown.fetch_open_graph_image", return_value=None): converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

Look at the new dropbox logo: https://www.dropbox.com/static/images/home_logo.png

\n
', ) def test_inline_dropbox_bad(self) -> None: # Don't fail on bad dropbox links msg = "https://zulip-test.dropbox.com/photos/cl/ROmr9K1XYtmpneM" with mock.patch("zerver.lib.markdown.fetch_open_graph_image", return_value=None): converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

https://zulip-test.dropbox.com/photos/cl/ROmr9K1XYtmpneM

', ) def test_inline_github_preview(self) -> None: # Test photo album previews msg = "Test: https://github.com/zulip/zulip/blob/master/static/images/logo/zulip-icon-128x128.png" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

Test: https://github.com/zulip/zulip/blob/master/static/images/logo/zulip-icon-128x128.png

\n
', ) msg = "Test: https://developer.github.com/assets/images/hero-circuit-bg.png" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

Test: https://developer.github.com/assets/images/hero-circuit-bg.png

\n
', ) def test_inline_youtube_preview(self) -> None: # Test YouTube URLs in spoilers msg = """\n```spoiler Check out this PyCon video\nhttps://www.youtube.com/watch?v=0c46YHS3RY8\n```""" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""
\n

Check out this PyCon video

\n
""", ) # Test YouTube URLs in normal messages. msg = "[YouTube link](https://www.youtube.com/watch?v=0c46YHS3RY8)" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

YouTube link

\n
""", ) msg = "https://www.youtube.com/watch?v=0c46YHS3RY8\n\nSample text\n\nhttps://www.youtube.com/watch?v=lXFO2ULktEI" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, f"""

https://www.youtube.com/watch?v=0c46YHS3RY8

\n

Sample text

\n

https://www.youtube.com/watch?v=lXFO2ULktEI

\n
""", ) def test_twitter_id_extraction(self) -> None: self.assertEqual( get_tweet_id("http://twitter.com/#!/VizzQuotes/status/409030735191097344"), "409030735191097344", ) self.assertEqual( get_tweet_id("http://twitter.com/VizzQuotes/status/409030735191097344"), "409030735191097344", ) self.assertEqual( get_tweet_id("http://twitter.com/VizzQuotes/statuses/409030735191097344"), "409030735191097344", ) self.assertEqual(get_tweet_id("https://twitter.com/wdaher/status/1017581858"), "1017581858") self.assertEqual( get_tweet_id("https://twitter.com/wdaher/status/1017581858/"), "1017581858" ) self.assertEqual( get_tweet_id("https://twitter.com/windyoona/status/410766290349879296/photo/1"), "410766290349879296", ) self.assertEqual( get_tweet_id("https://twitter.com/windyoona/status/410766290349879296/"), "410766290349879296", ) def test_inline_interesting_links(self) -> None: def make_link(url: str) -> str: return f'{url}' normal_tweet_html = ( '@Twitter " "meets @seepicturely at #tcdisrupt cc." '@boscomonkey " '@episod " 'http://instagr.am/p/MuW67/" ) mention_in_link_tweet_html = """http://foo.com""" media_tweet_html = ( '' "http://twitter.com/NEVNBoston/status/421654515616849920/photo/1" ) emoji_in_tweet_html = """Zulip is :100:% open-source!""" def make_inline_twitter_preview(url: str, tweet_html: str, image_html: str = "") -> str: ## As of right now, all previews are mocked to be the exact same tweet return ( '
' '
' f'' '' "" f"

{tweet_html}

" "- Eoin McMillan (@imeoin)" f"{image_html}" "
" "
" ) msg = "http://www.twitter.com" converted = markdown_convert_wrapper(msg) self.assertEqual(converted, "

{}

".format(make_link("http://www.twitter.com"))) msg = "http://www.twitter.com/wdaher/" converted = markdown_convert_wrapper(msg) self.assertEqual(converted, "

{}

".format(make_link("http://www.twitter.com/wdaher/"))) msg = "http://www.twitter.com/wdaher/status/3" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

".format(make_link("http://www.twitter.com/wdaher/status/3")) ) # id too long msg = "http://www.twitter.com/wdaher/status/2879779692873154569" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

".format( make_link("http://www.twitter.com/wdaher/status/2879779692873154569") ), ) # id too large (i.e. tweet doesn't exist) msg = "http://www.twitter.com/wdaher/status/999999999999999999" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

".format( make_link("http://www.twitter.com/wdaher/status/999999999999999999") ), ) msg = "http://www.twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("http://www.twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://www.twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) msg = "https://www.twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("https://www.twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "https://www.twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) msg = "http://twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) # Repeated links will only be converted once msg = ( "http://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315457 " "http://twitter.com/wdaher/status/287977969287315457 " "http://twitter.com/wdaher/status/287977969287315457" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{} {} {} {}

\n{}{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), ), ) # A max of 3 will be converted msg = ( "http://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315457 " "https://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315460" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{} {} {} {}

\n{}{}{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("https://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315460"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), make_inline_twitter_preview( "https://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) # Test smart in-place inlining behavior: msg = ( "Paragraph 1: http://twitter.com/wdaher/status/287977969287315456\n\n" "Paragraph 2\n\n" "Paragraph 3: http://twitter.com/wdaher/status/287977969287315457" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

Paragraph 1: {}

\n{}

Paragraph 2

\n

Paragraph 3: {}

\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), ), ) # Tweet has a mention in a URL, only the URL is linked msg = "http://twitter.com/wdaher/status/287977969287315458" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315458"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315458", mention_in_link_tweet_html, ), ), ) # Tweet with an image msg = "http://twitter.com/wdaher/status/287977969287315459" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315459"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315459", media_tweet_html, ( '
' '' f"""""" "" "
" ), ), ), ) msg = "http://twitter.com/wdaher/status/287977969287315460" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "

{}

\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315460"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315460", emoji_in_tweet_html ), ), ) # Test Twitter previews in spoiler tags. msg = "```spoiler secret tweet\nTweet: http://twitter.com/wdaher/status/287977969287315456\n```" converted = markdown_convert_wrapper(msg) rendered_spoiler = '
\n

secret tweet

\n
' self.assertEqual( converted, rendered_spoiler.format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) def test_fetch_tweet_data_settings_validation(self) -> None: with self.settings(TEST_SUITE=False, TWITTER_CONSUMER_KEY=None): self.assertIs(None, fetch_tweet_data("287977969287315459")) def test_content_has_emoji(self) -> None: self.assertFalse(content_has_emoji_syntax("boring")) self.assertFalse(content_has_emoji_syntax("hello: world")) self.assertFalse(content_has_emoji_syntax(":foobar")) self.assertFalse(content_has_emoji_syntax("::: hello :::")) self.assertTrue(content_has_emoji_syntax("foo :whatever:")) self.assertTrue(content_has_emoji_syntax("\n:whatever:")) self.assertTrue(content_has_emoji_syntax(":smile: ::::::")) def test_realm_emoji(self) -> None: def emoji_img(name: str, file_name: str, realm_id: int) -> str: return '{}'.format( name, get_emoji_url(file_name, realm_id), name[1:-1].replace("_", " ") ) realm = get_realm("zulip") # Needs to mock an actual message because that's how Markdown obtains the realm msg = Message(sender=self.example_user("hamlet")) converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) realm_emoji = RealmEmoji.objects.filter( realm=realm, name="green_tick", deactivated=False ).get() self.assertEqual( converted.rendered_content, "

{}

".format(emoji_img(":green_tick:", realm_emoji.file_name, realm.id)), ) # Deactivate realm emoji. do_remove_realm_emoji(realm, "green_tick") converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted.rendered_content, "

:green_tick:

") def test_deactivated_realm_emoji(self) -> None: # Deactivate realm emoji. realm = get_realm("zulip") do_remove_realm_emoji(realm, "green_tick") msg = Message(sender=self.example_user("hamlet")) converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted.rendered_content, "

:green_tick:

") def test_unicode_emoji(self) -> None: msg = "\u2615" # ☕ converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

:coffee:

', ) msg = "\u2615\u2615" # ☕☕ converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '

:coffee::coffee:

', ) def test_no_translate_emoticons_if_off(self) -> None: user_profile = self.example_user("othello") do_set_user_display_setting(user_profile, "translate_emoticons", False) msg = Message(sender=user_profile, sending_client=get_client("test")) content = ":)" expected = "

:)

" converted = render_markdown(msg, content) self.assertEqual(converted.rendered_content, expected) def test_same_markup(self) -> None: msg = "\u2615" # ☕ unicode_converted = markdown_convert_wrapper(msg) msg = ":coffee:" # ☕☕ converted = markdown_convert_wrapper(msg) self.assertEqual(converted, unicode_converted) def test_links_in_topic_name(self) -> None: realm = get_realm("zulip") msg = Message(sender=self.example_user("othello")) msg.set_topic_name("https://google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [{"url": "https://google.com/hello-world", "text": "https://google.com/hello-world"}], ) msg.set_topic_name("http://google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [{"url": "http://google.com/hello-world", "text": "http://google.com/hello-world"}], ) msg.set_topic_name("Without scheme google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [{"url": "https://google.com/hello-world", "text": "google.com/hello-world"}], ) msg.set_topic_name("Without scheme random.words/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, []) msg.set_topic_name( "Try out http://ftp.debian.org, https://google.com/ and https://google.in/." ) converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [ {"url": "http://ftp.debian.org", "text": "http://ftp.debian.org"}, {"url": "https://google.com/", "text": "https://google.com/"}, {"url": "https://google.in/", "text": "https://google.in/"}, ], ) # test order for links without scheme msg.set_topic_name("google.in google.com") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [ {"url": "https://google.in", "text": "google.in"}, {"url": "https://google.com", "text": "google.com"}, ], ) def test_realm_patterns(self) -> None: realm = get_realm("zulip") url_format_string = r"https://trac.example.com/ticket/%(id)s" linkifier = RealmFilter( realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string ) linkifier.save() self.assertEqual( linkifier.__str__(), "[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>", ) msg = Message(sender=self.example_user("othello")) msg.set_topic_name("#444") flush_per_request_caches() content = "We should fix #224 and #115, but not issue#124 or #1124z or [trac #15](https://trac.example.com/ticket/16) today." converted = markdown_convert(content, message_realm=realm, message=msg) converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted.rendered_content, '

We should fix #224 and #115, but not issue#124 or #1124z or trac #15 today.

', ) self.assertEqual( converted_topic, [{"url": "https://trac.example.com/ticket/444", "text": "#444"}] ) msg.set_topic_name("#444 https://google.com") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, [ {"url": "https://trac.example.com/ticket/444", "text": "#444"}, {"url": "https://google.com", "text": "https://google.com"}, ], ) RealmFilter( realm=realm, pattern=r"#(?P[a-zA-Z]+-[0-9]+)", url_format_string=r"https://trac.example.com/ticket/%(id)s", ).save() msg = Message(sender=self.example_user("hamlet")) content = "#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging" converted = markdown_convert(content, message_realm=realm, message=msg) self.assertEqual( converted.rendered_content, '

#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging

', ) def assert_conversion(content: str, should_have_converted: bool = True) -> None: converted = markdown_convert(content, message_realm=realm, message=msg).rendered_content converted_topic = topic_links(realm.id, content) if should_have_converted: self.assertTrue("https://trac.example.com" in converted) self.assert_length(converted_topic, 1) self.assertEqual( converted_topic[0], {"url": "https://trac.example.com/ticket/123", "text": "#123"}, ) else: self.assertTrue("https://trac.example.com" not in converted) self.assert_length(converted_topic, 0) assert_conversion("Hello #123 World") assert_conversion("Hello #123World", False) assert_conversion("Hello#123 World", False) assert_conversion("Hello#123World", False) # Ideally, these should be converted, but Markdown doesn't # handle word boundary detection in languages that don't use # whitespace for that correctly yet. assert_conversion("チケットは#123です", False) assert_conversion("チケットは #123です", False) assert_conversion("チケットは#123 です", False) assert_conversion("チケットは #123 です") assert_conversion("(#123)") assert_conversion("#123>") assert_conversion('"#123"') assert_conversion("#123@") assert_conversion(")#123(", False) assert_conversion("##123", False) # test nested realm patterns should avoid double matching RealmFilter( realm=realm, pattern=r"hello#(?P[0-9]+)", url_format_string=r"https://trac.example.com/hello/%(id)s", ).save() converted_topic = topic_links(realm.id, "hello#123 #234") self.assertEqual( converted_topic, [ {"url": "https://trac.example.com/hello/123", "text": "hello#123"}, {"url": "https://trac.example.com/ticket/234", "text": "#234"}, ], ) # test correct order when realm pattern and normal links are both present. converted_topic = topic_links(realm.id, "#234 https://google.com") self.assertEqual( converted_topic, [ {"url": "https://trac.example.com/ticket/234", "text": "#234"}, {"url": "https://google.com", "text": "https://google.com"}, ], ) def test_multiple_matching_realm_patterns(self) -> None: realm = get_realm("zulip") url_format_string = r"https://trac.example.com/ticket/%(id)s" linkifier_1 = RealmFilter( realm=realm, pattern=r"(?PABC\-[0-9]+)(?![A-Z0-9-])", url_format_string=url_format_string, ) linkifier_1.save() self.assertEqual( linkifier_1.__str__(), r"ABC\-[0-9]+)(?![A-Z0-9-])" " https://trac.example.com/ticket/%(id)s>", ) url_format_string = r"https://other-trac.example.com/ticket/%(id)s" linkifier_2 = RealmFilter( realm=realm, pattern=r"(?P[A-Z][A-Z0-9]*\-[0-9]+)(?![A-Z0-9-])", url_format_string=url_format_string, ) linkifier_2.save() self.assertEqual( linkifier_2.__str__(), r"[A-Z][A-Z0-9]*\-[0-9]+)(?![A-Z0-9-])" " https://other-trac.example.com/ticket/%(id)s>", ) msg = Message(sender=self.example_user("othello")) msg.set_topic_name("ABC-123") flush_per_request_caches() content = ( "We should fix ABC-123 or [trac ABC-123](https://trac.example.com/ticket/16) today." ) converted = markdown_convert(content, message_realm=realm, message=msg) converted_topic = topic_links(realm.id, msg.topic_name()) # The second linkifier (which was saved later) was ignored as the content was marked AtomicString after first conversion. # There was no easy way to support parsing both linkifiers and not run into an infinite loop, hence the second linkifier is ignored. self.assertEqual( converted.rendered_content, '

We should fix ABC-123 or trac ABC-123 today.

', ) # Both the links should be generated in topics. self.assertEqual( converted_topic, [ {"url": "https://trac.example.com/ticket/ABC-123", "text": "ABC-123"}, {"url": "https://other-trac.example.com/ticket/ABC-123", "text": "ABC-123"}, ], ) def test_flush_linkifier(self) -> None: realm = get_realm("zulip") def flush() -> None: """ flush_linkifiers is a post-save hook, so calling it directly for testing is kind of awkward """ class Instance: realm_id: Optional[int] = None instance = Instance() instance.realm_id = realm.id flush_linkifiers(sender=RealmFilter, instance=cast(RealmFilter, instance)) def save_new_linkifier() -> None: linkifier = RealmFilter(realm=realm, pattern=r"whatever", url_format_string="whatever") linkifier.save() # start fresh for our realm flush() self.assertFalse(realm_in_local_linkifiers_cache(realm.id)) # call this just for side effects of populating the cache linkifiers_for_realm(realm.id) self.assertTrue(realm_in_local_linkifiers_cache(realm.id)) # Saving a new RealmFilter should have the side effect of # flushing the cache. save_new_linkifier() self.assertFalse(realm_in_local_linkifiers_cache(realm.id)) # and flush it one more time, to make sure we don't get a KeyError flush() self.assertFalse(realm_in_local_linkifiers_cache(realm.id)) def test_realm_patterns_negative(self) -> None: realm = get_realm("zulip") RealmFilter( realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=r"https://trac.example.com/ticket/%(id)s", ).save() boring_msg = Message(sender=self.example_user("othello")) boring_msg.set_topic_name("no match here") converted_boring_topic = topic_links(realm.id, boring_msg.topic_name()) self.assertEqual(converted_boring_topic, []) def test_is_status_message(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "/me makes a list\n* one\n* two" rendering_result = render_markdown(msg, content) self.assertEqual( rendering_result.rendered_content, "

/me makes a list

\n
    \n
  • one
    • \n
  • two
    • \n
", ) self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) content = "/me takes a walk" rendering_result = render_markdown(msg, content) self.assertEqual( rendering_result.rendered_content, "

/me takes a walk

", ) self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) content = "/me writes a second line\nline" rendering_result = render_markdown(msg, content) self.assertEqual( rendering_result.rendered_content, "

/me writes a second line
\nline

", ) self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) def test_alert_words(self) -> None: user_profile = self.example_user("othello") do_add_alert_words(user_profile, ["ALERTWORD", "scaryword"]) msg = Message(sender=user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = "We have an ALERTWORD day today!" rendering_result = render(msg, content) self.assertEqual( rendering_result.rendered_content, "

We have an ALERTWORD day today!

" ) self.assertEqual(rendering_result.user_ids_with_alert_words, {user_profile.id}) msg = Message(sender=user_profile, sending_client=get_client("test")) content = "We have a NOTHINGWORD day today!" rendering_result = render(msg, content) self.assertEqual( rendering_result.rendered_content, "

We have a NOTHINGWORD day today!

" ) self.assertEqual(rendering_result.user_ids_with_alert_words, set()) def test_alert_words_returns_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["how"], "cordelia": ["this possible"], "iago": ["hello"], "prospero": ["hello"], "othello": ["how are you"], "aaron": ["hey"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = "hello how is this possible how are you doing today" rendering_result = render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, user_profiles["iago"].id, user_profiles["prospero"].id, user_profiles["othello"].id, } # All users except aaron have their alert word appear in the message content self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_1(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["provisioning", "Prod deployment"], "cordelia": ["test", "Prod"], "iago": ["prod"], "prospero": ["deployment"], "othello": ["last"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """Hello, everyone. Prod deployment has been completed And this is a new line to test out how Markdown convert this into something line ending split array and this is a new line last""" rendering_result = render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, user_profiles["iago"].id, user_profiles["prospero"].id, user_profiles["othello"].id, } # All users have their alert word appear in the message content self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_in_french(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["réglementaire", "une politique", "une merveille"], "cordelia": ["énormément", "Prod"], "iago": ["prod"], "prospero": ["deployment"], "othello": ["last"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """This is to test out alert words work in languages with accented characters too bonjour est (énormément) ce a quoi ressemble le français et j'espère qu'il n'y n' réglementaire a pas de mots d'alerte dans ce texte français """ rendering_result = render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id, user_profiles["cordelia"].id} # Only hamlet and cordelia have their alert-words appear in the message content self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_empty_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": [], "cordelia": [], "iago": [], "prospero": [], "othello": [], "aaron": [], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """hello how is this possible how are you doing today This is to test that the no user_ids who have alrert wourldword is participating in sending of the message """ rendering_result = render(msg, content) expected_user_ids: Set[int] = set() # None of the users have their alert-words appear in the message content self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def get_mock_alert_words(self, num_words: int, word_length: int) -> List[str]: alert_words = ["x" * word_length] * num_words # type List[str] return alert_words def test_alert_words_with_empty_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": [], "cordelia": [], "iago": [], "othello": [], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """This is to test a empty alert words i.e. no user has any alert-words set""" rendering_result = render(msg, content) expected_user_ids: Set[int] = set() self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_retuns_user_ids_with_alert_words_with_huge_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["issue124"], "cordelia": self.get_mock_alert_words(500, 10), "iago": self.get_mock_alert_words(500, 10), "othello": self.get_mock_alert_words(500, 10), } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """The code above will print 10 random values of numbers between 1 and 100. The second line, for x in range(10), determines how many values will be printed (when you use range(x), the number that you use in place of x will be the amount of values that you'll have printed. if you want 20 values, use range(20). use range(5) if you only want 5 values returned, etc.). I was talking abou the issue124 on github. Then the third line: print random.randint(1,101) will automatically select a random integer between 1 and 100 for you. The process is fairly simple """ rendering_result = render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id} # Only hamlet has alert-word 'issue124' present in the message content self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_default_code_block_language(self) -> None: realm = get_realm("zulip") self.assertEqual(realm.default_code_block_language, None) text = "```{}\nconsole.log('Hello World');\n```\n" # Render without default language msg_with_js = markdown_convert_wrapper(text.format("js")) msg_with_python = markdown_convert_wrapper(text.format("python")) msg_without_language = markdown_convert_wrapper(text.format("")) msg_with_quote = markdown_convert_wrapper(text.format("quote")) msg_with_math = markdown_convert_wrapper(text.format("math")) msg_with_none = markdown_convert_wrapper(text.format("none")) # Render with default=javascript do_set_realm_property(realm, "default_code_block_language", "javascript", acting_user=None) msg_without_language_default_js = markdown_convert_wrapper(text.format("")) msg_with_python_default_js = markdown_convert_wrapper(text.format("python")) # Render with default=python do_set_realm_property(realm, "default_code_block_language", "python", acting_user=None) msg_without_language_default_py = markdown_convert_wrapper(text.format("")) msg_with_none_default_py = markdown_convert_wrapper(text.format("none")) # Render with default=quote do_set_realm_property(realm, "default_code_block_language", "quote", acting_user=None) msg_without_language_default_quote = markdown_convert_wrapper(text.format("")) # Render with default=math do_set_realm_property(realm, "default_code_block_language", "math", acting_user=None) msg_without_language_default_math = markdown_convert_wrapper(text.format("")) # Render without default language do_set_realm_property(realm, "default_code_block_language", None, acting_user=None) msg_without_language_final = markdown_convert_wrapper(text.format("")) self.assertTrue(msg_with_js == msg_without_language_default_js) self.assertTrue( msg_with_python == msg_with_python_default_js == msg_without_language_default_py ) self.assertTrue(msg_with_quote == msg_without_language_default_quote) self.assertTrue(msg_with_math == msg_without_language_default_math) self.assertTrue(msg_without_language == msg_without_language_final) self.assertTrue(msg_with_none == msg_with_none_default_py) # Test checking inside nested quotes nested_text = "````quote\n\n{}\n\n{}````".format(text.format("js"), text.format("")) do_set_realm_property(realm, "default_code_block_language", "javascript", acting_user=None) rendered = markdown_convert_wrapper(nested_text) with_language, without_language = re.findall(r"
(.*?)$", rendered, re.MULTILINE)
        self.assertTrue(with_language == without_language)

        do_set_realm_property(realm, "default_code_block_language", None, acting_user=None)
        rendered = markdown_convert_wrapper(nested_text)
        with_language, without_language = re.findall(r"
(.*?)$", rendered, re.MULTILINE)
        self.assertFalse(with_language == without_language)

    def test_mention_wildcard(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@**all** test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
' "@all" " test
",
        )
        self.assertTrue(rendering_result.mentions_wildcard)

    def test_mention_everyone(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@**everyone** test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
' "@everyone" " test
",
        )
        self.assertTrue(rendering_result.mentions_wildcard)

    def test_mention_stream(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@**stream** test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
' "@stream" " test
",
        )
        self.assertTrue(rendering_result.mentions_wildcard)

    def test_mention_at_wildcard(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@all test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, "
@all test
")
        self.assertFalse(rendering_result.mentions_wildcard)
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_mention_at_everyone(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@everyone test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, "
@everyone test
")
        self.assertFalse(rendering_result.mentions_wildcard)
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_mention_word_starting_with_at_wildcard(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "test @alleycat.com test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, "
test @alleycat.com test
")
        self.assertFalse(rendering_result.mentions_wildcard)
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_mention_at_normal_user(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@aaron test"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, "
@aaron test
")
        self.assertFalse(rendering_result.mentions_wildcard)
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_mention_single(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id

        content = "@**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
' "@King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})

        content = f"@**|{user_id}**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
' "@King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})

    def test_mention_silent(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id

        content = "@_**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_silent_wildcard_mention(self) -> None:
        user_profile = self.example_user("othello")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        wildcards = ["all", "everyone", "stream"]
        for wildcard in wildcards:
            content = f"@_**{wildcard}**"
            rendering_result = render_markdown(msg, content)
            self.assertEqual(
                rendering_result.rendered_content,
                f'
{wildcard}
',
            )
            self.assertFalse(rendering_result.mentions_wildcard)

    def test_mention_invalid_followed_by_valid(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id

        content = "@**Invalid user** and @**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
@Invalid user and '
            "@King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})

    def test_invalid_mention_not_uses_valid_mention_data(self) -> None:
        sender_user_profile = self.example_user("othello")
        hamlet = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        # Even though King Hamlet will be present in mention data as
        # it was was fetched for first mention but second mention is
        # incorrect(as it uses hamlet's id) so it should not be able
        # to use that data for creating a valid mention.

        content = f"@**King Hamlet|{hamlet.id}** and @**aaron|{hamlet.id}**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            f'
'
            f"@King Hamlet and @aaron|{hamlet.id}
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id})

    def test_silent_mention_invalid_followed_by_valid(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id

        content = "@_**Invalid user** and @_**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
@_Invalid user and '
            "King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

        content = f"@_**|123456789** and @_**|{user_id}**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
@_|123456789 and "
            ''
            "King Hamlet
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_possible_mentions(self) -> None:
        def assert_mentions(content: str, names: Set[str], has_wildcards: bool = False) -> None:
            self.assertEqual(possible_mentions(content), (names, has_wildcards))

        aaron = self.example_user("aaron")

        assert_mentions("", set())
        assert_mentions("boring", set())
        assert_mentions("@**all**", set(), True)
        assert_mentions("smush@**steve**smush", set())

        assert_mentions(
            f"Hello @**King Hamlet**, @**|{aaron.id}** and @**Cordelia, Lear's daughter**\n@**Foo van Barson|1234** @**all**",
            {"King Hamlet", f"|{aaron.id}", "Cordelia, Lear's daughter", "Foo van Barson|1234"},
            True,
        )

    def test_mention_multiple(self) -> None:
        sender_user_profile = self.example_user("othello")
        hamlet = self.example_user("hamlet")
        cordelia = self.example_user("cordelia")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "@**King Hamlet** and @**Cordelia, Lear's daughter**, check this out"

        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
"
            '@King Hamlet and '
            '@Cordelia, Lear\'s daughter, '
            "check this out
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id, cordelia.id})

    def test_mention_in_quotes(self) -> None:
        othello = self.example_user("othello")
        hamlet = self.example_user("hamlet")
        cordelia = self.example_user("cordelia")
        msg = Message(sender=othello, sending_client=get_client("test"))

        content = "> @**King Hamlet** and @**Othello, the Moor of Venice**\n\n @**King Hamlet** and @**Cordelia, Lear's daughter**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
\n
"
            f'King Hamlet'
            " and "
            f'Othello, the Moor of Venice'
            "
\n
\n"
            "
"
            f'@King Hamlet'
            " and "
            f'@Cordelia, Lear\'s daughter'
            "
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id, cordelia.id})

        # Both fenced quote and > quote should be identical for both silent and regular syntax.
        expected = (
            "
\n
"
            f'King Hamlet'
            "
\n
"
        )
        content = "```quote\n@**King Hamlet**\n```"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, expected)
        self.assertEqual(rendering_result.mentions_user_ids, set())
        content = "> @**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, expected)
        self.assertEqual(rendering_result.mentions_user_ids, set())
        content = "```quote\n@_**King Hamlet**\n```"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, expected)
        self.assertEqual(rendering_result.mentions_user_ids, set())
        content = "> @_**King Hamlet**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(rendering_result.rendered_content, expected)
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_wildcard_mention_in_quotes(self) -> None:
        user_profile = self.example_user("othello")
        message = Message(sender=user_profile, sending_client=get_client("test"))

        def assert_silent_mention(content: str, wildcard: str) -> None:
            expected = (
                "
\n
"
                f'{wildcard}'
                "
\n
"
            )
            rendering_result = render_markdown(message, content)
            self.assertEqual(rendering_result.rendered_content, expected)
            self.assertFalse(rendering_result.mentions_wildcard)

        wildcards = ["all", "everyone", "stream"]
        for wildcard in wildcards:
            assert_silent_mention(f"> @**{wildcard}**", wildcard)
            assert_silent_mention(f"> @_**{wildcard}**", wildcard)
            assert_silent_mention(f"```quote\n@**{wildcard}**\n```", wildcard)
            assert_silent_mention(f"```quote\n@_**{wildcard}**\n```", wildcard)

    def test_mention_duplicate_full_name(self) -> None:
        realm = get_realm("zulip")

        def make_user(email: str, full_name: str) -> UserProfile:
            return create_user(
                email=email,
                password="whatever",
                realm=realm,
                full_name=full_name,
            )

        sender_user_profile = self.example_user("othello")
        twin1 = make_user("twin1@example.com", "Mark Twin")
        twin2 = make_user("twin2@example.com", "Mark Twin")
        cordelia = self.example_user("cordelia")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = f"@**Mark Twin|{twin1.id}**, @**Mark Twin|{twin2.id}** and @**Cordelia, Lear's daughter**, hi."

        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
"
            '@Mark Twin, '
            '@Mark Twin and '
            '@Cordelia, Lear\'s daughter, '
            "hi.
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {twin1.id, twin2.id, cordelia.id})

    def test_mention_invalid(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "Hey @**Nonexistent User**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content, "
Hey @Nonexistent User
"
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_user_mention_atomic_string(self) -> None:
        sender_user_profile = self.example_user("othello")
        realm = get_realm("zulip")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        # Create a linkifier.
        url_format_string = r"https://trac.example.com/ticket/%(id)s"
        linkifier = RealmFilter(
            realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
        )
        linkifier.save()
        self.assertEqual(
            linkifier.__str__(),
            "[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
        )
        # Create a user that potentially interferes with the pattern.
        test_user = create_user(
            email="atomic@example.com",
            password="whatever",
            realm=realm,
            full_name="Atomic #123",
        )
        content = "@**Atomic #123**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "@Atomic #123
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {test_user.id})
        content = "@_**Atomic #123**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "Atomic #123
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def create_user_group_for_test(self, user_group_name: str) -> UserGroup:
        othello = self.example_user("othello")
        return create_user_group(user_group_name, [othello], get_realm("zulip"))

    def test_user_group_mention_single(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id
        user_group = self.create_user_group_for_test("support")

        content = "@**King Hamlet** @*support*"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "@King Hamlet "
            ''
            "@support
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})
        self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id})

    def test_invalid_user_group_followed_by_valid_mention_single(self) -> None:
        sender_user_profile = self.example_user("othello")
        user_profile = self.example_user("hamlet")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_id = user_profile.id
        user_group = self.create_user_group_for_test("support")

        content = "@**King Hamlet** @*Invalid user group* @*support*"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "@King Hamlet "
            "@Invalid user group "
            ''
            "@support
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})
        self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id})

    def test_user_group_mention_atomic_string(self) -> None:
        sender_user_profile = self.example_user("othello")
        realm = get_realm("zulip")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        user_profile = self.example_user("hamlet")
        # Create a linkifier.
        url_format_string = r"https://trac.example.com/ticket/%(id)s"
        linkifier = RealmFilter(
            realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
        )
        linkifier.save()
        self.assertEqual(
            linkifier.__str__(),
            "[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
        )
        # Create a user-group that potentially interferes with the pattern.
        user_id = user_profile.id
        user_group = self.create_user_group_for_test("support #123")

        content = "@**King Hamlet** @*support #123*"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            '
'
            "@King Hamlet "
            ''
            "@support #123
",
        )
        self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id})
        self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id})

    def test_possible_user_group_mentions(self) -> None:
        def assert_mentions(content: str, names: Set[str]) -> None:
            self.assertEqual(possible_user_group_mentions(content), names)

        assert_mentions("", set())
        assert_mentions("boring", set())
        assert_mentions("@**all**", set())
        assert_mentions("smush@*steve*smush", set())

        assert_mentions(
            "@*support* Hello @**King Hamlet** and @**Cordelia, Lear's daughter**\n"
            "@**Foo van Barson** @**all**",
            {"support"},
        )

        assert_mentions(
            "Attention @*support*, @*frontend* and @*backend*\ngroups.",
            {"support", "frontend", "backend"},
        )

    def test_user_group_mention_multiple(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        support = self.create_user_group_for_test("support")
        backend = self.create_user_group_for_test("backend")

        content = "@*support* and @*backend*, check this out"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
"
            ''
            "@support "
            "and "
            ''
            "@backend, "
            "check this out"
            "
",
        )

        self.assertEqual(rendering_result.mentions_user_group_ids, {support.id, backend.id})

    def test_user_group_mention_edit(self) -> None:
        sender_user_profile = self.example_user("hamlet")
        user_profile = self.example_user("othello")
        self.create_user_group_for_test("support")
        self.login("hamlet")

        msg_id = self.send_stream_message(
            sender_user_profile, "Denmark", topic_name="editing", content="test"
        )

        def update_message_and_check_flag(content: str, mentioned: bool) -> None:
            result = self.client_patch(
                "/json/messages/" + str(msg_id),
                {
                    "message_id": msg_id,
                    "content": content,
                },
            )
            self.assert_json_success(result)
            um = UserMessage.objects.get(
                user_profile_id=user_profile.id,
                message_id=msg_id,
            )
            if mentioned:
                self.assertIn("mentioned", um.flags_list())
            else:
                self.assertNotIn("mentioned", um.flags_list())

        update_message_and_check_flag("@*support*", True)
        update_message_and_check_flag("@*support-invalid* edited", False)
        update_message_and_check_flag("@*support* edited", True)
        update_message_and_check_flag("edited", False)
        update_message_and_check_flag("@*support*", True)
        update_message_and_check_flag("@_*support*", False)

    def test_user_group_mention_invalid(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "Hey @*Nonexistent group*"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content, "
Hey @Nonexistent group
"
        )
        self.assertEqual(rendering_result.mentions_user_group_ids, set())

    def test_user_group_silent_mention(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        support = self.create_user_group_for_test("support")

        content = "We'll add you to @_*support* user group."
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content,
            "
We'll add you to "
            f'support'
            " user group.
",
        )

        self.assertEqual(rendering_result.mentions_user_group_ids, set())

    def test_user_group_mention_in_quotes(self) -> None:
        user_profile = self.example_user("othello")
        message = Message(sender=user_profile, sending_client=get_client("test"))
        backend = self.create_user_group_for_test("backend")

        def assert_silent_mention(content: str) -> None:
            expected = (
                "
\n
"
                f'backend'
                "
\n
"
            )
            rendering_result = render_markdown(message, content)
            self.assertEqual(rendering_result.rendered_content, expected)
            self.assertEqual(rendering_result.mentions_user_group_ids, set())

        assert_silent_mention("> @*backend*")
        assert_silent_mention("> @_*backend*")
        assert_silent_mention("```quote\n@*backend*\n```")
        assert_silent_mention("```quote\n@_*backend*\n```")

    def test_stream_single(self) -> None:
        denmark = get_stream("Denmark", get_realm("zulip"))
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**Denmark**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{d.name}
'.format(
                d=denmark,
            ),
        )

    def test_invalid_stream_followed_by_valid_mention(self) -> None:
        denmark = get_stream("Denmark", get_realm("zulip"))
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**Invalid** and #**Denmark**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#Invalid and #{d.name}
'.format(
                d=denmark,
            ),
        )

    def test_stream_multiple(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        realm = get_realm("zulip")
        denmark = get_stream("Denmark", realm)
        scotland = get_stream("Scotland", realm)
        content = "Look to #**Denmark** and #**Scotland**, there something"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            "
Look to "
            '#{denmark.name} and '
            '#{scotland.name}, '
            "there something
".format(denmark=denmark, scotland=scotland),
        )

    def test_stream_case_sensitivity(self) -> None:
        realm = get_realm("zulip")
        case_sens = Stream.objects.create(name="CaseSens", realm=realm)
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**CaseSens**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{s.name}
'.format(
                s=case_sens,
            ),
        )

    def test_stream_case_sensitivity_nonmatching(self) -> None:
        """#StreamName requires the stream be spelled with the correct case
        currently.  If we change that in the future, we'll need to change this
        test."""
        realm = get_realm("zulip")
        Stream.objects.create(name="CaseSens", realm=realm)
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**casesens**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content, "
#casesens
"
        )

    def test_topic_single(self) -> None:
        denmark = get_stream("Denmark", get_realm("zulip"))
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**Denmark>some topic**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{d.name} > some topic
'.format(
                d=denmark,
            ),
        )

    def test_topic_atomic_string(self) -> None:
        realm = get_realm("zulip")
        # Create a linkifier.
        sender_user_profile = self.example_user("othello")
        url_format_string = r"https://trac.example.com/ticket/%(id)s"
        linkifier = RealmFilter(
            realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
        )
        linkifier.save()
        self.assertEqual(
            linkifier.__str__(),
            "[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
        )
        # Create a topic link that potentially interferes with the pattern.
        denmark = get_stream("Denmark", realm)
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**Denmark>#1234**"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{d.name} > #1234
'.format(
                d=denmark,
            ),
        )

    def test_topic_multiple(self) -> None:
        denmark = get_stream("Denmark", get_realm("zulip"))
        scotland = get_stream("Scotland", get_realm("zulip"))
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "This has two links: #**Denmark>some topic** and #**Scotland>other topic**."
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            "
This has two links: "
            ''
            "#{denmark.name} > some topic"
            " and "
            ''
            "#{scotland.name} > other topic"
            ".
".format(denmark=denmark, scotland=scotland),
        )

    def test_possible_stream_names(self) -> None:
        content = """#**test here**
            This mentions #**Denmark** too.
            #**garçon** #**천국** @**Ignore Person**
        """
        self.assertEqual(
            possible_linked_stream_names(content),
            {"test here", "Denmark", "garçon", "천국"},
        )

    def test_stream_unicode(self) -> None:
        realm = get_realm("zulip")
        uni = Stream.objects.create(name="привет", realm=realm)
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**привет**"
        quoted_name = ".D0.BF.D1.80.D0.B8.D0.B2.D0.B5.D1.82"
        href = f"/#narrow/stream/{uni.id}-{quoted_name}"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{s.name}
'.format(
                s=uni,
                href=href,
            ),
        )

    def test_stream_atomic_string(self) -> None:
        realm = get_realm("zulip")
        # Create a linkifier.
        sender_user_profile = self.example_user("othello")
        url_format_string = r"https://trac.example.com/ticket/%(id)s"
        linkifier = RealmFilter(
            realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
        )
        linkifier.save()
        self.assertEqual(
            linkifier.__str__(),
            "[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
        )
        # Create a stream that potentially interferes with the pattern.
        stream = Stream.objects.create(name="Stream #1234", realm=realm)
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
        content = "#**Stream #1234**"
        href = f"/#narrow/stream/{stream.id}-Stream-.231234"
        self.assertEqual(
            render_markdown(msg, content).rendered_content,
            '
#{s.name}
'.format(
                s=stream,
                href=href,
            ),
        )

    def test_stream_invalid(self) -> None:
        sender_user_profile = self.example_user("othello")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "There #**Nonexistentstream**"
        rendering_result = render_markdown(msg, content)
        self.assertEqual(
            rendering_result.rendered_content, "
There #Nonexistentstream
"
        )
        self.assertEqual(rendering_result.mentions_user_ids, set())

    def test_image_preview_title(self) -> None:
        msg = "[My favorite image](https://example.com/testimage.png)"
        converted = markdown_convert_wrapper(msg)
        self.assertEqual(
            converted,
            "
"
            'My favorite image'
            "
\n"
            '
'
            ''
            ''
            ""
            "
",
        )

    def test_mit_rendering(self) -> None:
        """Test the Markdown configs for the MIT Zephyr mirroring system;
        verifies almost all inline patterns are disabled, but
        inline_interesting_links is still enabled"""
        msg = "**test**"
        realm = get_realm("zephyr")
        client = get_client("zephyr_mirror")
        message = Message(sending_client=client, sender=self.mit_user("sipbtest"))
        converted = markdown_convert(msg, message_realm=realm, message=message)
        self.assertEqual(
            converted.rendered_content,
            "
**test**
",
        )
        msg = "* test"
        converted = markdown_convert(msg, message_realm=realm, message=message)
        self.assertEqual(
            converted.rendered_content,
            "
* test
",
        )
        msg = "https://lists.debian.org/debian-ctte/2014/02/msg00173.html"
        converted = markdown_convert(msg, message_realm=realm, message=message)
        self.assertEqual(
            converted.rendered_content,
            '
https://lists.debian.org/debian-ctte/2014/02/msg00173.html
',
        )

    def test_url_to_a(self) -> None:
        url = "javascript://example.com/invalidURL"
        converted = url_to_a(db_data=None, url=url, text=url)
        self.assertEqual(
            converted,
            "javascript://example.com/invalidURL",
        )

    def test_disabled_code_block_processor(self) -> None:
        msg = (
            "Hello,\n\n"
            + "    I am writing this message to test something. I am writing this message to test something."
        )
        converted = markdown_convert_wrapper(msg)
        expected_output = (
            "
Hello,
\n"
            + '
I am writing this message to test something. I am writing this message to test something.\n'
            + "
"
        )
        self.assertEqual(converted, expected_output)

        realm = do_create_realm(
            string_id="code_block_processor_test", name="code_block_processor_test"
        )
        maybe_update_markdown_engines(realm.id, True)
        rendering_result = markdown_convert(msg, message_realm=realm, email_gateway=True)
        expected_output = (
            "
Hello,
\n"
            + "
I am writing this message to test something. I am writing this message to test something.
"
        )
        self.assertEqual(rendering_result.rendered_content, expected_output)

    def test_normal_link(self) -> None:
        realm = get_realm("zulip")
        sender_user_profile = self.example_user("othello")
        message = Message(sender=sender_user_profile, sending_client=get_client("test"))
        msg = "http://example.com/#settings/"

        self.assertEqual(
            markdown_convert(msg, message_realm=realm, message=message).rendered_content,
            '
http://example.com/#settings/
',
        )

    def test_relative_link(self) -> None:
        realm = get_realm("zulip")
        sender_user_profile = self.example_user("othello")
        message = Message(sender=sender_user_profile, sending_client=get_client("test"))
        msg = "http://zulip.testserver/#narrow/stream/999-hello"

        self.assertEqual(
            markdown_convert(msg, message_realm=realm, message=message).rendered_content,
            '
http://zulip.testserver/#narrow/stream/999-hello
',
        )

    def test_relative_link_streams_page(self) -> None:
        realm = get_realm("zulip")
        sender_user_profile = self.example_user("othello")
        message = Message(sender=sender_user_profile, sending_client=get_client("test"))
        msg = "http://zulip.testserver/#streams/all"

        self.assertEqual(
            markdown_convert(msg, message_realm=realm, message=message).rendered_content,
            '
http://zulip.testserver/#streams/all
',
        )

    def test_md_relative_link(self) -> None:
        realm = get_realm("zulip")
        sender_user_profile = self.example_user("othello")
        message = Message(sender=sender_user_profile, sending_client=get_client("test"))
        msg = "[hello](http://zulip.testserver/#narrow/stream/999-hello)"

        self.assertEqual(
            markdown_convert(msg, message_realm=realm, message=message).rendered_content,
            '
hello
',
        )

    def test_html_entity_conversion(self) -> None:
        msg = """\
            Test raw: Hello, ©
            Test inline code: `©`

            Test fenced code:
            ```
            ©
            ©
            ```

            Test quote:
            ~~~quote
            ©
            ~~~

            Test a list:
            * ©
            * `©`
            * ```©```

            Test an indented block:

                ©"""

        expected_output = """\
            
Test raw: Hello, ©

            Test inline code: &copy;

            
Test fenced code:

            
&copy;
            &copy;
            

            
Test quote:

            

            
©

            

            
Test a list:

            
    
            
  • ©
    • 
            
  • &copy;
    • 
            
  • &copy;
    • 
            

            
Test an indented block:

            
&copy;
            
"""

        converted = markdown_convert_wrapper(dedent(msg))
        self.assertEqual(converted, dedent(expected_output))


class MarkdownApiTests(ZulipTestCase):
    def test_render_message_api(self) -> None:
        content = "That is a **bold** statement"
        result = self.api_post(
            self.example_user("othello"),
            "/api/v1/messages/render",
            dict(content=content),
        )
        self.assert_json_success(result)
        self.assertEqual(
            result.json()["rendered"], "
That is a bold statement
"
        )

    def test_render_mention_stream_api(self) -> None:
        """Determines whether we're correctly passing the realm context"""
        content = "This mentions #**Denmark** and @**King Hamlet**."
        result = self.api_post(
            self.example_user("othello"),
            "/api/v1/messages/render",
            dict(content=content),
        )
        self.assert_json_success(result)
        user_id = self.example_user("hamlet").id
        stream_id = get_stream("Denmark", get_realm("zulip")).id
        self.assertEqual(
            result.json()["rendered"],
            f'
This mentions #Denmark and @King Hamlet.
',
        )


class MarkdownErrorTests(ZulipTestCase):
    def test_markdown_error_handling(self) -> None:
        with self.simulated_markdown_failure():
            with self.assertRaises(MarkdownRenderingException):
                markdown_convert_wrapper("")

    def test_send_message_errors(self) -> None:

        message = "whatever"
        with self.simulated_markdown_failure():
            # We don't use assertRaisesRegex because it seems to not
            # handle i18n properly here on some systems.
            with self.assertRaises(JsonableError):
                self.send_stream_message(self.example_user("othello"), "Denmark", message)

    @override_settings(MAX_MESSAGE_LENGTH=10)
    def test_ultra_long_rendering(self) -> None:
        """A rendered message with an ultra-long length (> 100 * MAX_MESSAGE_LENGTH)
        throws an exception"""
        msg = "mock rendered message\n" * 10 * settings.MAX_MESSAGE_LENGTH

        with mock.patch("zerver.lib.markdown.timeout", return_value=msg), mock.patch(
            "zerver.lib.markdown.markdown_logger"
        ):
            with self.assertRaises(MarkdownRenderingException):
                markdown_convert_wrapper(msg)

    def test_curl_code_block_validation(self) -> None:
        processor = SimulatedFencedBlockPreprocessor(Markdown())
        processor.run_content_validators = True

        markdown_input = [
            "``` curl",
            "curl {{ api_url }}/v1/register",
            "    -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
            '    -d "queue_id=1375801870:2942"',
            "```",
        ]

        with self.assertRaises(MarkdownRenderingException):
            processor.run(markdown_input)

    def test_curl_code_block_without_validation(self) -> None:
        processor = SimulatedFencedBlockPreprocessor(Markdown())

        markdown_input = [
            "``` curl",
            "curl {{ api_url }}/v1/register",
            "    -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
            '    -d "queue_id=1375801870:2942"',
            "```",
        ]
        expected = [
            "",
            "**curl:curl {{ api_url }}/v1/register",
            "    -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
            '    -d "queue_id=1375801870:2942"**',
            "",
            "",
        ]

        result = processor.run(markdown_input)
        self.assertEqual(result, expected)