The quick brown fox jumps over the lazy dog. Animal stories are fun, yeah
" str2 = "The fast fox jumps over the lazy dogs and cats. Animal stories are fun
" expected_diff = "\u001b[34m-\u001b[0mThe \u001b[33mquick brown\u001b[0m fox jumps over the lazy dog. Animal stories are fun\u001b[31m, yeah\u001b[0m
\n\u001b[34m+\u001b[0mThe \u001b[33mfast\u001b[0m fox jumps over the lazy dog\u001b[32ms and cats\u001b[0m. Animal stories are fun
\n" self.assertEqual(diff_strings(str1, str2), expected_diff) def test_get_possible_mentions_info(self) -> None: realm = get_realm("zulip") def make_user(email: str, full_name: str) -> UserProfile: return create_user( email=email, password="whatever", realm=realm, full_name=full_name, ) fred1 = make_user("fred1@example.com", "Fred Flintstone") fred1.is_active = False fred1.save() fred2 = make_user("fred2@example.com", "Fred Flintstone") fred3 = make_user("fred3@example.com", "Fred Flintstone") fred3.is_active = False fred3.save() fred4 = make_user("fred4@example.com", "Fred Flintstone") lst = get_possible_mentions_info( realm.id, {"Fred Flintstone", "cordelia LEAR", "Not A User"} ) set_of_names = set(map(lambda x: x["full_name"].lower(), lst)) self.assertEqual(set_of_names, {"fred flintstone", "cordelia lear"}) by_id = {row["id"]: row for row in lst} self.assertEqual( by_id.get(fred2.id), dict( email=fred2.email, full_name="Fred Flintstone", id=fred2.id, ), ) self.assertEqual( by_id.get(fred4.id), dict( email=fred4.email, full_name="Fred Flintstone", id=fred4.id, ), ) def test_mention_data(self) -> None: realm = get_realm("zulip") hamlet = self.example_user("hamlet") cordelia = self.example_user("cordelia") content = "@**King Hamlet** @**Cordelia lear**" mention_data = MentionData(realm.id, content) self.assertEqual(mention_data.get_user_ids(), {hamlet.id, cordelia.id}) self.assertEqual( mention_data.get_user_by_id(hamlet.id), dict( email=hamlet.email, full_name=hamlet.full_name, id=hamlet.id, ), ) user = mention_data.get_user_by_name("king hamLET") assert user is not None self.assertEqual(user["email"], hamlet.email) self.assertFalse(mention_data.message_has_wildcards()) content = "@**King Hamlet** @**Cordelia lear** @**all**" mention_data = MentionData(realm.id, content) self.assertTrue(mention_data.message_has_wildcards()) def test_invalid_katex_path(self) -> None: with self.settings(DEPLOY_ROOT="/nonexistent"): with self.assertLogs(level="ERROR") as m: render_tex("random text") self.assertEqual(m.output, ["ERROR:root:Cannot find KaTeX for latex rendering!"]) class MarkdownListPreprocessorTest(ZulipTestCase): # We test that the preprocessor inserts blank lines at correct places. # We use <> to indicate that we need to insert a blank line here. def split_message(self, msg: str) -> Tuple[List[str], List[str]]: original = msg.replace("<>", "").split("\n") expected = re.split(r"\n|<>", msg) return original, expected def test_basic_list(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message("List without a gap\n<>* One\n* Two") self.assertEqual(preprocessor.run(original), expected) def test_list_after_quotes(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message( "```quote\nSomething\n```\n\nList without a gap\n<>* One\n* Two" ) self.assertEqual(preprocessor.run(original), expected) def test_list_in_code(self) -> None: preprocessor = MarkdownListPreprocessor() original, expected = self.split_message("```\nList without a gap\n* One\n* Two\n```") self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_different_fences(self) -> None: preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one * two ~~~ This is a nested code fence, do not make changes here: * one * two ````quote Quote in code fence. Should not convert: * one * two ```` ~~~ Back in the quote. We should convert:<> * one * two ``` Outside. Should convert:<> * one * two """ original, expected = self.split_message(msg) self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_same_fence(self) -> None: preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one * two ```python This is a nested code fence, do not make changes here: * one * two ```quote Quote in code fence. Should not convert: * one * two ``` ``` Back in the quote. We should convert:<> * one * two ``` Outside. Should convert:<> * one * two """ original, expected = self.split_message(msg) self.assertEqual(preprocessor.run(original), expected) class MarkdownTest(ZulipTestCase): def setUp(self) -> None: super().setUp() clear_state_for_testing() def assertEqual(self, first: Any, second: Any, msg: str = "") -> None: if isinstance(first, str) and isinstance(second, str): if first != second: raise AssertionError( "Actual and expected outputs do not match; showing diff.\n" + diff_strings(first, second) + msg ) else: super().assertEqual(first, second) def load_markdown_tests(self) -> Tuple[Dict[str, Any], List[List[str]]]: test_fixtures = {} with open( os.path.join(os.path.dirname(__file__), "fixtures/markdown_test_cases.json"), "rb" ) as f: data = orjson.loads(f.read()) for test in data["regular_tests"]: test_fixtures[test["name"]] = test return test_fixtures, data["linkify_tests"] def test_markdown_no_ignores(self) -> None: # We do not want any ignored tests to be committed and merged. format_tests, linkify_tests = self.load_markdown_tests() for name, test in format_tests.items(): message = f'Test "{name}" shouldn\'t be ignored.' is_ignored = test.get("ignore", False) self.assertFalse(is_ignored, message) def test_markdown_fixtures(self) -> None: format_tests, linkify_tests = self.load_markdown_tests() valid_keys = { "name", "input", "expected_output", "backend_only_rendering", "marked_expected_output", "text_content", "translate_emoticons", "ignore", } for name, test in format_tests.items(): with self.subTest(markdown_test_case=name): # Check that there aren't any unexpected keys as those are often typos self.assertEqual(len(set(test.keys()) - valid_keys), 0) # Ignore tests if specified if test.get("ignore", False): continue # nocoverage if test.get("translate_emoticons", False): # Create a userprofile and send message with it. user_profile = self.example_user("othello") do_set_user_display_setting(user_profile, "translate_emoticons", True) msg = Message(sender=user_profile, sending_client=get_client("test")) converted = render_markdown(msg, test["input"]) else: converted = markdown_convert_wrapper(test["input"]) self.assertEqual(converted, test["expected_output"]) def replaced(payload: str, url: str, phrase: str = "") -> str: if url[:4] == "http": href = url elif "@" in url: href = "mailto:" + url else: href = "http://" + url return payload % (f'{url}',) with mock.patch( "zerver.lib.url_preview.preview.link_embed_data_from_cache", return_value=None ): for inline_url, reference, url in linkify_tests: try: match = replaced(reference, url, phrase=inline_url) except TypeError: match = reference converted = markdown_convert_wrapper(inline_url) self.assertEqual(match, converted) def test_inline_file(self) -> None: msg = "Check out this file file:///Volumes/myserver/Users/Shared/pi.py" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, 'Check out this file file:///Volumes/myserver/Users/Shared/pi.py
', ) clear_state_for_testing() with self.settings(ENABLE_FILE_LINKS=False): realm = do_create_realm(string_id="file_links_test", name="file_links_test") maybe_update_markdown_engines(realm.id, False) converted = markdown_convert(msg, message_realm=realm) self.assertEqual( converted, "Check out this file file:///Volumes/myserver/Users/Shared/pi.py
" ) def test_inline_bitcoin(self) -> None: msg = "To bitcoin:1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa or not to bitcoin" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, 'To bitcoin:1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa or not to bitcoin
', ) def test_inline_youtube(self) -> None: msg = "Check out the debate: http://www.youtube.com/watch?v=hx1mjT73xYE" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, 'Check out the debate: http://www.youtube.com/watch?v=hx1mjT73xYE
\nhttp://www.youtube.com/watch?v=hx1mjT73xYE
\nhttps://www.youtube.com/playlist?list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo
', ) msg = ( "https://www.youtube.com/playlist?v=O5nskjZ_GoI&list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, 'https://www.youtube.com/playlist?v=O5nskjZ_GoI&list=PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo
\n
http://www.youtube.com/watch_videos?video_ids=nOJgD4fcZhI,i96UO8-GFvw
\n
Check out the debate: https://vimeo.com/246979354
', ) msg = "https://vimeo.com/246979354" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, '', ) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_thumbnail_url(self) -> None: realm = get_realm("zephyr") msg = "[foobar](/user_uploads/{realm_id}/50/w2G6ok9kr8AMCQCTNAUOFMln/IMG_0677.JPG)" msg = msg.format(realm_id=realm.id) thumbnail_img = '
<'
thumbnail_img = thumbnail_img.format(realm_id=realm.id)
converted = markdown_convert_wrapper(msg)
self.assertIn(thumbnail_img, converted)
msg = "https://www.google.com/images/srpr/logo4w.png"
thumbnail_img = '
'
converted = markdown_convert_wrapper(msg)
self.assertIn(thumbnail_img, converted)
msg = "www.google.com/images/srpr/logo4w.png"
thumbnail_img = '
'
converted = markdown_convert_wrapper(msg)
self.assertIn(thumbnail_img, converted)
msg = "https://www.google.com/images/srpr/logo4w.png"
thumbnail_img = '
http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg
' content = "http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg" sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted, with_preview) realm = msg.get_realm() setattr(realm, "inline_image_preview", False) realm.save() sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted, without_preview) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_quoted_blocks(self) -> None: content = "http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg" expected = '\n\n\n
Awesome!
' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted, expected) content = ">* http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg\n\nAwesome!" expected = '\n\n\n
Awesome!
' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) self.assertEqual(converted, expected) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_preview_order(self) -> None: realm = get_realm("zulip") content = "http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg" expected = 'http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg
\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg
\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg
\n\nhttp://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg
\n
Test 1
\n21136101110_1dde1c1a7e_o.jpg
Next Image
\nIMG_20161116_023910.jpg
Another Screenshot
\nScreenshot-from-2016-06-01-16-22-42.png
Look at how hilarious our old office was: https://www.dropbox.com/s/ymdijjcg67hv2ta/IMG_0923.JPG
\n
Look at my hilarious drawing folder: https://www.dropbox.com/sh/cm39k9e04z7fhim/AAAII5NK-9daee3FcF41anEua?dl=
\n
https://www.dropbox.com/sc/tditp9nitko60n5/03rEiZldy5
\n
Look at the new dropbox logo: https://www.dropbox.com/static/images/home_logo.png
\n
https://zulip-test.dropbox.com/photos/cl/ROmr9K1XYtmpneM
', ) def test_inline_github_preview(self) -> None: # Test photo album previews msg = "Test: https://github.com/zulip/zulip/blob/master/static/images/logo/zulip-icon-128x128.png" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, 'Test: https://github.com/zulip/zulip/blob/master/static/images/logo/zulip-icon-128x128.png
\n
Test: https://developer.github.com/assets/images/hero-circuit-bg.png
\n
Check out this PyCon video
\n
https://www.youtube.com/watch?v=0c46YHS3RY8
\nSample text
\nhttps://www.youtube.com/watch?v=lXFO2ULktEI
\n
{}
".format(make_link("http://www.twitter.com"))) msg = "http://www.twitter.com/wdaher/" converted = markdown_convert_wrapper(msg) self.assertEqual(converted, "{}
".format(make_link("http://www.twitter.com/wdaher/"))) msg = "http://www.twitter.com/wdaher/status/3" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
".format(make_link("http://www.twitter.com/wdaher/status/3")) ) # id too long msg = "http://www.twitter.com/wdaher/status/2879779692873154569" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
".format( make_link("http://www.twitter.com/wdaher/status/2879779692873154569") ), ) # id too large (i.e. tweet doesn't exist) msg = "http://www.twitter.com/wdaher/status/999999999999999999" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
".format( make_link("http://www.twitter.com/wdaher/status/999999999999999999") ), ) msg = "http://www.twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
\n{}".format( make_link("http://www.twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://www.twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) msg = "https://www.twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
\n{}".format( make_link("https://www.twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "https://www.twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) msg = "http://twitter.com/wdaher/status/287977969287315456" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) # Repeated links will only be converted once msg = ( "http://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315457 " "http://twitter.com/wdaher/status/287977969287315457 " "http://twitter.com/wdaher/status/287977969287315457" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{} {} {} {}
\n{}{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), ), ) # A max of 3 will be converted msg = ( "http://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315457 " "https://twitter.com/wdaher/status/287977969287315456 " "http://twitter.com/wdaher/status/287977969287315460" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{} {} {} {}
\n{}{}{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_link("https://twitter.com/wdaher/status/287977969287315456"), make_link("http://twitter.com/wdaher/status/287977969287315460"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), make_inline_twitter_preview( "https://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), ), ) # Test smart in-place inlining behavior: msg = ( "Paragraph 1: http://twitter.com/wdaher/status/287977969287315456\n\n" "Paragraph 2\n\n" "Paragraph 3: http://twitter.com/wdaher/status/287977969287315457" ) converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "Paragraph 1: {}
\n{}Paragraph 2
\nParagraph 3: {}
\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315456"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315456", normal_tweet_html ), make_link("http://twitter.com/wdaher/status/287977969287315457"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315457", normal_tweet_html ), ), ) # Tweet has a mention in a URL, only the URL is linked msg = "http://twitter.com/wdaher/status/287977969287315458" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315458"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315458", mention_in_link_tweet_html, ), ), ) # Tweet with an image msg = "http://twitter.com/wdaher/status/287977969287315459" converted = markdown_convert_wrapper(msg) self.assertEqual( converted, "{}
\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315459"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315459", media_tweet_html, ( '
'
""
"{}
\n{}".format( make_link("http://twitter.com/wdaher/status/287977969287315460"), make_inline_twitter_preview( "http://twitter.com/wdaher/status/287977969287315460", emoji_in_tweet_html ), ), ) # Test Twitter previews in spoiler tags. msg = "```spoiler secret tweet\nTweet: http://twitter.com/wdaher/status/287977969287315456\n```" converted = markdown_convert_wrapper(msg) rendered_spoiler = 'secret tweet
\n{}
".format(emoji_img(":green_tick:", realm_emoji.file_name, realm.id)), ) # Deactivate realm emoji. do_remove_realm_emoji(realm, "green_tick") converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted, ":green_tick:
") def test_deactivated_realm_emoji(self) -> None: # Deactivate realm emoji. realm = get_realm("zulip") do_remove_realm_emoji(realm, "green_tick") msg = Message(sender=self.example_user("hamlet")) converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted, ":green_tick:
") def test_unicode_emoji(self) -> None: msg = "\u2615" # ☕ converted = markdown_convert_wrapper(msg) self.assertEqual( converted, ':coffee:
', ) msg = "\u2615\u2615" # ☕☕ converted = markdown_convert_wrapper(msg) self.assertEqual( converted, ':coffee::coffee:
', ) def test_no_translate_emoticons_if_off(self) -> None: user_profile = self.example_user("othello") do_set_user_display_setting(user_profile, "translate_emoticons", False) msg = Message(sender=user_profile, sending_client=get_client("test")) content = ":)" expected = ":)
" converted = render_markdown(msg, content) self.assertEqual(converted, expected) def test_same_markup(self) -> None: msg = "\u2615" # ☕ unicode_converted = markdown_convert_wrapper(msg) msg = ":coffee:" # ☕☕ converted = markdown_convert_wrapper(msg) self.assertEqual(converted, unicode_converted) def test_links_in_topic_name(self) -> None: realm = get_realm("zulip") msg = Message(sender=self.example_user("othello")) msg.set_topic_name("https://google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ["https://google.com/hello-world"]) msg.set_topic_name("http://google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ["http://google.com/hello-world"]) msg.set_topic_name("Without scheme google.com/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ["https://google.com/hello-world"]) msg.set_topic_name("Without scheme random.words/hello-world") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, []) msg.set_topic_name( "Try out http://ftp.debian.org, https://google.com/ and https://google.in/." ) converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, ["http://ftp.debian.org", "https://google.com/", "https://google.in/"] ) def test_realm_patterns(self) -> None: realm = get_realm("zulip") url_format_string = r"https://trac.example.com/ticket/%(id)s" realm_filter = RealmFilter( realm=realm, pattern=r"#(?PWe should fix #224 and #115, but not issue#124 or #1124z or trac #15 today.
', ) self.assertEqual(converted_topic, ["https://trac.example.com/ticket/444"]) msg.set_topic_name("#444 https://google.com") converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( converted_topic, ["https://trac.example.com/ticket/444", "https://google.com"] ) RealmFilter( realm=realm, pattern=r"#(?P#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging
', ) def assert_conversion(content: str, should_have_converted: bool = True) -> None: converted = markdown_convert(content, message_realm=realm, message=msg) converted_topic = topic_links(realm.id, content) if should_have_converted: self.assertTrue("https://trac.example.com" in converted) self.assertEqual(len(converted_topic), 1) self.assertTrue("https://trac.example.com" in converted_topic[0]) else: self.assertTrue("https://trac.example.com" not in converted) self.assertEqual(len(converted_topic), 0) assert_conversion("Hello #123 World") assert_conversion("Hello #123World", False) assert_conversion("Hello#123 World", False) assert_conversion("Hello#123World", False) # Ideally, these should be converted, but Markdown doesn't # handle word boundary detection in languages that don't use # whitespace for that correctly yet. assert_conversion("チケットは#123です", False) assert_conversion("チケットは #123です", False) assert_conversion("チケットは#123 です", False) assert_conversion("チケットは #123 です") assert_conversion("(#123)") assert_conversion("#123>") assert_conversion('"#123"') assert_conversion("#123@") assert_conversion(")#123(", False) assert_conversion("##123", False) # test nested realm patterns should avoid double matching RealmFilter( realm=realm, pattern=r"hello#(?PWe should fix ABC-123 or trac ABC-123 today.
', ) # Both the links should be generated in topics. self.assertEqual( converted_topic, [ "https://trac.example.com/ticket/ABC-123", "https://other-trac.example.com/ticket/ABC-123", ], ) def test_maybe_update_markdown_engines(self) -> None: realm = get_realm("zulip") url_format_string = r"https://trac.example.com/ticket/%(id)s" realm_filter = RealmFilter( realm=realm, pattern=r"#(?P/me makes a list
\n/me takes a walk
", ) self.assertTrue(Message.is_status_message(content, rendered_content)) content = "/me writes a second line\nline" rendered_content = render_markdown(msg, content) self.assertEqual( rendered_content, "/me writes a second line
\nline
We have an ALERTWORD day today!
") self.assertEqual(msg.user_ids_with_alert_words, {user_profile.id}) msg = Message(sender=user_profile, sending_client=get_client("test")) content = "We have a NOTHINGWORD day today!" self.assertEqual(render(msg, content), "We have a NOTHINGWORD day today!
") self.assertEqual(msg.user_ids_with_alert_words, set()) def test_alert_words_returns_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["how"], "cordelia": ["this possible"], "iago": ["hello"], "prospero": ["hello"], "othello": ["how are you"], "aaron": ["hey"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = "hello how is this possible how are you doing today" render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, user_profiles["iago"].id, user_profiles["prospero"].id, user_profiles["othello"].id, } # All users except aaron have their alert word appear in the message content self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_1(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["provisioning", "Prod deployment"], "cordelia": ["test", "Prod"], "iago": ["prod"], "prospero": ["deployment"], "othello": ["last"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """Hello, everyone. Prod deployment has been completed And this is a new line to test out how Markdown convert this into something line ending split array and this is a new line last""" render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, user_profiles["iago"].id, user_profiles["prospero"].id, user_profiles["othello"].id, } # All users have their alert word appear in the message content self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_in_french(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["réglementaire", "une politique", "une merveille"], "cordelia": ["énormément", "Prod"], "iago": ["prod"], "prospero": ["deployment"], "othello": ["last"], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """This is to test out alert words work in languages with accented characters too bonjour est (énormément) ce a quoi ressemble le français et j'espère qu'il n'y n' réglementaire a pas de mots d'alerte dans ce texte français """ render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id, user_profiles["cordelia"].id} # Only hamlet and cordelia have their alert-words appear in the message content self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_empty_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": [], "cordelia": [], "iago": [], "prospero": [], "othello": [], "aaron": [], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """hello how is this possible how are you doing today This is to test that the no user_ids who have alrert wourldword is participating in sending of the message """ render(msg, content) expected_user_ids: Set[int] = set() # None of the users have their alert-words appear in the message content self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def get_mock_alert_words(self, num_words: int, word_length: int) -> List[str]: alert_words = ["x" * word_length] * num_words # type List[str] return alert_words def test_alert_words_with_empty_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": [], "cordelia": [], "iago": [], "othello": [], } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """This is to test a empty alert words i.e. no user has any alert-words set""" render(msg, content) expected_user_ids: Set[int] = set() self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def test_alert_words_retuns_user_ids_with_alert_words_with_huge_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { "hamlet": ["issue124"], "cordelia": self.get_mock_alert_words(500, 10), "iago": self.get_mock_alert_words(500, 10), "othello": self.get_mock_alert_words(500, 10), } user_profiles: Dict[str, UserProfile] = {} for (username, alert_words) in alert_words_for_users.items(): user_profile = self.example_user(username) user_profiles.update({username: user_profile}) do_add_alert_words(user_profile, alert_words) sender_user_profile = self.example_user("polonius") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) def render(msg: Message, content: str) -> str: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """The code above will print 10 random values of numbers between 1 and 100. The second line, for x in range(10), determines how many values will be printed (when you use range(x), the number that you use in place of x will be the amount of values that you'll have printed. if you want 20 values, use range(20). use range(5) if you only want 5 values returned, etc.). I was talking abou the issue124 on github. Then the third line: print random.randint(1,101) will automatically select a random integer between 1 and 100 for you. The process is fairly simple """ render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id} # Only hamlet has alert-word 'issue124' present in the message content self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) def test_default_code_block_language(self) -> None: realm = get_realm("zulip") self.assertEqual(realm.default_code_block_language, None) text = "```{}\nconsole.log('Hello World');\n```\n" # Render without default language msg_with_js = markdown_convert_wrapper(text.format("js")) msg_with_python = markdown_convert_wrapper(text.format("python")) msg_without_language = markdown_convert_wrapper(text.format("")) msg_with_quote = markdown_convert_wrapper(text.format("quote")) msg_with_math = markdown_convert_wrapper(text.format("math")) msg_with_none = markdown_convert_wrapper(text.format("none")) # Render with default=javascript do_set_realm_property(realm, "default_code_block_language", "javascript") msg_without_language_default_js = markdown_convert_wrapper(text.format("")) msg_with_python_default_js = markdown_convert_wrapper(text.format("python")) # Render with default=python do_set_realm_property(realm, "default_code_block_language", "python") msg_without_language_default_py = markdown_convert_wrapper(text.format("")) msg_with_none_default_py = markdown_convert_wrapper(text.format("none")) # Render with default=quote do_set_realm_property(realm, "default_code_block_language", "quote") msg_without_language_default_quote = markdown_convert_wrapper(text.format("")) # Render with default=math do_set_realm_property(realm, "default_code_block_language", "math") msg_without_language_default_math = markdown_convert_wrapper(text.format("")) # Render without default language do_set_realm_property(realm, "default_code_block_language", None) msg_without_language_final = markdown_convert_wrapper(text.format("")) self.assertTrue(msg_with_js == msg_without_language_default_js) self.assertTrue( msg_with_python == msg_with_python_default_js == msg_without_language_default_py ) self.assertTrue(msg_with_quote == msg_without_language_default_quote) self.assertTrue(msg_with_math == msg_without_language_default_math) self.assertTrue(msg_without_language == msg_without_language_final) self.assertTrue(msg_with_none == msg_with_none_default_py) # Test checking inside nested quotes nested_text = "````quote\n\n{}\n\n{}````".format(text.format("js"), text.format("")) do_set_realm_property(realm, "default_code_block_language", "javascript") rendered = markdown_convert_wrapper(nested_text) with_language, without_language = re.findall(r"(.*?)$", rendered, re.MULTILINE)
self.assertTrue(with_language == without_language)
do_set_realm_property(realm, "default_code_block_language", None)
rendered = markdown_convert_wrapper(nested_text)
with_language, without_language = re.findall(r"(.*?)$", rendered, re.MULTILINE)
self.assertFalse(with_language == without_language)
def test_mention_wildcard(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@**all** test"
self.assertEqual(
render_markdown(msg, content),
'' "@all" " test
",
)
self.assertTrue(msg.mentions_wildcard)
def test_mention_everyone(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@**everyone** test"
self.assertEqual(
render_markdown(msg, content),
'' "@everyone" " test
",
)
self.assertTrue(msg.mentions_wildcard)
def test_mention_stream(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@**stream** test"
self.assertEqual(
render_markdown(msg, content),
'' "@stream" " test
",
)
self.assertTrue(msg.mentions_wildcard)
def test_mention_at_wildcard(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@all test"
self.assertEqual(render_markdown(msg, content), "@all test
")
self.assertFalse(msg.mentions_wildcard)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_at_everyone(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@everyone test"
self.assertEqual(render_markdown(msg, content), "@everyone test
")
self.assertFalse(msg.mentions_wildcard)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_word_starting_with_at_wildcard(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "test @alleycat.com test"
self.assertEqual(render_markdown(msg, content), "test @alleycat.com test
")
self.assertFalse(msg.mentions_wildcard)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_at_normal_user(self) -> None:
user_profile = self.example_user("othello")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@aaron test"
self.assertEqual(render_markdown(msg, content), "@aaron test
")
self.assertFalse(msg.mentions_wildcard)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_single(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
content = "@**King Hamlet**"
self.assertEqual(
render_markdown(msg, content),
'' "@King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
content = f"@**|{user_id}**"
self.assertEqual(
render_markdown(msg, content),
'' "@King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
def test_mention_silent(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
content = "@_**King Hamlet**"
self.assertEqual(
render_markdown(msg, content),
''
"King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_invalid_followed_by_valid(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
content = "@**Invalid user** and @**King Hamlet**"
self.assertEqual(
render_markdown(msg, content),
'@Invalid user and '
"@King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
def test_invalid_mention_not_uses_valid_mention_data(self) -> None:
sender_user_profile = self.example_user("othello")
hamlet = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
# Even though King Hamlet will be present in mention data as
# it was was fetched for first mention but second mention is
# incorrect(as it uses hamlet's id) so it should not be able
# to use that data for creating a valid mention.
content = f"@**King Hamlet|10** and @**aaron|{hamlet.id}**"
self.assertEqual(
render_markdown(msg, content),
f''
f"@King Hamlet and @aaron|{hamlet.id}
",
)
self.assertEqual(msg.mentions_user_ids, {hamlet.id})
def test_silent_mention_invalid_followed_by_valid(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
content = "@_**Invalid user** and @_**King Hamlet**"
self.assertEqual(
render_markdown(msg, content),
'@_Invalid user and '
"King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, set())
content = f"@_**|123456789** and @_**|{user_id}**"
self.assertEqual(
render_markdown(msg, content),
"@_|123456789 and "
''
"King Hamlet
",
)
self.assertEqual(msg.mentions_user_ids, set())
def test_possible_mentions(self) -> None:
def assert_mentions(content: str, names: Set[str], has_wildcards: bool = False) -> None:
self.assertEqual(possible_mentions(content), (names, has_wildcards))
aaron = self.example_user("aaron")
assert_mentions("", set())
assert_mentions("boring", set())
assert_mentions("@**all**", set(), True)
assert_mentions("smush@**steve**smush", set())
assert_mentions(
f"Hello @**King Hamlet**, @**|{aaron.id}** and @**Cordelia Lear**\n@**Foo van Barson|1234** @**all**",
{"King Hamlet", f"|{aaron.id}", "Cordelia Lear", "Foo van Barson|1234"},
True,
)
def test_mention_multiple(self) -> None:
sender_user_profile = self.example_user("othello")
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "@**King Hamlet** and @**Cordelia Lear**, check this out"
self.assertEqual(
render_markdown(msg, content),
""
'@King Hamlet and '
'@Cordelia Lear, '
"check this out
",
)
self.assertEqual(msg.mentions_user_ids, {hamlet.id, cordelia.id})
def test_mention_in_quotes(self) -> None:
othello = self.example_user("othello")
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
msg = Message(sender=othello, sending_client=get_client("test"))
content = "> @**King Hamlet** and @**Othello, the Moor of Venice**\n\n @**King Hamlet** and @**Cordelia Lear**"
self.assertEqual(
render_markdown(msg, content),
"\n"
f'King Hamlet'
" and "
f'Othello, the Moor of Venice'
"
\n
\n"
""
f'@King Hamlet'
" and "
f'@Cordelia Lear'
"
",
)
self.assertEqual(msg.mentions_user_ids, {hamlet.id, cordelia.id})
# Both fenced quote and > quote should be identical for both silent and regular syntax.
expected = (
"\n"
f'King Hamlet'
"
\n
"
)
content = "```quote\n@**King Hamlet**\n```"
self.assertEqual(render_markdown(msg, content), expected)
self.assertEqual(msg.mentions_user_ids, set())
content = "> @**King Hamlet**"
self.assertEqual(render_markdown(msg, content), expected)
self.assertEqual(msg.mentions_user_ids, set())
content = "```quote\n@_**King Hamlet**\n```"
self.assertEqual(render_markdown(msg, content), expected)
self.assertEqual(msg.mentions_user_ids, set())
content = "> @_**King Hamlet**"
self.assertEqual(render_markdown(msg, content), expected)
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_duplicate_full_name(self) -> None:
realm = get_realm("zulip")
def make_user(email: str, full_name: str) -> UserProfile:
return create_user(
email=email,
password="whatever",
realm=realm,
full_name=full_name,
)
sender_user_profile = self.example_user("othello")
twin1 = make_user("twin1@example.com", "Mark Twin")
twin2 = make_user("twin2@example.com", "Mark Twin")
cordelia = self.example_user("cordelia")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = (
f"@**Mark Twin|{twin1.id}**, @**Mark Twin|{twin2.id}** and @**Cordelia Lear**, hi."
)
self.assertEqual(
render_markdown(msg, content),
""
'@Mark Twin, '
'@Mark Twin and '
'@Cordelia Lear, '
"hi.
",
)
self.assertEqual(msg.mentions_user_ids, {twin1.id, twin2.id, cordelia.id})
def test_mention_invalid(self) -> None:
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "Hey @**Nonexistent User**"
self.assertEqual(
render_markdown(msg, content), "Hey @Nonexistent User
"
)
self.assertEqual(msg.mentions_user_ids, set())
def test_user_mention_atomic_string(self) -> None:
sender_user_profile = self.example_user("othello")
realm = get_realm("zulip")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
# Create a linkifier.
url_format_string = r"https://trac.example.com/ticket/%(id)s"
realm_filter = RealmFilter(
realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
)
realm_filter.save()
self.assertEqual(
realm_filter.__str__(),
"[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
)
# Create a user that potentially interferes with the pattern.
test_user = create_user(
email="atomic@example.com",
password="whatever",
realm=realm,
full_name="Atomic #123",
)
content = "@**Atomic #123**"
self.assertEqual(
render_markdown(msg, content),
''
"@Atomic #123
",
)
self.assertEqual(msg.mentions_user_ids, {test_user.id})
content = "@_**Atomic #123**"
self.assertEqual(
render_markdown(msg, content),
''
"Atomic #123
",
)
self.assertEqual(msg.mentions_user_ids, set())
def create_user_group_for_test(self, user_group_name: str) -> UserGroup:
othello = self.example_user("othello")
return create_user_group(user_group_name, [othello], get_realm("zulip"))
def test_user_group_mention_single(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
user_group = self.create_user_group_for_test("support")
content = "@**King Hamlet** @*support*"
self.assertEqual(
render_markdown(msg, content),
''
"@King Hamlet "
''
"@support
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
self.assertEqual(msg.mentions_user_group_ids, {user_group.id})
def test_invalid_user_group_followed_by_valid_mention_single(self) -> None:
sender_user_profile = self.example_user("othello")
user_profile = self.example_user("hamlet")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_id = user_profile.id
user_group = self.create_user_group_for_test("support")
content = "@**King Hamlet** @*Invalid user group* @*support*"
self.assertEqual(
render_markdown(msg, content),
''
"@King Hamlet "
"@Invalid user group "
''
"@support
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
self.assertEqual(msg.mentions_user_group_ids, {user_group.id})
def test_user_group_mention_atomic_string(self) -> None:
sender_user_profile = self.example_user("othello")
realm = get_realm("zulip")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
user_profile = self.example_user("hamlet")
# Create a linkifier.
url_format_string = r"https://trac.example.com/ticket/%(id)s"
realm_filter = RealmFilter(
realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
)
realm_filter.save()
self.assertEqual(
realm_filter.__str__(),
"[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
)
# Create a user-group that potentially interferes with the pattern.
user_id = user_profile.id
user_group = self.create_user_group_for_test("support #123")
content = "@**King Hamlet** @*support #123*"
self.assertEqual(
render_markdown(msg, content),
''
"@King Hamlet "
''
"@support #123
",
)
self.assertEqual(msg.mentions_user_ids, {user_profile.id})
self.assertEqual(msg.mentions_user_group_ids, {user_group.id})
def test_possible_user_group_mentions(self) -> None:
def assert_mentions(content: str, names: Set[str]) -> None:
self.assertEqual(possible_user_group_mentions(content), names)
assert_mentions("", set())
assert_mentions("boring", set())
assert_mentions("@**all**", set())
assert_mentions("smush@*steve*smush", set())
assert_mentions(
"@*support* Hello @**King Hamlet** and @**Cordelia Lear**\n"
"@**Foo van Barson** @**all**",
{"support"},
)
assert_mentions(
"Attention @*support*, @*frontend* and @*backend*\ngroups.",
{"support", "frontend", "backend"},
)
def test_user_group_mention_multiple(self) -> None:
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
support = self.create_user_group_for_test("support")
backend = self.create_user_group_for_test("backend")
content = "@*support* and @*backend*, check this out"
self.assertEqual(
render_markdown(msg, content),
""
''
"@support "
"and "
''
"@backend, "
"check this out"
"
",
)
self.assertEqual(msg.mentions_user_group_ids, {support.id, backend.id})
def test_user_group_mention_edit(self) -> None:
sender_user_profile = self.example_user("hamlet")
user_profile = self.example_user("othello")
self.create_user_group_for_test("support")
self.login("hamlet")
msg_id = self.send_stream_message(
sender_user_profile, "Denmark", topic_name="editing", content="test"
)
def update_message_and_check_flag(content: str, mentioned: bool) -> None:
result = self.client_patch(
"/json/messages/" + str(msg_id),
{
"message_id": msg_id,
"content": content,
},
)
self.assert_json_success(result)
um = UserMessage.objects.get(
user_profile_id=user_profile.id,
message_id=msg_id,
)
if mentioned:
self.assertIn("mentioned", um.flags_list())
else:
self.assertNotIn("mentioned", um.flags_list())
update_message_and_check_flag("@*support*", True)
update_message_and_check_flag("@*support-invalid* edited", False)
update_message_and_check_flag("@*support* edited", True)
update_message_and_check_flag("edited", False)
update_message_and_check_flag("@*support*", True)
def test_user_group_mention_invalid(self) -> None:
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "Hey @*Nonexistent group*"
self.assertEqual(render_markdown(msg, content), "Hey @Nonexistent group
")
self.assertEqual(msg.mentions_user_group_ids, set())
def test_stream_single(self) -> None:
denmark = get_stream("Denmark", get_realm("zulip"))
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**Denmark**"
self.assertEqual(
render_markdown(msg, content),
''.format(
d=denmark,
),
)
def test_invalid_stream_followed_by_valid_mention(self) -> None:
denmark = get_stream("Denmark", get_realm("zulip"))
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**Invalid** and #**Denmark**"
self.assertEqual(
render_markdown(msg, content),
'#Invalid and #{d.name}
'.format(
d=denmark,
),
)
def test_stream_multiple(self) -> None:
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
realm = get_realm("zulip")
denmark = get_stream("Denmark", realm)
scotland = get_stream("Scotland", realm)
content = "Look to #**Denmark** and #**Scotland**, there something"
self.assertEqual(
render_markdown(msg, content),
"Look to "
'#{denmark.name} and '
'#{scotland.name}, '
"there something
".format(denmark=denmark, scotland=scotland),
)
def test_stream_case_sensitivity(self) -> None:
realm = get_realm("zulip")
case_sens = Stream.objects.create(name="CaseSens", realm=realm)
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**CaseSens**"
self.assertEqual(
render_markdown(msg, content),
''.format(
s=case_sens,
),
)
def test_stream_case_sensitivity_nonmatching(self) -> None:
"""#StreamName requires the stream be spelled with the correct case
currently. If we change that in the future, we'll need to change this
test."""
realm = get_realm("zulip")
Stream.objects.create(name="CaseSens", realm=realm)
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**casesens**"
self.assertEqual(render_markdown(msg, content), "#casesens
")
def test_topic_single(self) -> None:
denmark = get_stream("Denmark", get_realm("zulip"))
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**Denmark>some topic**"
self.assertEqual(
render_markdown(msg, content),
''.format(
d=denmark,
),
)
def test_topic_atomic_string(self) -> None:
realm = get_realm("zulip")
# Create a linkifier.
sender_user_profile = self.example_user("othello")
url_format_string = r"https://trac.example.com/ticket/%(id)s"
realm_filter = RealmFilter(
realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
)
realm_filter.save()
self.assertEqual(
realm_filter.__str__(),
"[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
)
# Create a topic link that potentially interferes with the pattern.
denmark = get_stream("Denmark", realm)
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**Denmark>#1234**"
self.assertEqual(
render_markdown(msg, content),
''.format(
d=denmark,
),
)
def test_topic_multiple(self) -> None:
denmark = get_stream("Denmark", get_realm("zulip"))
scotland = get_stream("Scotland", get_realm("zulip"))
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "This has two links: #**Denmark>some topic** and #**Scotland>other topic**."
self.assertEqual(
render_markdown(msg, content),
"This has two links: "
''
"#{denmark.name} > some topic"
" and "
''
"#{scotland.name} > other topic"
".
".format(denmark=denmark, scotland=scotland),
)
def test_possible_stream_names(self) -> None:
content = """#**test here**
This mentions #**Denmark** too.
#**garçon** #**천국** @**Ignore Person**
"""
self.assertEqual(
possible_linked_stream_names(content),
{"test here", "Denmark", "garçon", "천국"},
)
def test_stream_unicode(self) -> None:
realm = get_realm("zulip")
uni = Stream.objects.create(name="привет", realm=realm)
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**привет**"
quoted_name = ".D0.BF.D1.80.D0.B8.D0.B2.D0.B5.D1.82"
href = f"/#narrow/stream/{uni.id}-{quoted_name}"
self.assertEqual(
render_markdown(msg, content),
''.format(
s=uni,
href=href,
),
)
def test_stream_atomic_string(self) -> None:
realm = get_realm("zulip")
# Create a linkifier.
sender_user_profile = self.example_user("othello")
url_format_string = r"https://trac.example.com/ticket/%(id)s"
realm_filter = RealmFilter(
realm=realm, pattern=r"#(?P[0-9]{2,8})", url_format_string=url_format_string
)
realm_filter.save()
self.assertEqual(
realm_filter.__str__(),
"[0-9]{2,8}) https://trac.example.com/ticket/%(id)s>",
)
# Create a stream that potentially interferes with the pattern.
stream = Stream.objects.create(name="Stream #1234", realm=realm)
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "#**Stream #1234**"
href = f"/#narrow/stream/{stream.id}-Stream-.231234"
self.assertEqual(
render_markdown(msg, content),
''.format(
s=stream,
href=href,
),
)
def test_stream_invalid(self) -> None:
sender_user_profile = self.example_user("othello")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "There #**Nonexistentstream**"
self.assertEqual(
render_markdown(msg, content), "There #Nonexistentstream
"
)
self.assertEqual(msg.mentions_user_ids, set())
def test_image_preview_title(self) -> None:
msg = "[My favorite image](https://example.com/testimage.png)"
converted = markdown_convert_wrapper(msg)
self.assertEqual(
converted,
""
'My favorite image'
"
\n"
''
''
'
'
""
"",
)
def test_mit_rendering(self) -> None:
"""Test the Markdown configs for the MIT Zephyr mirroring system;
verifies almost all inline patterns are disabled, but
inline_interesting_links is still enabled"""
msg = "**test**"
realm = get_realm("zephyr")
client = get_client("zephyr_mirror")
message = Message(sending_client=client, sender=self.mit_user("sipbtest"))
converted = markdown_convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
"**test**
",
)
msg = "* test"
converted = markdown_convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
"* test
",
)
msg = "https://lists.debian.org/debian-ctte/2014/02/msg00173.html"
converted = markdown_convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
'https://lists.debian.org/debian-ctte/2014/02/msg00173.html
',
)
def test_url_to_a(self) -> None:
url = "javascript://example.com/invalidURL"
converted = url_to_a(db_data=None, url=url, text=url)
self.assertEqual(
converted,
"javascript://example.com/invalidURL",
)
def test_disabled_code_block_processor(self) -> None:
msg = (
"Hello,\n\n"
+ " I am writing this message to test something. I am writing this message to test something."
)
converted = markdown_convert_wrapper(msg)
expected_output = (
"Hello,
\n"
+ 'I am writing this message to test something. I am writing this message to test something.\n'
+ "
"
)
self.assertEqual(converted, expected_output)
realm = do_create_realm(
string_id="code_block_processor_test", name="code_block_processor_test"
)
maybe_update_markdown_engines(realm.id, True)
converted = markdown_convert(msg, message_realm=realm, email_gateway=True)
expected_output = (
"Hello,
\n"
+ "I am writing this message to test something. I am writing this message to test something.
"
)
self.assertEqual(converted, expected_output)
def test_normal_link(self) -> None:
realm = get_realm("zulip")
sender_user_profile = self.example_user("othello")
message = Message(sender=sender_user_profile, sending_client=get_client("test"))
msg = "http://example.com/#settings/"
self.assertEqual(
markdown_convert(msg, message_realm=realm, message=message),
'',
)
def test_relative_link(self) -> None:
realm = get_realm("zulip")
sender_user_profile = self.example_user("othello")
message = Message(sender=sender_user_profile, sending_client=get_client("test"))
msg = "http://zulip.testserver/#narrow/stream/999-hello"
self.assertEqual(
markdown_convert(msg, message_realm=realm, message=message),
'http://zulip.testserver/#narrow/stream/999-hello
',
)
def test_relative_link_streams_page(self) -> None:
realm = get_realm("zulip")
sender_user_profile = self.example_user("othello")
message = Message(sender=sender_user_profile, sending_client=get_client("test"))
msg = "http://zulip.testserver/#streams/all"
self.assertEqual(
markdown_convert(msg, message_realm=realm, message=message),
'http://zulip.testserver/#streams/all
',
)
def test_md_relative_link(self) -> None:
realm = get_realm("zulip")
sender_user_profile = self.example_user("othello")
message = Message(sender=sender_user_profile, sending_client=get_client("test"))
msg = "[hello](http://zulip.testserver/#narrow/stream/999-hello)"
self.assertEqual(
markdown_convert(msg, message_realm=realm, message=message),
'',
)
def test_html_entity_conversion(self) -> None:
msg = """\
Test raw: Hello, ©
Test inline code: `©`
Test fenced code:
```
©
©
```
Test quote:
~~~quote
©
~~~
Test a list:
* ©
* `©`
* ```©```
Test an indented block:
©"""
expected_output = """\
Test raw: Hello, ©
Test inline code: ©
Test fenced code:
©
©
Test quote:
©
Test a list:
- ©
©©
Test an indented block:
©
"""
converted = markdown_convert_wrapper(dedent(msg))
self.assertEqual(converted, dedent(expected_output))
class MarkdownApiTests(ZulipTestCase):
def test_render_message_api(self) -> None:
content = "That is a **bold** statement"
result = self.api_post(
self.example_user("othello"),
"/api/v1/messages/render",
dict(content=content),
)
self.assert_json_success(result)
self.assertEqual(
result.json()["rendered"], "That is a bold statement
"
)
def test_render_mention_stream_api(self) -> None:
"""Determines whether we're correctly passing the realm context"""
content = "This mentions #**Denmark** and @**King Hamlet**."
result = self.api_post(
self.example_user("othello"),
"/api/v1/messages/render",
dict(content=content),
)
self.assert_json_success(result)
user_id = self.example_user("hamlet").id
stream_id = get_stream("Denmark", get_realm("zulip")).id
self.assertEqual(
result.json()["rendered"],
f'This mentions #Denmark and @King Hamlet.
',
)
class MarkdownErrorTests(ZulipTestCase):
def test_markdown_error_handling(self) -> None:
with self.simulated_markdown_failure():
with self.assertRaises(MarkdownRenderingException):
markdown_convert_wrapper("")
def test_send_message_errors(self) -> None:
message = "whatever"
with self.simulated_markdown_failure():
# We don't use assertRaisesRegex because it seems to not
# handle i18n properly here on some systems.
with self.assertRaises(JsonableError):
self.send_stream_message(self.example_user("othello"), "Denmark", message)
def test_ultra_long_rendering(self) -> None:
"""A rendered message with an ultra-long length (> 10 * MAX_MESSAGE_LENGTH)
throws an exception"""
msg = "mock rendered message\n" * MAX_MESSAGE_LENGTH
with mock.patch("zerver.lib.markdown.timeout", return_value=msg), mock.patch(
"zerver.lib.markdown.markdown_logger"
):
with self.assertRaises(MarkdownRenderingException):
markdown_convert_wrapper(msg)
def test_curl_code_block_validation(self) -> None:
processor = SimulatedFencedBlockPreprocessor(Markdown())
processor.run_content_validators = True
markdown_input = [
"``` curl",
"curl {{ api_url }}/v1/register",
" -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
' -d "queue_id=1375801870:2942"',
"```",
]
with self.assertRaises(MarkdownRenderingException):
processor.run(markdown_input)
def test_curl_code_block_without_validation(self) -> None:
processor = SimulatedFencedBlockPreprocessor(Markdown())
markdown_input = [
"``` curl",
"curl {{ api_url }}/v1/register",
" -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
' -d "queue_id=1375801870:2942"',
"```",
]
expected = [
"",
"**curl:curl {{ api_url }}/v1/register",
" -u BOT_EMAIL_ADDRESS:BOT_API_KEY",
' -d "queue_id=1375801870:2942"**',
"",
"",
]
result = processor.run(markdown_input)
self.assertEqual(result, expected)
'
"