web-public message
") self.assertEqual(msg["flags"], ["read"]) self.assertEqual(msg["sender_email"], sender.email) self.assertEqual(msg["avatar_url"], avatar_url(sender)) def test_unauthenticated_narrow_to_web_public_channels(self) -> None: self.setup_web_public_test() post_params: Dict[str, Union[int, str, bool]] = { "anchor": 1, "num_before": 1, "num_after": 1, "narrow": orjson.dumps( [ dict(operator="channels", operand="web-public"), dict(operator="channel", operand="web-public-channel"), ] ).decode(), } result = self.client_get("/json/messages", dict(post_params)) self.verify_web_public_query_result_success(result, 1) def test_get_messages_with_web_public(self) -> None: """ An unauthenticated call to GET /json/messages with valid parameters including channels:web-public narrow returns list of messages in the web-public channels. """ self.setup_web_public_test(num_web_public_message=8) post_params = { "anchor": "first_unread", "num_before": 5, "num_after": 1, "narrow": orjson.dumps([dict(operator="channels", operand="web-public")]).decode(), } result = self.client_get("/json/messages", dict(post_params)) # Of the last 7 (num_before + num_after + 1) messages, only 5 # messages are returned, which were all web-public messages. # The other two messages should not be returned even though # they are the most recent. self.verify_web_public_query_result_success(result, 5) def test_client_avatar(self) -> None: """ The client_gravatar flag determines whether we send avatar_url. """ hamlet = self.example_user("hamlet") self.login_user(hamlet) do_change_user_setting( hamlet, "email_address_visibility", UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE, acting_user=None, ) self.send_personal_message(hamlet, self.example_user("iago")) result = self.get_and_check_messages( dict(anchor="newest", client_gravatar=orjson.dumps(False).decode()) ) message = result["messages"][0] self.assertIn("gravatar.com", message["avatar_url"]) result = self.get_and_check_messages( dict(anchor="newest", client_gravatar=orjson.dumps(True).decode()) ) message = result["messages"][0] self.assertEqual(message["avatar_url"], None) # Now verify client_gravatar doesn't run with EMAIL_ADDRESS_VISIBILITY_ADMINS do_change_user_setting( hamlet, "email_address_visibility", UserProfile.EMAIL_ADDRESS_VISIBILITY_ADMINS, acting_user=None, ) result = self.get_and_check_messages( dict(anchor="newest", client_gravatar=orjson.dumps(True).decode()) ) message = result["messages"][0] self.assertIn("gravatar.com", message["avatar_url"]) def test_get_messages_with_narrow_dm(self) -> None: """ A request for old messages with a narrow by direct message only returns conversations with that user. """ me = self.example_user("hamlet") def dr_emails(dr: List[UserDisplayRecipient]) -> str: assert isinstance(dr, list) return ",".join(sorted({*(r["email"] for r in dr), me.email})) def dr_ids(dr: List[UserDisplayRecipient]) -> List[int]: assert isinstance(dr, list) return sorted({*(r["id"] for r in dr), self.example_user("hamlet").id}) self.send_personal_message(me, self.example_user("iago")) self.send_group_direct_message( me, [self.example_user("iago"), self.example_user("cordelia")], ) # Send a 1:1 and group direct message containing Aaron. # Then deactivate Aaron to test "dm" narrow includes messages # from deactivated users also. self.send_personal_message(me, self.example_user("aaron")) self.send_group_direct_message( me, [self.example_user("iago"), self.example_user("aaron")], ) aaron = self.example_user("aaron") do_deactivate_user(aaron, acting_user=None) self.assertFalse(aaron.is_active) personals = [ m for m in get_user_messages(self.example_user("hamlet")) if not m.is_stream_message() ] for personal in personals: emails = dr_emails(get_display_recipient(personal.recipient)) self.login_user(me) narrow: List[Dict[str, Any]] = [dict(operator="dm", operand=emails)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(dr_emails(message["display_recipient"]), emails) # check passing user IDs is consistent with passing user emails as operand ids = dr_ids(get_display_recipient(personal.recipient)) narrow = [dict(operator="dm", operand=ids)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(dr_emails(message["display_recipient"]), emails) def test_get_messages_with_nonexistent_group_dm(self) -> None: me = self.example_user("hamlet") # Direct message group which doesn't match anything gets no results non_existent_direct_message_group = [ me.id, self.example_user("iago").id, self.example_user("othello").id, ] self.login_user(me) narrow: List[Dict[str, Any]] = [ dict(operator="dm", operand=non_existent_direct_message_group) ] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) self.assertEqual(result["messages"], []) narrow = [dict(operator="dm", operand=non_existent_direct_message_group, negated=True)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) self.assertEqual([m["id"] for m in result["messages"]], [1, 3]) def test_get_visible_messages_with_narrow_dm(self) -> None: me = self.example_user("hamlet") self.login_user(me) self.subscribe(self.example_user("hamlet"), "Scotland") message_ids = [self.send_personal_message(me, self.example_user("iago")) for i in range(5)] narrow = [dict(operator="dm", operand=self.example_user("iago").email)] self.message_visibility_test(narrow, message_ids, 2) def test_get_messages_with_narrow_dm_including(self) -> None: """ A request for old messages with a narrow by "dm-including" only returns direct messages (both group and 1:1) with that user. """ me = self.example_user("hamlet") iago = self.example_user("iago") cordelia = self.example_user("cordelia") othello = self.example_user("othello") matching_message_ids = [ # group direct message, sent by current user self.send_group_direct_message( me, [iago, cordelia, othello], ), # group direct message, sent by searched user self.send_group_direct_message( cordelia, [me, othello], ), # group direct message, sent by another user self.send_group_direct_message( othello, [me, cordelia], ), # direct 1:1 message, sent by current user to searched user self.send_personal_message(me, cordelia), # direct 1:1 message, sent by searched user to current user self.send_personal_message(cordelia, me), ] non_matching_message_ids = [ # direct 1:1 message, does not include current user self.send_personal_message(iago, cordelia), # direct 1:1 message, does not include searched user self.send_personal_message(iago, me), # direct 1:1 message, current user to self self.send_personal_message(me, me), # group direct message, sent by current user self.send_group_direct_message( me, [iago, othello], ), # group direct message, sent by searched user self.send_group_direct_message( cordelia, [iago, othello], ), ] self.login_user(me) test_operands = [cordelia.email, cordelia.id] for operand in test_operands: narrow = [dict(operator="dm-including", operand=operand)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertIn(message["id"], matching_message_ids) self.assertNotIn(message["id"], non_matching_message_ids) def test_get_visible_messages_with_narrow_dm_including(self) -> None: me = self.example_user("hamlet") self.login_user(me) iago = self.example_user("iago") cordelia = self.example_user("cordelia") othello = self.example_user("othello") message_ids = [ self.send_group_direct_message( me, [iago, cordelia, othello], ), self.send_personal_message(me, cordelia), self.send_group_direct_message( cordelia, [me, othello], ), self.send_personal_message(cordelia, me), self.send_group_direct_message( iago, [cordelia, me], ), ] narrow = [dict(operator="dm-including", operand=cordelia.email)] self.message_visibility_test(narrow, message_ids, 2) def test_get_messages_with_narrow_group_pm_with(self) -> None: """ A request for old messages with a narrow by deprecated "group-pm-with" only returns direct message group conversations with that user. """ me = self.example_user("hamlet") iago = self.example_user("iago") cordelia = self.example_user("cordelia") othello = self.example_user("othello") matching_message_ids = [ self.send_group_direct_message( me, [iago, cordelia, othello], ), self.send_group_direct_message( me, [cordelia, othello], ), ] non_matching_message_ids = [ self.send_personal_message(me, cordelia), self.send_group_direct_message( me, [iago, othello], ), self.send_group_direct_message( self.example_user("cordelia"), [iago, othello], ), ] self.login_user(me) test_operands = [cordelia.email, cordelia.id] for operand in test_operands: narrow = [dict(operator="group-pm-with", operand=operand)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertIn(message["id"], matching_message_ids) self.assertNotIn(message["id"], non_matching_message_ids) def test_get_visible_messages_with_narrow_group_pm_with(self) -> None: me = self.example_user("hamlet") self.login_user(me) iago = self.example_user("iago") cordelia = self.example_user("cordelia") othello = self.example_user("othello") message_ids = [ self.send_group_direct_message( me, [iago, cordelia, othello], ), self.send_group_direct_message( me, [cordelia, othello], ), self.send_group_direct_message( me, [cordelia, iago], ), ] narrow = [dict(operator="group-pm-with", operand=cordelia.email)] self.message_visibility_test(narrow, message_ids, 1) def test_include_history(self) -> None: hamlet = self.example_user("hamlet") cordelia = self.example_user("cordelia") channel_name = "test channel" self.subscribe(cordelia, channel_name) old_message_id = self.send_stream_message(cordelia, channel_name, content="foo") self.subscribe(hamlet, channel_name) content = "hello @**King Hamlet**" new_message_id = self.send_stream_message(cordelia, channel_name, content=content) self.login_user(hamlet) narrow = [ dict(operator="channel", operand=channel_name), ] req = dict( narrow=orjson.dumps(narrow).decode(), anchor=LARGER_THAN_MAX_MESSAGE_ID, num_before=100, num_after=100, ) payload = self.client_get("/json/messages", req) self.assert_json_success(payload) result = orjson.loads(payload.content) messages = result["messages"] self.assert_length(messages, 2) for message in messages: if message["id"] == old_message_id: old_message = message elif message["id"] == new_message_id: new_message = message self.assertEqual(old_message["flags"], ["read", "historical"]) self.assertEqual(new_message["flags"], ["mentioned"]) def test_get_messages_with_narrow_channel(self) -> None: hamlet = self.example_user("hamlet") self.login_user(hamlet) realm = hamlet.realm num_messages_per_channel = 5 channel_names = ["Scotland", "Verona", "Venice"] def send_messages_to_all_channels() -> None: Message.objects.filter(realm_id=realm.id, recipient__type=Recipient.STREAM).delete() for channel_name in channel_names: self.subscribe(hamlet, channel_name) for i in range(num_messages_per_channel): message_id = self.send_stream_message(hamlet, channel_name, content=f"test {i}") message = Message.objects.get(id=message_id) self.assert_message_stream_name(message, channel_name) send_messages_to_all_channels() self.send_personal_message(hamlet, hamlet) messages = get_user_messages(hamlet) channel_messages = [msg for msg in messages if msg.is_stream_message()] self.assertGreater(len(messages), len(channel_messages)) self.assert_length(channel_messages, num_messages_per_channel * len(channel_names)) for channel_name in channel_names: channel = get_stream(channel_name, realm) for operand in [channel.name, channel.id]: narrow = [dict(operator="channel", operand=operand)] result = self.get_and_check_messages( dict(narrow=orjson.dumps(narrow).decode(), num_after=100) ) fetched_messages: List[Dict[str, object]] = result["messages"] self.assert_length(fetched_messages, num_messages_per_channel) for message_dict in fetched_messages: self.assertEqual(message_dict["type"], "stream") self.assertEqual(message_dict["display_recipient"], channel_name) self.assertEqual(message_dict["recipient_id"], channel.recipient_id) def test_get_visible_messages_with_narrow_channel(self) -> None: self.login("hamlet") self.subscribe(self.example_user("hamlet"), "Scotland") message_ids = [ self.send_stream_message(self.example_user("iago"), "Scotland") for i in range(5) ] narrow = [dict(operator="channel", operand="Scotland")] self.message_visibility_test(narrow, message_ids, 2) def test_get_messages_with_narrow_channel_mit_unicode_regex(self) -> None: """ A request for old messages for a user in the mit.edu relam with Unicode channel name should be correctly escaped in the database query. """ user = self.mit_user("starnine") self.login_user(user) # We need to subscribe to a channel and then send a message to # it to ensure that we actually have a channel message in this # narrow view. lambda_channel_name = "\u03bb-channel" channel = self.subscribe(user, lambda_channel_name) self.assertTrue(channel.is_in_zephyr_realm) lambda_channel_d_name = "\u03bb-channel.d" self.subscribe(user, lambda_channel_d_name) self.send_stream_message(user, "\u03bb-channel") self.send_stream_message(user, "\u03bb-channel.d") narrow = [dict(operator="channel", operand="\u03bb-channel")] result = self.get_and_check_messages( dict(num_after=2, narrow=orjson.dumps(narrow).decode()), subdomain="zephyr" ) messages = get_user_messages(self.mit_user("starnine")) channel_messages = [msg for msg in messages if msg.is_stream_message()] self.assert_length(result["messages"], 2) for i, message in enumerate(result["messages"]): self.assertEqual(message["type"], "stream") channel_id = channel_messages[i].recipient.id self.assertEqual(message["recipient_id"], channel_id) def test_get_messages_with_narrow_topic_mit_unicode_regex(self) -> None: """ A request for old messages for a user in the mit.edu realm with Unicode topic name should be correctly escaped in the database query. """ mit_user_profile = self.mit_user("starnine") self.login_user(mit_user_profile) # We need to subscribe to a channel and then send a message to # it to ensure that we actually have a channel message in this # narrow view. self.subscribe(mit_user_profile, "Scotland") self.send_stream_message(mit_user_profile, "Scotland", topic_name="\u03bb-topic") self.send_stream_message(mit_user_profile, "Scotland", topic_name="\u03bb-topic.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name="\u03bb-topic.d.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name="\u03bb-topic.d.d.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name="\u03bb-topic.d.d.d.d") narrow = [dict(operator="topic", operand="\u03bb-topic")] result = self.get_and_check_messages( dict(num_after=100, narrow=orjson.dumps(narrow).decode()), subdomain="zephyr" ) messages = get_user_messages(mit_user_profile) channel_messages = [msg for msg in messages if msg.is_stream_message()] self.assert_length(result["messages"], 5) for i, message in enumerate(result["messages"]): self.assertEqual(message["type"], "stream") channel_id = channel_messages[i].recipient.id self.assertEqual(message["recipient_id"], channel_id) def test_get_messages_with_narrow_topic_mit_personal(self) -> None: """ We handle .d grouping for MIT realm personal messages correctly. """ mit_user_profile = self.mit_user("starnine") # We need to subscribe to a channel and then send a message to # it to ensure that we actually have a channel message in this # narrow view. self.login_user(mit_user_profile) self.subscribe(mit_user_profile, "Scotland") self.send_stream_message(mit_user_profile, "Scotland", topic_name=".d.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name="PERSONAL") self.send_stream_message(mit_user_profile, "Scotland", topic_name='(instance "").d') self.send_stream_message(mit_user_profile, "Scotland", topic_name=".d.d.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name="personal.d") self.send_stream_message(mit_user_profile, "Scotland", topic_name='(instance "")') self.send_stream_message(mit_user_profile, "Scotland", topic_name=".d.d.d.d") narrow = [dict(operator="topic", operand="personal.d.d")] result = self.get_and_check_messages( dict(num_before=50, num_after=50, narrow=orjson.dumps(narrow).decode()), subdomain="zephyr", ) messages = get_user_messages(mit_user_profile) channel_messages = [msg for msg in messages if msg.is_stream_message()] self.assert_length(result["messages"], 7) for i, message in enumerate(result["messages"]): self.assertEqual(message["type"], "stream") channel_id = channel_messages[i].recipient.id self.assertEqual(message["recipient_id"], channel_id) def test_get_messages_with_narrow_sender(self) -> None: """ A request for old messages with a narrow by sender only returns messages sent by that person. """ self.login("hamlet") hamlet = self.example_user("hamlet") othello = self.example_user("othello") iago = self.example_user("iago") # We need to send a message here to ensure that we actually # have a channel message in this narrow view. self.send_stream_message(hamlet, "Denmark") self.send_stream_message(othello, "Denmark") self.send_personal_message(othello, hamlet) self.send_stream_message(iago, "Denmark") test_operands = [othello.email, othello.id] for operand in test_operands: narrow = [dict(operator="sender", operand=operand)] result = self.get_and_check_messages(dict(narrow=orjson.dumps(narrow).decode())) for message in result["messages"]: self.assertEqual(message["sender_id"], othello.id) def _update_tsvector_index(self) -> None: # We use brute force here and update our text search index # for the entire zerver_message table (which is small in test # mode). In production there is an async process which keeps # the search index up to date. with connection.cursor() as cursor: cursor.execute( """ UPDATE zerver_message SET search_tsvector = to_tsvector('zulip.english_us_search', subject || rendered_content) """ ) @override_settings(USING_PGROONGA=False) def test_messages_in_narrow(self) -> None: user = self.example_user("cordelia") self.login_user(user) def send(content: str) -> int: msg_id = self.send_stream_message( user, "Verona", content=content, ) return msg_id good_id = send("KEYWORDMATCH and should work") bad_id = send("no match") msg_ids = [good_id, bad_id] send("KEYWORDMATCH but not in msg_ids") self._update_tsvector_index() narrow = [ dict(operator="search", operand="KEYWORDMATCH"), ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) params = {k: orjson.dumps(v).decode() for k, v in raw_params.items()} result = self.client_get("/json/messages/matches_narrow", params) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) message = messages[str(good_id)] self.assertEqual( message["match_content"], 'KEYWORDMATCH and should work
', ) narrow = [ dict(operator="search", operand="KEYWORDMATCH"), dict(operator="search", operand="work"), ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) params = {k: orjson.dumps(v).decode() for k, v in raw_params.items()} result = self.client_get("/json/messages/matches_narrow", params) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) message = messages[str(good_id)] self.assertEqual( message["match_content"], 'KEYWORDMATCH and should work
', ) @override_settings(USING_PGROONGA=False) def test_get_messages_with_search(self) -> None: self.login("cordelia") messages_to_search = [ ("breakfast", "there are muffins in the conference room"), ("lunch plans", "I am hungry!"), ("meetings", "discuss lunch after lunch"), ("meetings", "please bring your laptops to take notes"), ("dinner", "Anybody staying late tonight?"), ("urltest", "https://google.com"), ("日本", "こんに ちは 。 今日は いい 天気ですね。"), ("日本", "今朝はごはんを食べました。"), ("日本", "昨日、日本 のお菓子を送りました。"), ("english", "I want to go to 日本!"), ] next_message_id = self.get_last_message().id + 1 cordelia = self.example_user("cordelia") for topic, content in messages_to_search: self.send_stream_message( cordelia, "Verona", content=content, topic_name=topic, ) self._update_tsvector_index() narrow = [ dict(operator="sender", operand=cordelia.email), dict(operator="search", operand="lunch"), ] result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_before=0, num_after=10, ) ) self.assert_length(result["messages"], 2) messages = result["messages"] narrow = [dict(operator="search", operand="https://google.com")] link_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_before=0, num_after=10, ) ) self.assert_length(link_search_result["messages"], 1) self.assertEqual( link_search_result["messages"][0]["match_content"], '', ) (meeting_message,) = (m for m in messages if m[TOPIC_NAME] == "meetings") self.assertEqual(meeting_message[MATCH_TOPIC], "meetings") self.assertEqual( meeting_message["match_content"], ( 'discuss lunch after
I am hungry!
") # Should not crash when multiple search operands are present multi_search_narrow = [ dict(operator="search", operand="discuss"), dict(operator="search", operand="after"), ] multi_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(multi_search_result["messages"], 1) self.assertEqual( multi_search_result["messages"][0]["match_content"], 'discuss lunch after lunch
', ) # Test searching in messages with Unicode characters narrow = [ dict(operator="search", operand="日本"), ] result = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(result["messages"], 4) messages = result["messages"] japanese_message = [m for m in messages if m[TOPIC_NAME] == "日本"][-1] self.assertEqual(japanese_message[MATCH_TOPIC], '日本') self.assertEqual( japanese_message["match_content"], '昨日、日本 のお菓子を送りました。
', ) (english_message,) = (m for m in messages if m[TOPIC_NAME] == "english") self.assertEqual(english_message[MATCH_TOPIC], "english") self.assertIn( english_message["match_content"], 'I want to go to 日本!
', ) # Multiple search operands with Unicode multi_search_narrow = [ dict(operator="search", operand="ちは"), dict(operator="search", operand="今日は"), ] multi_search_result = self.get_and_check_messages( dict( narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(multi_search_result["messages"], 1) self.assertEqual( multi_search_result["messages"][0]["match_content"], 'こんに ちは 。 今日は いい 天気ですね。
', ) @override_settings(USING_PGROONGA=False) def test_get_visible_messages_with_search(self) -> None: self.login("hamlet") self.subscribe(self.example_user("hamlet"), "Scotland") messages_to_search = [ ("Gryffindor", "Hogwart's house which values courage, bravery, nerve, and chivalry"), ( "Hufflepuff", "Hogwart's house which values hard work, patience, justice, and loyalty.", ), ( "Ravenclaw", "Hogwart's house which values intelligence, creativity, learning, and wit", ), ( "Slytherin", "Hogwart's house which values ambition, cunning, leadership, and resourcefulness", ), ] message_ids = [ self.send_stream_message( self.example_user("iago"), "Scotland", topic_name=topic, content=content ) for topic, content in messages_to_search ] self._update_tsvector_index() narrow = [dict(operator="search", operand="Hogwart's")] self.message_visibility_test(narrow, message_ids, 2) @override_settings(USING_PGROONGA=False) def test_get_messages_with_search_not_subscribed(self) -> None: """Verify support for searching a channel you're not subscribed to""" self.subscribe(self.example_user("hamlet"), "new-channel") self.send_stream_message( self.example_user("hamlet"), "new-channel", content="Public special content!", topic_name="new", ) self._update_tsvector_index() self.login("cordelia") channel_search_narrow = [ dict(operator="search", operand="special"), dict(operator="channel", operand="new-channel"), ] channel_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(channel_search_narrow).decode(), anchor=0, num_after=10, num_before=10, ) ) self.assert_length(channel_search_result["messages"], 1) self.assertEqual( channel_search_result["messages"][0]["match_content"], 'Public special content!
', ) @override_settings(USING_PGROONGA=True) def test_get_messages_with_search_pgroonga(self) -> None: self.login("cordelia") next_message_id = self.get_last_message().id + 1 messages_to_search = [ ("日本語", "こんにちは。今日はいい天気ですね。"), ("日本語", "今朝はごはんを食べました。"), ("日本語", "昨日、日本のお菓子を送りました。"), ("english", "I want to go to 日本!"), ("english", "Can you speak https://en.wikipedia.org/wiki/Japanese?"), ("english", "https://domain.com/path/to.something-I,want/"), ("english", "foo.cht"), ("bread & butter", "chalk & cheese"), ] for topic, content in messages_to_search: self.send_stream_message( self.example_user("cordelia"), "Verona", content=content, topic_name=topic, ) # We use brute force here and update our text search index # for the entire zerver_message table (which is small in test # mode). In production there is an async process which keeps # the search index up to date. with connection.cursor() as cursor: cursor.execute( """ UPDATE zerver_message SET search_pgroonga = escape_html(subject) || ' ' || rendered_content """ ) narrow = [ dict(operator="search", operand="日本"), ] result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(result["messages"], 4) messages = result["messages"] japanese_message = [m for m in messages if m[TOPIC_NAME] == "日本語"][-1] self.assertEqual(japanese_message[MATCH_TOPIC], '日本語') self.assertEqual( japanese_message["match_content"], '昨日、日本のお菓子を送りました。
', ) [english_message] = (m for m in messages if m[TOPIC_NAME] == "english") self.assertEqual(english_message[MATCH_TOPIC], "english") self.assertEqual( english_message["match_content"], 'I want to go to 日本!
', ) # Should not crash when multiple search operands are present multi_search_narrow = [ dict(operator="search", operand="can"), dict(operator="search", operand="speak"), dict(operator="search", operand="wiki"), ] multi_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(multi_search_result["messages"], 1) self.assertEqual( multi_search_result["messages"][0]["match_content"], 'Can you speak https://en.wikipedia.org/wiki/Japanese?
', ) # Multiple search operands with Unicode multi_search_narrow = [ dict(operator="search", operand="朝は"), dict(operator="search", operand="べました"), ] multi_search_result = self.get_and_check_messages( dict( narrow=orjson.dumps(multi_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(multi_search_result["messages"], 1) self.assertEqual( multi_search_result["messages"][0]["match_content"], '今朝はごはんを食べました。
', ) def search(operand: str, link: Optional[str], highlight: str) -> None: narrow = [dict(operator="search", operand=operand)] link_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(link_search_result["messages"], 1) self.assertEqual( link_search_result["messages"][0]["match_content"], f'' if link else f"{highlight}
", ) search("foo.cht", None, 'foo.cht') search("foo", None, 'foo.cht') search("cht", None, 'foo.cht') url = "https://domain.com/path/to.something-I,want/" search(url, url, f'{url}') search( "https://domain", url, 'https://domain.com/path/to.something-I,want/', ) search( "domain", url, 'https://domain.com/path/to.something-I,want/', ) search( "domain.", url, 'https://domain.com/path/to.something-I,want/', ) search( "domain.com", url, 'https://domain.com/path/to.something-I,want/', ) search( "domain.com/", url, 'https://domain.com/path/to.something-I,want/', ) search( "domain.com/path", url, 'https://domain.com/path/to.something-I,want/', ) search( ".something", url, 'https://domain.com/path/to.something-I,want/', ) search( "to.something", url, 'https://domain.com/path/to.something-I,want/', ) search( "something-I", url, 'https://domain.com/path/to.something-I,want/', ) search( ",want", url, 'https://domain.com/path/to.something-I,want/', ) search( "I,want", url, 'https://domain.com/path/to.something-I,want/', ) # Search operands with HTML special characters special_search_narrow = [ dict(operator="search", operand="butter"), ] special_search_result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(special_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(special_search_result["messages"], 1) self.assertEqual( special_search_result["messages"][0][MATCH_TOPIC], 'bread & butter', ) special_search_narrow = [ dict(operator="search", operand="&"), ] special_search_result = self.get_and_check_messages( dict( narrow=orjson.dumps(special_search_narrow).decode(), anchor=next_message_id, num_after=10, num_before=0, ) ) self.assert_length(special_search_result["messages"], 1) self.assertEqual( special_search_result["messages"][0][MATCH_TOPIC], 'bread & butter', ) self.assertEqual( special_search_result["messages"][0]["match_content"], 'chalk & cheese
', ) def test_messages_in_narrow_for_non_search(self) -> None: user = self.example_user("cordelia") self.login_user(user) def send(content: str) -> int: msg_id = self.send_stream_message( user, "Verona", topic_name="test_topic", content=content, ) return msg_id good_id = send("http://foo.com") bad_id = send("no link here") msg_ids = [good_id, bad_id] send("http://bar.com but not in msg_ids") narrow = [ dict(operator="has", operand="link"), ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) params = {k: orjson.dumps(v).decode() for k, v in raw_params.items()} result = self.client_get("/json/messages/matches_narrow", params) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) message = messages[str(good_id)] self.assertIn("a href=", message["match_content"]) self.assertIn("http://foo.com", message["match_content"]) self.assertEqual(message[MATCH_TOPIC], "test_topic") def test_get_messages_with_only_searching_anchor(self) -> None: """ Test that specifying an anchor but 0 for num_before and num_after returns at most 1 message. """ self.login("cordelia") cordelia = self.example_user("cordelia") anchor = self.send_stream_message(cordelia, "Verona") narrow = [dict(operator="sender", operand=cordelia.email)] result: Dict[str, Any] = self.get_and_check_messages( dict( narrow=orjson.dumps(narrow).decode(), anchor=anchor, num_before=0, num_after=0, ) ) self.assert_length(result["messages"], 1) narrow = [dict(operator="is", operand="mentioned")] result = self.get_and_check_messages( dict(narrow=orjson.dumps(narrow).decode(), anchor=anchor, num_before=0, num_after=0) ) self.assert_length(result["messages"], 0) def test_get_messages_for_resolved_topics(self) -> None: self.login("cordelia") cordelia = self.example_user("cordelia") self.send_stream_message(cordelia, "Verona", "whatever1") resolved_topic_name = RESOLVED_TOPIC_PREFIX + "foo" anchor = self.send_stream_message(cordelia, "Verona", "whatever2", resolved_topic_name) self.send_stream_message(cordelia, "Verona", "whatever3") narrow = [dict(operator="is", operand="resolved")] result = self.get_and_check_messages( dict(narrow=orjson.dumps(narrow).decode(), anchor=anchor, num_before=0, num_after=0) ) self.assert_length(result["messages"], 1) self.assertEqual(result["messages"][0]["id"], anchor) def test_get_visible_messages_with_anchor(self) -> None: def messages_matches_ids(messages: List[Dict[str, Any]], message_ids: List[int]) -> None: self.assert_length(messages, len(message_ids)) for message in messages: assert message["id"] in message_ids self.login("hamlet") Message.objects.all().delete() message_ids = [ self.send_stream_message(self.example_user("cordelia"), "Verona") for i in range(10) ] data = self.get_messages_response(anchor=message_ids[9], num_before=9, num_after=0) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[9], num_before=9, num_after=0) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], True) messages_matches_ids(messages, message_ids[5:]) with first_visible_id_as(message_ids[2]): data = self.get_messages_response(anchor=message_ids[6], num_before=9, num_after=0) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], True) messages_matches_ids(messages, message_ids[2:7]) with first_visible_id_as(message_ids[9] + 1): data = self.get_messages_response(anchor=message_ids[9], num_before=9, num_after=0) messages = data["messages"] self.assert_length(messages, 0) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], True) data = self.get_messages_response(anchor=message_ids[5], num_before=0, num_after=5) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[5:]) with first_visible_id_as(message_ids[7]): data = self.get_messages_response(anchor=message_ids[5], num_before=0, num_after=5) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[7:]) with first_visible_id_as(message_ids[2]): data = self.get_messages_response(anchor=message_ids[0], num_before=0, num_after=5) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[2:7]) with first_visible_id_as(message_ids[9] + 1): data = self.get_messages_response(anchor=message_ids[0], num_before=0, num_after=5) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) self.assert_length(messages, 0) # Verify that with anchor=0 we always get found_oldest=True with first_visible_id_as(0): data = self.get_messages_response(anchor=0, num_before=0, num_after=5) messages = data["messages"] messages_matches_ids(messages, message_ids[0:5]) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) # Verify that with anchor=-1 we always get found_oldest=True # anchor=-1 is arguably invalid input, but it used to be supported with first_visible_id_as(0): data = self.get_messages_response(anchor=-1, num_before=0, num_after=5) messages = data["messages"] messages_matches_ids(messages, message_ids[0:5]) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) # And anchor='first' does the same thing. with first_visible_id_as(0): data = self.get_messages_response(anchor="oldest", num_before=0, num_after=5) messages = data["messages"] messages_matches_ids(messages, message_ids[0:5]) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) data = self.get_messages_response(anchor=message_ids[5], num_before=5, num_after=4) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids) data = self.get_messages_response(anchor=message_ids[5], num_before=10, num_after=10) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[5], num_before=5, num_after=4) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], True) messages_matches_ids(messages, message_ids[5:]) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[2], num_before=5, num_after=3) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], True) messages_matches_ids(messages, message_ids[5:8]) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[2], num_before=10, num_after=10) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], True) messages_matches_ids(messages, message_ids[5:]) with first_visible_id_as(message_ids[9] + 1): data = self.get_messages_response(anchor=message_ids[5], num_before=5, num_after=4) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], True) self.assert_length(messages, 0) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[5], num_before=0, num_after=0) messages = data["messages"] self.assertEqual(data["found_anchor"], True) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[5:6]) with first_visible_id_as(message_ids[5]): data = self.get_messages_response(anchor=message_ids[2], num_before=0, num_after=0) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) self.assert_length(messages, 0) # Verify some additional behavior of found_newest. with first_visible_id_as(0): data = self.get_messages_response( anchor=LARGER_THAN_MAX_MESSAGE_ID, num_before=5, num_after=0 ) messages = data["messages"] self.assert_length(messages, 5) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) # The anchor value of 'last' behaves just like LARGER_THAN_MAX_MESSAGE_ID. with first_visible_id_as(0): data = self.get_messages_response(anchor="newest", num_before=5, num_after=0) messages = data["messages"] self.assert_length(messages, 5) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) with first_visible_id_as(0): data = self.get_messages_response( anchor=LARGER_THAN_MAX_MESSAGE_ID + 1, num_before=5, num_after=0 ) messages = data["messages"] self.assert_length(messages, 5) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) with first_visible_id_as(0): data = self.get_messages_response( anchor=LARGER_THAN_MAX_MESSAGE_ID, num_before=20, num_after=0 ) messages = data["messages"] self.assert_length(messages, 10) self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], True) self.assertEqual(data["found_newest"], True) self.assertEqual(data["history_limited"], False) data = self.get_messages_response( anchor=message_ids[5], num_before=3, num_after=0, include_anchor=False ) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[2:5]) data = self.get_messages_response( anchor=message_ids[5], num_before=0, num_after=3, include_anchor=False ) messages = data["messages"] self.assertEqual(data["found_anchor"], False) self.assertEqual(data["found_oldest"], False) self.assertEqual(data["found_newest"], False) self.assertEqual(data["history_limited"], False) messages_matches_ids(messages, message_ids[6:9]) def test_missing_params(self) -> None: """ anchor, num_before, and num_after are all required POST parameters for get_messages. """ self.login("hamlet") required_args: Tuple[Tuple[str, int], ...] = (("num_before", 1), ("num_after", 1)) for i in range(len(required_args)): post_params = dict(required_args[:i] + required_args[i + 1 :]) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, f"Missing '{required_args[i][0]}' argument") def test_get_messages_limits(self) -> None: """ A call to GET /json/messages requesting more than MAX_MESSAGES_PER_FETCH messages returns an error message. """ self.login("hamlet") result = self.client_get("/json/messages", dict(anchor=1, num_before=3000, num_after=3000)) self.assert_json_error(result, "Too many messages requested (maximum 5000).") result = self.client_get("/json/messages", dict(anchor=1, num_before=6000, num_after=0)) self.assert_json_error(result, "Too many messages requested (maximum 5000).") result = self.client_get("/json/messages", dict(anchor=1, num_before=0, num_after=6000)) self.assert_json_error(result, "Too many messages requested (maximum 5000).") def test_bad_int_params(self) -> None: """ num_before, num_after, and narrow must all be non-negative integers or strings that can be converted to non-negative integers. """ self.login("hamlet") other_params = {"narrow": {}, "anchor": 0} int_params = ["num_before", "num_after"] invalid_parameters: List[InvalidParam] = [ InvalidParam(value=False, expected_error="is not valid JSON"), InvalidParam(value="", expected_error="is not valid JSON"), InvalidParam(value="-1", expected_error="is too small"), InvalidParam(value=-1, expected_error="is too small"), ] for idx, param in enumerate(int_params): for invalid_parameter in invalid_parameters: # Rotate through every bad type for every integer # parameter, one at a time. post_params = { **other_params, param: invalid_parameter.value, **dict.fromkeys(int_params[:idx] + int_params[idx + 1 :], 0), } result = self.client_get("/json/messages", post_params) self.assert_json_error(result, f"{param} {invalid_parameter.expected_error}") def test_bad_include_anchor(self) -> None: self.login("hamlet") result = self.client_get( "/json/messages", dict(anchor=1, num_before=1, num_after=1, include_anchor="false") ) self.assert_json_error(result, "The anchor can only be excluded at an end of the range") def test_bad_narrow_type(self) -> None: """ narrow must be a list of string pairs. """ self.login("hamlet") other_params = {"anchor": 0, "num_before": 0, "num_after": 0} invalid_parameters: List[InvalidParam] = [ InvalidParam(value=False, expected_error="narrow is not valid JSON"), InvalidParam(value=0, expected_error="narrow is not a list"), InvalidParam(value="", expected_error="narrow is not valid JSON"), InvalidParam(value="{malformed json,", expected_error="narrow is not valid JSON"), InvalidParam(value="{foo: 3}", expected_error="narrow is not valid JSON"), InvalidParam( value="[1,2]", expected_error="Invalid narrow[0]: Value error, dict or list required", ), InvalidParam( value='[["x","y","z"]]', expected_error="Invalid narrow[0]: Value error, element is not a string pair", ), ] for invalid_parameter in invalid_parameters: post_params = {**other_params, "narrow": invalid_parameter.value} result = self.client_get("/json/messages", post_params) self.assert_json_error(result, invalid_parameter.expected_error) def test_bad_narrow_operator(self) -> None: """ Unrecognized narrow operators are rejected. """ self.login("hamlet") for operator in ["", "foo", "channel:verona", "__init__"]: narrow = [dict(operator=operator, operand="")] params = dict(anchor=0, num_before=0, num_after=0, narrow=orjson.dumps(narrow).decode()) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, "Invalid narrow operator: unknown operator") def test_invalid_narrow_operand_in_dict(self) -> None: self.login("hamlet") # str or int is required for "id", "sender", "channel", "dm-including" and "group-pm-with" # operators invalid_operands: List[InvalidParam] = [ InvalidParam(value=["1"], expected_error="operand is not a string or integer"), InvalidParam(value=["2"], expected_error="operand is not a string or integer"), InvalidParam( value=None, expected_error="Invalid narrow[0]: Value error, operand is missing" ), ] for operand in ["id", "sender", "channel", "dm-including", "group-pm-with"]: self.exercise_bad_narrow_operand_using_dict_api(operand, invalid_operands) # str or int list is required for "dm" and "pm-with" operator # First set of invalid operands invalid_operands = [ InvalidParam( value=None, expected_error="Invalid narrow[0]: Value error, operand is missing" ) ] for operand in ["dm", "pm-with"]: self.exercise_bad_narrow_operand_using_dict_api(operand, invalid_operands) # Second set of invalid operands invalid_operands = [ InvalidParam(value=["2"], expected_error="operand[0] is not an integer"), ] for operand in ["dm", "pm-with"]: self.exercise_bad_narrow_operand_using_dict_api(operand, invalid_operands) # Third set of invalid operands invalid_operands = [ InvalidParam(value=2, expected_error="operand is not a string"), InvalidParam( value=None, expected_error="Invalid narrow[0]: Value error, operand is missing" ), InvalidParam(value=[1], expected_error="operand is not a string"), ] for operand in ["is", "near", "has"]: self.exercise_bad_narrow_operand_using_dict_api(operand, invalid_operands) # Disallow empty search terms invalid_operands = [InvalidParam(value="", expected_error="operand cannot be blank.")] self.exercise_bad_narrow_operand_using_dict_api("search", invalid_operands) # The exercise_bad_narrow_operand helper method uses legacy tuple format to # test bad narrow, this method uses the current dict API format def exercise_bad_narrow_operand_using_dict_api( self, operator: str, operands: Sequence[InvalidParam] ) -> None: for operand in operands: narrow = [dict(operator=operator, operand=operand.value)] params = dict(anchor=0, num_before=0, num_after=0, narrow=orjson.dumps(narrow).decode()) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, operand.expected_error) def exercise_bad_narrow_operand(self, operator: str, operands: Sequence[InvalidParam]) -> None: other_params = {"anchor": "0", "num_before": "0", "num_after": "0"} for operand in operands: post_params = { **other_params, "narrow": orjson.dumps([[operator, operand.value]]).decode(), } result = self.client_get("/json/messages", post_params) self.assert_json_error_contains(result, operand.expected_error) def test_bad_narrow_channel_content(self) -> None: """ If an invalid channel name is requested in get_messages, an error is returned. """ self.login("hamlet") error_msg = "Invalid narrow[0]: Value error, element is not a string pair" bad_channel_content: List[InvalidParam] = [ InvalidParam(value=0, expected_error=error_msg), InvalidParam(value=[], expected_error=error_msg), InvalidParam(value=["x", "y"], expected_error=error_msg), ] self.exercise_bad_narrow_operand("channel", bad_channel_content) def test_bad_narrow_one_on_one_email_content(self) -> None: """ If an invalid "dm" narrow is requested in get_messages, an error is returned. """ self.login("hamlet") error_msg = "Invalid narrow[0]: Value error, element is not a string pair" bad_channel_content: List[InvalidParam] = [ InvalidParam(value=0, expected_error=error_msg), InvalidParam(value=[], expected_error=error_msg), InvalidParam(value=["x", "y"], expected_error=error_msg), ] self.exercise_bad_narrow_operand("dm", bad_channel_content) def test_bad_narrow_nonexistent_channel(self) -> None: self.login("hamlet") non_existing_channel_id_operand: List[InvalidParam] = [ InvalidParam( value="non-existent channel", expected_error="Invalid narrow operator: unknown channel", ), ] self.exercise_bad_narrow_operand("channel", non_existing_channel_id_operand) non_existing_channel_id_operand = [ InvalidParam( value=1232891381239, expected_error="Invalid narrow operator: unknown channel" ), ] self.exercise_bad_narrow_operand_using_dict_api("channel", non_existing_channel_id_operand) def test_bad_narrow_nonexistent_email(self) -> None: self.login("hamlet") error_msg = "Invalid narrow operator: unknown user" invalid_operands: List[InvalidParam] = [ InvalidParam(value="non-existent-user@zulip.com", expected_error=error_msg), ] self.exercise_bad_narrow_operand("dm", invalid_operands) def test_bad_narrow_dm_id_list(self) -> None: self.login("hamlet") invalid_operands: List[InvalidParam] = [ InvalidParam( value=-24, expected_error="Invalid narrow[0]: Value error, element is not a string pair", ) ] self.exercise_bad_narrow_operand("dm", invalid_operands) def test_message_without_rendered_content(self) -> None: """Older messages may not have rendered_content in the database""" m = self.get_last_message() m.rendered_content = m.rendered_content_version = None m.content = "test content" wide_dict = MessageDict.wide_dict(m) final_dict = MessageDict.finalize_payload( wide_dict, apply_markdown=True, client_gravatar=False, ) self.assertEqual(final_dict["content"], "test content
") def common_check_get_messages_query(self, query_params: Dict[str, Any], expected: str) -> None: user_profile = self.example_user("hamlet") request = HostRequestMock(query_params, user_profile) with queries_captured() as queries: get_messages_backend( request, user_profile, num_before=query_params["num_before"], num_after=query_params["num_after"], ) for query in queries: sql = str(query.sql) if "/* get_messages */" in sql: sql = sql.replace(" /* get_messages */", "") self.assertEqual(sql, expected) return raise AssertionError("get_messages query not found") def test_find_first_unread_anchor(self) -> None: hamlet = self.example_user("hamlet") cordelia = self.example_user("cordelia") othello = self.example_user("othello") self.make_stream("England") self.subscribe(cordelia, "England") # Send a few messages that Hamlet won't have UserMessage rows for. unsub_message_id = self.send_stream_message(cordelia, "England") self.send_personal_message(cordelia, othello) self.subscribe(hamlet, "England") muted_topics = [ ["England", "muted"], ] set_topic_visibility_policy(hamlet, muted_topics, UserTopic.VisibilityPolicy.MUTED) # send a muted message muted_message_id = self.send_stream_message(cordelia, "England", topic_name="muted") # finally send Hamlet a "normal" message first_message_id = self.send_stream_message(cordelia, "England") # send a few more messages extra_message_id = self.send_stream_message(cordelia, "England") self.send_personal_message(cordelia, hamlet) user_profile = hamlet with get_sqlalchemy_connection() as sa_conn: anchor = find_first_unread_anchor( sa_conn=sa_conn, user_profile=user_profile, narrow=[], ) self.assertEqual(anchor, first_message_id) # With the same data setup, we now want to test that a reasonable # search still gets the first message sent to Hamlet (before he # subscribed) and other recent messages to the channel. query_params = dict( anchor="first_unread", num_before=10, num_after=10, narrow='[["channel", "England"]]', ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], first_message_id) self.assertEqual(result["found_newest"], True) self.assertEqual(result["found_oldest"], True) messages = result["messages"] self.assertEqual( {msg["id"] for msg in messages}, {unsub_message_id, muted_message_id, first_message_id, extra_message_id}, ) def test_parse_anchor_value(self) -> None: hamlet = self.example_user("hamlet") cordelia = self.example_user("cordelia") # Send the first message to Hamlet first_message_id = self.send_personal_message(cordelia, hamlet) # Send another message self.send_personal_message(cordelia, hamlet) user_profile = hamlet # Check if the anchor value in response is correct for different # values of anchor parameter in request # With anchor input as first_unread, see if response anchor # value is same as the id of first unread message of Hamlet query_params = dict( anchor="first_unread", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], first_message_id) # With anchor input as oldest, see if response anchor value is 0 query_params = dict( anchor="oldest", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], 0) # With anchor input as newest, see if response # anchor value is LARGER_THAN_MAX_MESSAGE_ID query_params = dict( anchor="newest", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], LARGER_THAN_MAX_MESSAGE_ID) # With anchor input negative, see if # response anchor value is clamped to 0 query_params = dict( anchor="-1", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], 0) # With anchor input more than LARGER_THAN_MAX_MESSAGE_ID, # see if response anchor value is clamped down to LARGER_THAN_MAX_MESSAGE_ID query_params = dict( anchor="10000000000000001", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) payload = get_messages_backend( request, user_profile, num_before=10, num_after=10, ) result = orjson.loads(payload.content) self.assertEqual(result["anchor"], LARGER_THAN_MAX_MESSAGE_ID) def test_use_first_unread_anchor_with_some_unread_messages(self) -> None: user_profile = self.example_user("hamlet") # Have Othello send messages to Hamlet that he hasn't read. # Here, Hamlet isn't subscribed to the channel Scotland self.send_stream_message(self.example_user("othello"), "Scotland") first_unread_message_id = self.send_personal_message( self.example_user("othello"), self.example_user("hamlet"), ) # Add a few messages that help us test that our query doesn't # look at messages that are irrelevant to Hamlet. self.send_personal_message(self.example_user("othello"), self.example_user("cordelia")) self.send_personal_message(self.example_user("othello"), self.example_user("iago")) query_params = dict( anchor="first_unread", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend( request, user_profile, num_before=10, num_after=10, ) # Verify the query for old messages looks correct. queries = [q for q in all_queries if "/* get_messages */" in q.sql] self.assert_length(queries, 1) sql = queries[0].sql self.assertNotIn(f"AND message_id = {LARGER_THAN_MAX_MESSAGE_ID}", sql) self.assertIn("ORDER BY message_id ASC", sql) cond = ( f"WHERE user_profile_id = {user_profile.id} AND message_id >= {first_unread_message_id}" ) self.assertIn(cond, sql) cond = f"WHERE user_profile_id = {user_profile.id} AND message_id <= {first_unread_message_id - 1}" self.assertIn(cond, sql) self.assertIn("UNION", sql) def test_visible_messages_use_first_unread_anchor_with_some_unread_messages(self) -> None: user_profile = self.example_user("hamlet") # Have Othello send messages to Hamlet that he hasn't read. self.subscribe(self.example_user("hamlet"), "Scotland") first_unread_message_id = self.send_stream_message(self.example_user("othello"), "Scotland") self.send_stream_message(self.example_user("othello"), "Scotland") self.send_stream_message(self.example_user("othello"), "Scotland") self.send_personal_message( self.example_user("othello"), self.example_user("hamlet"), ) # Add a few messages that help us test that our query doesn't # look at messages that are irrelevant to Hamlet. self.send_personal_message(self.example_user("othello"), self.example_user("cordelia")) self.send_personal_message(self.example_user("othello"), self.example_user("iago")) query_params = dict( anchor="first_unread", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) first_visible_message_id = first_unread_message_id + 2 with first_visible_id_as(first_visible_message_id): with queries_captured() as all_queries: get_messages_backend( request, user_profile, num_before=10, num_after=10, ) queries = [q for q in all_queries if "/* get_messages */" in q.sql] self.assert_length(queries, 1) sql = queries[0].sql self.assertNotIn(f"AND message_id = {LARGER_THAN_MAX_MESSAGE_ID}", sql) self.assertIn("ORDER BY message_id ASC", sql) cond = f"WHERE user_profile_id = {user_profile.id} AND message_id <= {first_unread_message_id - 1}" self.assertIn(cond, sql) cond = f"WHERE user_profile_id = {user_profile.id} AND message_id >= {first_visible_message_id}" self.assertIn(cond, sql) def test_use_first_unread_anchor_with_no_unread_messages(self) -> None: user_profile = self.example_user("hamlet") query_params = dict( anchor="first_unread", num_before=10, num_after=10, narrow="[]", ) request = HostRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend( request, user_profile, num_before=10, num_after=10, ) queries = [q for q in all_queries if "/* get_messages */" in q.sql] self.assert_length(queries, 1) sql = queries[0].sql self.assertNotIn("AND message_id <=", sql) self.assertNotIn("AND message_id >=", sql) request = HostRequestMock(query_params, user_profile) first_visible_message_id = 5 with first_visible_id_as(first_visible_message_id): with queries_captured() as all_queries: get_messages_backend( request, user_profile, num_before=10, num_after=10, ) queries = [q for q in all_queries if "/* get_messages */" in q.sql] sql = queries[0].sql self.assertNotIn("AND message_id <=", sql) self.assertNotIn("AND message_id >=", sql) def test_use_first_unread_anchor_with_muted_topics(self) -> None: """ Test that our logic related to `use_first_unread_anchor` invokes the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack for the `/* get_messages */` query when relevant muting is in effect. This is a very arcane test on arcane, but very heavily field-tested, logic in get_messages_backend(). If this test breaks, be absolutely sure you know what you're doing. """ realm = get_realm("zulip") self.make_stream("web stuff") self.make_stream("bogus") user_profile = self.example_user("hamlet") muted_topics = [ ["Scotland", "golf"], ["web stuff", "css"], ["bogus", "bogus"], ] set_topic_visibility_policy(user_profile, muted_topics, UserTopic.VisibilityPolicy.MUTED) query_params = dict( anchor="first_unread", num_before=0, num_after=0, narrow='[["channel", "Scotland"]]', ) request = HostRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend( request, user_profile, num_before=0, num_after=0, ) # Do some tests on the main query, to verify the muting logic # runs on this code path. queries = [q for q in all_queries if q.sql.startswith("SELECT message_id, flags")] self.assert_length(queries, 1) channel = get_stream("Scotland", realm) assert channel.recipient is not None recipient_id = channel.recipient.id cond = f"AND NOT (recipient_id = {recipient_id} AND upper(subject) = upper('golf'))" self.assertIn(cond, queries[0].sql) # Next, verify the use_first_unread_anchor setting invokes # the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack. queries = [q for q in all_queries if "/* get_messages */" in q.sql] self.assert_length(queries, 1) self.assertIn(f"AND zerver_message.id = {LARGER_THAN_MAX_MESSAGE_ID}", queries[0].sql) def test_exclude_muting_conditions(self) -> None: realm = get_realm("zulip") self.make_stream("web stuff") user_profile = self.example_user("hamlet") self.make_stream("irrelevant_channel") # Test the do-nothing case first. muted_topics = [ ["irrelevant_channel", "irrelevant_topic"], ] set_topic_visibility_policy(user_profile, muted_topics, UserTopic.VisibilityPolicy.MUTED) # If nothing relevant is muted, then exclude_muting_conditions() # should return an empty list. narrow: List[NarrowParameter] = [ NarrowParameter(operator="channel", operand="Scotland"), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) self.assertEqual(muting_conditions, []) # Also test that passing channel ID works channel_id = get_stream("Scotland", realm).id narrow = [ NarrowParameter(operator="channel", operand=channel_id), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) self.assertEqual(muting_conditions, []) # Ok, now set up our muted topics to include a topic relevant to our narrow. muted_topics = [ ["Scotland", "golf"], ["web stuff", "css"], ] set_topic_visibility_policy(user_profile, muted_topics, UserTopic.VisibilityPolicy.MUTED) # And verify that our query will exclude them. narrow = [ NarrowParameter(operator="channel", operand="Scotland"), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select(column("id", Integer).label("message_id")).select_from( table("zerver_message") ) query = query.where(*muting_conditions) expected_query = """\ SELECT id AS message_id \n\ FROM zerver_message \n\ WHERE NOT (recipient_id = %(recipient_id_1)s AND upper(subject) = upper(%(param_1)s))\ """ self.assertEqual(get_sqlalchemy_sql(query), expected_query) params = get_sqlalchemy_query_params(query) self.assertEqual( params["recipient_id_1"], get_recipient_id_for_channel_name(realm, "Scotland") ) self.assertEqual(params["param_1"], "golf") mute_channel(realm, user_profile, "Verona") # Using a bogus channel name should be similar to using no narrow at # all, and we'll exclude all mutes. narrow = [ NarrowParameter(operator="channel", operand="bogus-channel-name"), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select(column("id", Integer)).select_from(table("zerver_message")) query = query.where(and_(*muting_conditions)) expected_query = """\ SELECT id \n\ FROM zerver_message \n\ WHERE (recipient_id NOT IN (__[POSTCOMPILE_recipient_id_1])) \ AND NOT \ (recipient_id = %(recipient_id_2)s AND upper(subject) = upper(%(param_1)s) OR \ recipient_id = %(recipient_id_3)s AND upper(subject) = upper(%(param_2)s))\ """ self.assertEqual(get_sqlalchemy_sql(query), expected_query) params = get_sqlalchemy_query_params(query) self.assertEqual( params["recipient_id_1"], [get_recipient_id_for_channel_name(realm, "Verona")] ) self.assertEqual( params["recipient_id_2"], get_recipient_id_for_channel_name(realm, "Scotland") ) self.assertEqual(params["param_1"], "golf") self.assertEqual( params["recipient_id_3"], get_recipient_id_for_channel_name(realm, "web stuff") ) self.assertEqual(params["param_2"], "css") def test_get_messages_queries(self) -> None: query_ids = self.get_query_ids() sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id = 0) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({"anchor": 0, "num_before": 0, "num_after": 0}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id = 0) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({"anchor": 0, "num_before": 1, "num_after": 0}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} ORDER BY message_id ASC \n LIMIT 2) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({"anchor": 0, "num_before": 0, "num_after": 1}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} ORDER BY message_id ASC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({"anchor": 0, "num_before": 0, "num_after": 10}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 100 ORDER BY message_id DESC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({"anchor": 100, "num_before": 10, "num_after": 0}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM ((SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 99 ORDER BY message_id DESC \n LIMIT 10) UNION ALL (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id >= 100 ORDER BY message_id ASC \n LIMIT 11)) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( {"anchor": 100, "num_before": 10, "num_after": 10}, sql ) def test_get_messages_with_narrow_queries(self) -> None: query_ids = self.get_query_ids() hamlet_email = self.example_user("hamlet").email othello_email = self.example_user("othello").email sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2048) != 0 AND realm_id = {realm_id} AND (sender_id = {othello_id} AND recipient_id = {hamlet_recipient} OR sender_id = {hamlet_id} AND recipient_id = {othello_recipient}) AND message_id = 0) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 0, "narrow": f'[["dm", "{othello_email}"]]', }, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2048) != 0 AND realm_id = {realm_id} AND (sender_id = {othello_id} AND recipient_id = {hamlet_recipient} OR sender_id = {hamlet_id} AND recipient_id = {othello_recipient}) AND message_id = 0) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 1, "num_after": 0, "narrow": f'[["dm", "{othello_email}"]]', }, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2048) != 0 AND realm_id = {realm_id} AND (sender_id = {othello_id} AND recipient_id = {hamlet_recipient} OR sender_id = {hamlet_id} AND recipient_id = {othello_recipient}) ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": f'[["dm", "{othello_email}"]]', }, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2) != 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( {"anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["is", "starred"]]'}, sql ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND sender_id = {othello_id} ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": f'[["sender", "{othello_email}"]]', }, sql, ) sql_template = "SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE realm_id = 2 AND recipient_id = {scotland_recipient} ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( {"anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["channel", "Scotland"]]'}, sql, ) sql_template = "SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE realm_id = 2 AND recipient_id IN ({public_channels_recipients}) ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( {"anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["channels", "public"]]'}, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (recipient_id NOT IN ({public_channels_recipients})) ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[{"operator":"channels", "operand":"public", "negated": true}]', }, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND upper(subject) = upper('blah') ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( {"anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["topic", "blah"]]'}, sql ) sql_template = "SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE realm_id = 2 AND recipient_id = {scotland_recipient} AND upper(subject) = upper('blah') ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["channel", "Scotland"], ["topic", "blah"]]', }, sql, ) # Narrow to direct messages with yourself sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2048) != 0 AND realm_id = {realm_id} AND sender_id = {hamlet_id} AND recipient_id = {hamlet_recipient} ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": f'[["dm", "{hamlet_email}"]]', }, sql, ) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND recipient_id = {scotland_recipient} AND (flags & 2) != 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query( { "anchor": 0, "num_before": 0, "num_after": 9, "narrow": '[["channel", "Scotland"], ["is", "starred"]]', }, sql, ) @override_settings(USING_PGROONGA=False) def test_get_messages_with_search_queries(self) -> None: query_ids = self.get_query_ids() sql_template = """\ SELECT anon_1.message_id, anon_1.flags, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.topic_matches \n\ FROM (SELECT message_id, flags, subject, rendered_content, array((SELECT ARRAY[sum(length(anon_3) - 11) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) + 11, strpos(anon_3, '') - 1] AS anon_2 \n\ FROM unnest(string_to_array(ts_headline('zulip.english_us_search', rendered_content, plainto_tsquery('zulip.english_us_search', 'jumping'), 'HighlightAll = TRUE, StartSel =How are you doing, ' '@Othello, the Moor of Venice?
', ) class MessageHasKeywordsTest(ZulipTestCase): """Test for keywords like has_link, has_image, has_attachment.""" def setup_dummy_attachments(self, user_profile: UserProfile) -> List[str]: realm_id = user_profile.realm_id dummy_files = [ ("zulip.txt", f"{realm_id}/31/4CBjtTLYZhk66pZrF8hnYGwc/zulip.txt"), ("temp_file.py", f"{realm_id}/31/4CBjtTLYZhk66pZrF8hnYGwc/temp_file.py"), ("abc.py", f"{realm_id}/31/4CBjtTLYZhk66pZrF8hnYGwc/abc.py"), ] for file_name, path_id in dummy_files: create_attachment( file_name, path_id, "text/plain", b"1234567890", user_profile, user_profile.realm ) # return path ids return [x[1] for x in dummy_files] def test_claim_attachment(self) -> None: user_profile = self.example_user("hamlet") dummy_path_ids = self.setup_dummy_attachments(user_profile) dummy_urls = [f"http://zulip.testserver/user_uploads/{x}" for x in dummy_path_ids] # Send message referring the attachment self.subscribe(user_profile, "Denmark") def assert_attachment_claimed(path_id: str, claimed: bool) -> None: attachment = Attachment.objects.get(path_id=path_id) self.assertEqual(attachment.is_claimed(), claimed) # This message should claim attachments 1 only because attachment 2 # is not being parsed as a link by Markdown. body = ( f"Some files here ...[zulip.txt]({dummy_urls[0]})" f"{dummy_urls[1]}.... Some more...." f"{dummy_urls[1]}" ) self.send_stream_message(user_profile, "Denmark", body, "test") assert_attachment_claimed(dummy_path_ids[0], True) assert_attachment_claimed(dummy_path_ids[1], False) # This message tries to claim the third attachment but fails because # Markdown would not set has_attachments = True here. body = f"Link in code: `{dummy_urls[2]}`" self.send_stream_message(user_profile, "Denmark", body, "test") assert_attachment_claimed(dummy_path_ids[2], False) # Another scenario where we wouldn't parse the link. body = f"Link to not parse: .{dummy_urls[2]}.`" self.send_stream_message(user_profile, "Denmark", body, "test") assert_attachment_claimed(dummy_path_ids[2], False) # Finally, claim attachment 3. body = f"Link: {dummy_urls[2]}" self.send_stream_message(user_profile, "Denmark", body, "test") assert_attachment_claimed(dummy_path_ids[2], True) assert_attachment_claimed(dummy_path_ids[1], False) def test_finds_all_links(self) -> None: msg_contents = ["foo.org", "[bar](baz.gov)", "http://quux.ca"] msg_ids = [ self.send_stream_message(self.example_user("hamlet"), "Denmark", content=msg_content) for msg_content in msg_contents ] msgs = [Message.objects.get(id=id) for id in msg_ids] self.assertTrue(all(msg.has_link for msg in msgs)) def test_finds_only_links(self) -> None: msg_contents = ["`example.org`", "``example.org```", "$$https://example.org$$", "foo"] msg_ids = [ self.send_stream_message(self.example_user("hamlet"), "Denmark", content=msg_content) for msg_content in msg_contents ] msgs = [Message.objects.get(id=id) for id in msg_ids] self.assertFalse(all(msg.has_link for msg in msgs)) def update_message(self, msg: Message, content: str) -> None: hamlet = self.example_user("hamlet") realm_id = hamlet.realm.id rendering_result = render_message_markdown(msg, content) mention_backend = MentionBackend(realm_id) mention_data = MentionData(mention_backend, content, msg.sender) do_update_message( hamlet, msg, None, None, "change_one", False, False, content, rendering_result, set(), mention_data=mention_data, ) def test_finds_link_after_edit(self) -> None: hamlet = self.example_user("hamlet") msg_id = self.send_stream_message(hamlet, "Denmark", content="a") msg = Message.objects.get(id=msg_id) self.assertFalse(msg.has_link) self.update_message(msg, "a http://foo.com") self.assertTrue(msg.has_link) self.update_message(msg, "a") self.assertFalse(msg.has_link) # Check in blockquotes work self.update_message(msg, "> http://bar.com") self.assertTrue(msg.has_link) self.update_message(msg, "a `http://foo.com`") self.assertFalse(msg.has_link) def test_has_image(self) -> None: msg_contents = [ "Link: foo.org", "Image: https://www.google.com/images/srpr/logo4w.png", "Image: https://www.google.com/images/srpr/logo4w.pdf", "[Google link](https://www.google.com/images/srpr/logo4w.png)", ] msg_ids = [ self.send_stream_message(self.example_user("hamlet"), "Denmark", content=msg_content) for msg_content in msg_contents ] msgs = [Message.objects.get(id=id) for id in msg_ids] self.assertEqual([False, True, False, True], [msg.has_image for msg in msgs]) self.update_message(msgs[0], "https://www.google.com/images/srpr/logo4w.png") self.assertTrue(msgs[0].has_image) self.update_message(msgs[0], "No image again") self.assertFalse(msgs[0].has_image) def test_has_attachment(self) -> None: hamlet = self.example_user("hamlet") dummy_path_ids = self.setup_dummy_attachments(hamlet) dummy_urls = [f"http://zulip.testserver/user_uploads/{x}" for x in dummy_path_ids] self.subscribe(hamlet, "Denmark") body = f"Files ...[zulip.txt]({dummy_urls[0]}) {dummy_urls[1]} {dummy_urls[2]}" msg_id = self.send_stream_message(hamlet, "Denmark", body, "test") msg = Message.objects.get(id=msg_id) self.assertTrue(msg.has_attachment) self.update_message(msg, "No attachments") self.assertFalse(msg.has_attachment) self.update_message(msg, body) self.assertTrue(msg.has_attachment) self.update_message(msg, f"Link in code: `{dummy_urls[1]}`") self.assertFalse(msg.has_attachment) # Test blockquotes self.update_message(msg, f"> {dummy_urls[1]}") self.assertTrue(msg.has_attachment) # Additional test to check has_attachment is being set is due to the correct attachment. self.update_message(msg, f"Outside: {dummy_urls[0]}. In code: `{dummy_urls[1]}`.") self.assertTrue(msg.has_attachment) self.assertTrue(msg.attachment_set.filter(path_id=dummy_path_ids[0])) self.assertEqual(msg.attachment_set.count(), 1) self.update_message(msg, f"Outside: {dummy_urls[1]}. In code: `{dummy_urls[0]}`.") self.assertTrue(msg.has_attachment) self.assertTrue(msg.attachment_set.filter(path_id=dummy_path_ids[1])) self.assertEqual(msg.attachment_set.count(), 1) self.update_message(msg, f"Both in code: `{dummy_urls[1]} {dummy_urls[0]}`.") self.assertFalse(msg.has_attachment) self.assertEqual(msg.attachment_set.count(), 0) def test_potential_attachment_path_ids(self) -> None: hamlet = self.example_user("hamlet") self.subscribe(hamlet, "Denmark") dummy_path_ids = self.setup_dummy_attachments(hamlet) body = "Hello" msg_id = self.send_stream_message(hamlet, "Denmark", body, "test") msg = Message.objects.get(id=msg_id) with mock.patch( "zerver.actions.uploads.do_claim_attachments", wraps=do_claim_attachments ) as m: self.update_message( msg, f"[link](http://{hamlet.realm.host}/user_uploads/{dummy_path_ids[0]})" ) self.assertTrue(m.called) m.reset_mock() self.update_message(msg, f"[link](/user_uploads/{dummy_path_ids[1]})") self.assertTrue(m.called) m.reset_mock() self.update_message(msg, f"[new text link](/user_uploads/{dummy_path_ids[1]})") self.assertFalse(m.called) m.reset_mock() # It's not clear this is correct behavior self.update_message(msg, f"[link](user_uploads/{dummy_path_ids[2]})") self.assertFalse(m.called) m.reset_mock() self.update_message(msg, f"[link](https://github.com/user_uploads/{dummy_path_ids[0]})") self.assertFalse(m.called) m.reset_mock() def test_has_reaction(self) -> None: self.login("iago") has_reaction_narrow = orjson.dumps([dict(operator="has", operand="reaction")]).decode() msg_id = self.send_stream_message(self.example_user("hamlet"), "Denmark", content="Hey") result = self.client_get( "/json/messages", dict(narrow=has_reaction_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 0) check_add_reaction( self.example_user("hamlet"), msg_id, "hamburger", "1f354", "unicode_emoji" ) result = self.client_get( "/json/messages", dict(narrow=has_reaction_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) msg_id = self.send_personal_message( self.example_user("iago"), self.example_user("cordelia"), "Hello Cordelia" ) result = self.client_get( "/json/messages", dict(narrow=has_reaction_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 0) check_add_reaction(self.example_user("iago"), msg_id, "hamburger", "1f354", "unicode_emoji") result = self.client_get( "/json/messages", dict(narrow=has_reaction_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) class MessageIsTest(ZulipTestCase): def test_message_is_followed(self) -> None: self.login("iago") is_followed_narrow = orjson.dumps([dict(operator="is", operand="followed")]).decode() # Sending a message in a topic that isn't followed by the user. msg_id = self.send_stream_message(self.example_user("hamlet"), "Denmark", topic_name="hey") result = self.client_get( "/json/messages", dict(narrow=is_followed_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 0) stream_id = self.get_stream_id("Denmark", self.example_user("hamlet").realm) # Following the topic. payload = { "stream_id": stream_id, "topic": "hey", "visibility_policy": int(UserTopic.VisibilityPolicy.FOLLOWED), } self.client_post("/json/user_topics", payload) result = self.client_get( "/json/messages", dict(narrow=is_followed_narrow, anchor=msg_id, num_before=0, num_after=0), ) messages = self.assert_json_success(result)["messages"] self.assert_length(messages, 1) class MessageVisibilityTest(ZulipTestCase): def test_update_first_visible_message_id(self) -> None: Message.objects.all().delete() message_ids = [ self.send_stream_message(self.example_user("othello"), "Scotland") for i in range(15) ] # If message_visibility_limit is None update_first_visible_message_id # should set first_visible_message_id to 0 realm = get_realm("zulip") realm.message_visibility_limit = None # Setting to a random value other than 0 as the default value of # first_visible_message_id is 0 realm.first_visible_message_id = 5 realm.save() update_first_visible_message_id(realm) self.assertEqual(get_first_visible_message_id(realm), 0) realm.message_visibility_limit = 10 realm.save() expected_message_id = message_ids[5] update_first_visible_message_id(realm) self.assertEqual(get_first_visible_message_id(realm), expected_message_id) # If the message_visibility_limit is greater than number of messages # get_first_visible_message_id should return 0 realm.message_visibility_limit = 50 realm.save() update_first_visible_message_id(realm) self.assertEqual(get_first_visible_message_id(realm), 0) def test_maybe_update_first_visible_message_id(self) -> None: realm = get_realm("zulip") lookback_hours = 30 realm.message_visibility_limit = None realm.save() end_time = timezone_now() - timedelta(hours=lookback_hours - 5) stat = COUNT_STATS["messages_sent:is_bot:hour"] RealmCount.objects.create(realm=realm, property=stat.property, end_time=end_time, value=5) with mock.patch("zerver.lib.message.update_first_visible_message_id") as m: maybe_update_first_visible_message_id(realm, lookback_hours) m.assert_not_called() realm.message_visibility_limit = 10 realm.save() RealmCount.objects.all().delete() with mock.patch("zerver.lib.message.update_first_visible_message_id") as m: maybe_update_first_visible_message_id(realm, lookback_hours) m.assert_not_called() RealmCount.objects.create(realm=realm, property=stat.property, end_time=end_time, value=5) with mock.patch("zerver.lib.message.update_first_visible_message_id") as m: maybe_update_first_visible_message_id(realm, lookback_hours) m.assert_called_once_with(realm) class PersonalMessagesNearTest(ZulipTestCase): def test_near_pm_message_url(self) -> None: realm = get_realm("zulip") message = dict( type="personal", id=555, display_recipient=[ dict(id=77), dict(id=80), ], ) url = near_message_url( realm=realm, message=message, ) self.assertEqual(url, "http://zulip.testserver/#narrow/dm/77,80-pm/near/555")