KEYWORDMATCH and should work
') @override_settings(USING_PGROONGA=False) def test_get_messages_with_search(self): # type: () -> None self.login(self.example_email("cordelia")) messages_to_search = [ ('breakfast', 'there are muffins in the conference room'), ('lunch plans', 'I am hungry!'), ('meetings', 'discuss lunch after lunch'), ('meetings', 'please bring your laptops to take notes'), ('dinner', 'Anybody staying late tonight?'), ('urltest', 'https://google.com'), ] next_message_id = self.get_last_message().id + 1 for topic, content in messages_to_search: self.send_message( sender_name=self.example_email("cordelia"), raw_recipients="Verona", message_type=Recipient.STREAM, content=content, subject=topic, ) self._update_tsvector_index() narrow = [ dict(operator='sender', operand=self.example_email("cordelia")), dict(operator='search', operand='lunch'), ] result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_before=0, num_after=10, )) # type: Dict[str, Dict] self.assertEqual(len(result['messages']), 2) messages = result['messages'] narrow = [dict(operator='search', operand='https://google.com')] link_search_result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_before=0, num_after=10, )) # type: Dict[str, Dict] self.assertEqual(len(link_search_result['messages']), 1) self.assertEqual(link_search_result['messages'][0]['match_content'], '') meeting_message = [m for m in messages if m['subject'] == 'meetings'][0] self.assertEqual( meeting_message['match_subject'], 'meetings') self.assertEqual( meeting_message['match_content'], 'discuss lunch after ' + 'lunch
') meeting_message = [m for m in messages if m['subject'] == 'lunch plans'][0] self.assertEqual( meeting_message['match_subject'], 'lunch plans') self.assertEqual( meeting_message['match_content'], 'I am hungry!
') # Should not crash when multiple search operands are present multi_search_narrow = [ dict(operator='search', operand='discuss'), dict(operator='search', operand='after'), ] multi_search_result = self.get_and_check_messages(dict( narrow=ujson.dumps(multi_search_narrow), anchor=next_message_id, num_after=10, num_before=0, )) # type: Dict[str, Dict] self.assertEqual(len(multi_search_result['messages']), 1) self.assertEqual(multi_search_result['messages'][0]['match_content'], 'discuss lunch after lunch
') @override_settings(USING_PGROONGA=False) def test_get_messages_with_search_not_subscribed(self): # type: () -> None """Verify support for searching a stream you're not subscribed to""" self.subscribe(self.example_user("hamlet"), "newstream") self.send_message( sender_name=self.example_email("hamlet"), raw_recipients="newstream", message_type=Recipient.STREAM, content="Public special content!", subject="new", ) self._update_tsvector_index() self.login(self.example_email("cordelia")) stream_search_narrow = [ dict(operator='search', operand='special'), dict(operator='stream', operand='newstream'), ] stream_search_result = self.get_and_check_messages(dict( narrow=ujson.dumps(stream_search_narrow), anchor=0, num_after=10, num_before=10, )) # type: Dict[str, Dict] self.assertEqual(len(stream_search_result['messages']), 1) self.assertEqual(stream_search_result['messages'][0]['match_content'], 'Public special content!
') @override_settings(USING_PGROONGA=True) def test_get_messages_with_search_pgroonga(self): # type: () -> None self.login(self.example_email("cordelia")) next_message_id = self.get_last_message().id + 1 messages_to_search = [ (u'日本語', u'こんにちは。今日はいい天気ですね。'), (u'日本語', u'今朝はごはんを食べました。'), (u'日本語', u'昨日、日本のお菓子を送りました。'), ('english', u'I want to go to 日本!'), ('english', 'Can you speak https://en.wikipedia.org/wiki/Japanese?'), ('english', 'https://google.com'), ] for topic, content in messages_to_search: self.send_message( sender_name=self.example_email("cordelia"), raw_recipients="Verona", message_type=Recipient.STREAM, content=content, subject=topic, ) # We use brute force here and update our text search index # for the entire zerver_message table (which is small in test # mode). In production there is an async process which keeps # the search index up to date. with connection.cursor() as cursor: cursor.execute(""" UPDATE zerver_message SET search_pgroonga = subject || ' ' || rendered_content """) narrow = [ dict(operator='search', operand=u'日本'), ] result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_after=10, num_before=0, )) # type: Dict[str, Dict] self.assertEqual(len(result['messages']), 4) messages = result['messages'] japanese_message = [m for m in messages if m['subject'] == u'日本語'][-1] self.assertEqual( japanese_message['match_subject'], u'日本語') self.assertEqual( japanese_message['match_content'], u'昨日、日本の' + u'お菓子を送りました。
') english_message = [m for m in messages if m['subject'] == 'english'][0] self.assertEqual( english_message['match_subject'], 'english') self.assertIn( english_message['match_content'], # NOTE: The whitespace here is off due to a pgroonga bug. # This bug is a pgroonga regression and according to one of # the author, this should be fixed in its next release. [u'I want to go to 日本!
', # This is correct. u'I want to go to 日本!
', ]) # Should not crash when multiple search operands are present multi_search_narrow = [ dict(operator='search', operand='can'), dict(operator='search', operand='speak'), dict(operator='search', operand='wiki'), ] multi_search_result = self.get_and_check_messages(dict( narrow=ujson.dumps(multi_search_narrow), anchor=next_message_id, num_after=10, num_before=0, )) # type: Dict[str, Dict] self.assertEqual(len(multi_search_result['messages']), 1) self.assertEqual(multi_search_result['messages'][0]['match_content'], 'Can you speak https://en.wikipedia.org/wiki/Japanese?
') narrow = [dict(operator='search', operand='https://google.com')] link_search_result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_after=10, num_before=0, )) # type: Dict[str, Dict] self.assertEqual(len(link_search_result['messages']), 1) self.assertEqual(link_search_result['messages'][0]['match_content'], '') def test_messages_in_narrow_for_non_search(self): # type: () -> None email = self.example_email("cordelia") self.login(email) def send(content): # type: (Text) -> int msg_id = self.send_message( sender_name=email, raw_recipients="Verona", message_type=Recipient.STREAM, subject='test_topic', content=content, ) return msg_id good_id = send('http://foo.com') bad_id = send('no link here') msg_ids = [good_id, bad_id] send('http://bar.com but not in msg_ids') narrow = [ dict(operator='has', operand='link'), ] raw_params = dict(msg_ids=msg_ids, narrow=narrow) params = {k: ujson.dumps(v) for k, v in raw_params.items()} result = self.client_get('/json/messages/matches_narrow', params) self.assert_json_success(result) messages = result.json()['messages'] self.assertEqual(len(list(messages.keys())), 1) message = messages[str(good_id)] self.assertIn('a href=', message['match_content']) self.assertIn('http://foo.com', message['match_content']) self.assertEqual(message['match_subject'], 'test_topic') def test_get_messages_with_only_searching_anchor(self): # type: () -> None """ Test that specifying an anchor but 0 for num_before and num_after returns at most 1 message. """ self.login(self.example_email("cordelia")) anchor = self.send_message(self.example_email("cordelia"), "Verona", Recipient.STREAM) narrow = [dict(operator='sender', operand=self.example_email("cordelia"))] result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow), anchor=anchor, num_before=0, num_after=0)) # type: Dict[str, Dict] self.assertEqual(len(result['messages']), 1) narrow = [dict(operator='is', operand='mentioned')] result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow), anchor=anchor, num_before=0, num_after=0)) self.assertEqual(len(result['messages']), 0) def test_missing_params(self): # type: () -> None """ anchor, num_before, and num_after are all required POST parameters for get_messages. """ self.login(self.example_email("hamlet")) required_args = (("anchor", 1), ("num_before", 1), ("num_after", 1)) # type: Tuple[Tuple[Text, int], ...] for i in range(len(required_args)): post_params = dict(required_args[:i] + required_args[i + 1:]) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Missing '%s' argument" % (required_args[i][0],)) def test_bad_int_params(self): # type: () -> None """ num_before, num_after, and narrow must all be non-negative integers or strings that can be converted to non-negative integers. """ self.login(self.example_email("hamlet")) other_params = [("narrow", {}), ("anchor", 0)] int_params = ["num_before", "num_after"] bad_types = (False, "", "-1", -1) for idx, param in enumerate(int_params): for type in bad_types: # Rotate through every bad type for every integer # parameter, one at a time. post_params = dict(other_params + [(param, type)] + [(other_param, 0) for other_param in int_params[:idx] + int_params[idx + 1:]] ) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Bad value for '%s': %s" % (param, type)) def test_bad_narrow_type(self): # type: () -> None """ narrow must be a list of string pairs. """ self.login(self.example_email("hamlet")) other_params = [("anchor", 0), ("num_before", 0), ("num_after", 0)] # type: List[Tuple[Text, Union[int, str, bool]]] bad_types = (False, 0, '', '{malformed json,', '{foo: 3}', '[1,2]', '[["x","y","z"]]') # type: Tuple[Union[int, str, bool], ...] for type in bad_types: post_params = dict(other_params + [("narrow", type)]) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Bad value for 'narrow': %s" % (type,)) def test_bad_narrow_operator(self): # type: () -> None """ Unrecognized narrow operators are rejected. """ self.login(self.example_email("hamlet")) for operator in ['', 'foo', 'stream:verona', '__init__']: narrow = [dict(operator=operator, operand='')] params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, "Invalid narrow operator: unknown operator") def test_non_string_narrow_operand_in_dict(self): # type: () -> None """ We expect search operands to be strings, not integers. """ self.login(self.example_email("hamlet")) not_a_string = 42 narrow = [dict(operator='stream', operand=not_a_string)] params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, 'elem["operand"] is not a string') def exercise_bad_narrow_operand(self, operator, operands, error_msg): # type: (Text, Sequence, Text) -> None other_params = [("anchor", 0), ("num_before", 0), ("num_after", 0)] # type: List for operand in operands: post_params = dict(other_params + [ ("narrow", ujson.dumps([[operator, operand]]))]) result = self.client_get("/json/messages", post_params) self.assert_json_error_contains(result, error_msg) def test_bad_narrow_stream_content(self): # type: () -> None """ If an invalid stream name is requested in get_messages, an error is returned. """ self.login(self.example_email("hamlet")) bad_stream_content = (0, [], ["x", "y"]) # type: Sequence self.exercise_bad_narrow_operand("stream", bad_stream_content, "Bad value for 'narrow'") def test_bad_narrow_one_on_one_email_content(self): # type: () -> None """ If an invalid 'pm-with' is requested in get_messages, an error is returned. """ self.login(self.example_email("hamlet")) bad_stream_content = (0, [], ["x", "y"]) # type: Tuple[int, List[None], List[Text]] self.exercise_bad_narrow_operand("pm-with", bad_stream_content, "Bad value for 'narrow'") def test_bad_narrow_nonexistent_stream(self): # type: () -> None self.login(self.example_email("hamlet")) self.exercise_bad_narrow_operand("stream", ['non-existent stream'], "Invalid narrow operator: unknown stream") def test_bad_narrow_nonexistent_email(self): # type: () -> None self.login(self.example_email("hamlet")) self.exercise_bad_narrow_operand("pm-with", ['non-existent-user@zulip.com'], "Invalid narrow operator: unknown user") def test_message_without_rendered_content(self): # type: () -> None """Older messages may not have rendered_content in the database""" m = self.get_last_message() m.rendered_content = m.rendered_content_version = None m.content = 'test content' # Use to_dict_uncached_helper directly to avoid having to deal with remote cache d = MessageDict.to_dict_uncached_helper(m, True) self.assertEqual(d['content'], 'test content
') def common_check_get_messages_query(self, query_params, expected): # type: (Dict[str, object], Text) -> None user_profile = self.example_user('hamlet') request = POSTRequestMock(query_params, user_profile) with queries_captured() as queries: get_messages_backend(request, user_profile) for query in queries: if "/* get_messages */" in query['sql']: sql = str(query['sql']).replace(" /* get_messages */", '') self.assertEqual(sql, expected) return raise AssertionError("get_messages query not found") def test_use_first_unread_anchor_with_some_unread_messages(self): # type: () -> None user_profile = self.example_user('hamlet') # Have Othello send messages to Hamlet that he hasn't read. self.send_message(self.example_email("othello"), "Scotland", Recipient.STREAM) last_message_id_to_hamlet = self.send_message(self.example_email("othello"), self.example_email("hamlet"), Recipient.PERSONAL) # Add a few messages that help us test that our query doesn't # look at messages that are irrelevant to Hamlet. self.send_message(self.example_email("othello"), self.example_email("cordelia"), Recipient.PERSONAL) self.send_message(self.example_email("othello"), self.example_email("iago"), Recipient.PERSONAL) query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=10, num_after=10, narrow='[]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend(request, user_profile) # Verify the query for old messages looks correct. queries = [q for q in all_queries if '/* get_messages */' in q['sql']] self.assertEqual(len(queries), 1) sql = queries[0]['sql'] self.assertNotIn('AND message_id = %s' % (LARGER_THAN_MAX_MESSAGE_ID,), sql) self.assertIn('ORDER BY message_id ASC', sql) cond = 'WHERE user_profile_id = %d AND message_id >= %d' % (user_profile.id, last_message_id_to_hamlet) self.assertIn(cond, sql) cond = 'WHERE user_profile_id = %d AND message_id <= %d' % (user_profile.id, last_message_id_to_hamlet - 1) self.assertIn(cond, sql) def test_use_first_unread_anchor_with_no_unread_messages(self): # type: () -> None user_profile = self.example_user('hamlet') query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=10, num_after=10, narrow='[]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend(request, user_profile) # Next, verify the use_first_unread_anchor setting invokes # the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack. queries = [q for q in all_queries if '/* get_messages */' in q['sql']] self.assertEqual(len(queries), 1) self.assertIn('AND message_id <= %d' % (LARGER_THAN_MAX_MESSAGE_ID - 1,), queries[0]['sql']) # There should not be an after_query in this case, since it'd be useless self.assertNotIn('AND message_id >= %d' % (LARGER_THAN_MAX_MESSAGE_ID,), queries[0]['sql']) def test_use_first_unread_anchor_with_muted_topics(self): # type: () -> None """ Test that our logic related to `use_first_unread_anchor` invokes the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack for the `/* get_messages */` query when relevant muting is in effect. This is a very arcane test on arcane, but very heavily field-tested, logic in get_messages_backend(). If this test breaks, be absolutely sure you know what you're doing. """ realm = get_realm('zulip') self.make_stream('web stuff') self.make_stream('bogus') user_profile = self.example_user('hamlet') muted_topics = [ ['Scotland', 'golf'], ['web stuff', 'css'], ['bogus', 'bogus'] ] set_topic_mutes(user_profile, muted_topics) query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=0, num_after=0, narrow='[["stream", "Scotland"]]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_messages_backend(request, user_profile) # Do some tests on the main query, to verify the muting logic # runs on this code path. queries = [q for q in all_queries if str(q['sql']).startswith("SELECT message_id, flags")] self.assertEqual(len(queries), 1) stream = get_stream('Scotland', realm) recipient_id = get_recipient(Recipient.STREAM, stream.id).id cond = '''AND NOT (recipient_id = {scotland} AND upper(subject) = upper('golf'))'''.format(scotland=recipient_id) self.assertIn(cond, queries[0]['sql']) # Next, verify the use_first_unread_anchor setting invokes # the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack. queries = [q for q in all_queries if '/* get_messages */' in q['sql']] self.assertEqual(len(queries), 1) self.assertIn('AND message_id = %d' % (LARGER_THAN_MAX_MESSAGE_ID,), queries[0]['sql']) def test_exclude_muting_conditions(self): # type: () -> None realm = get_realm('zulip') self.make_stream('web stuff') user_profile = self.example_user('hamlet') self.make_stream('irrelevant_stream') # Test the do-nothing case first. muted_topics = [ ['irrelevant_stream', 'irrelevant_topic'] ] set_topic_mutes(user_profile, muted_topics) # If nothing relevant is muted, then exclude_muting_conditions() # should return an empty list. narrow = [ dict(operator='stream', operand='Scotland'), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) self.assertEqual(muting_conditions, []) # Ok, now set up our muted topics to include a topic relevant to our narrow. muted_topics = [ ['Scotland', 'golf'], ['web stuff', 'css'], ] set_topic_mutes(user_profile, muted_topics) # And verify that our query will exclude them. narrow = [ dict(operator='stream', operand='Scotland'), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select([column("id").label("message_id")], None, table("zerver_message")) query = query.where(*muting_conditions) expected_query = ''' SELECT id AS message_id FROM zerver_message WHERE NOT (recipient_id = :recipient_id_1 AND upper(subject) = upper(:upper_1)) ''' self.assertEqual(fix_ws(query), fix_ws(expected_query)) params = get_sqlalchemy_query_params(query) self.assertEqual(params['recipient_id_1'], get_recipient_id_for_stream_name(realm, 'Scotland')) self.assertEqual(params['upper_1'], 'golf') mute_stream(realm, user_profile, 'Verona') # Using a bogus stream name should be similar to using no narrow at # all, and we'll exclude all mutes. narrow = [ dict(operator='stream', operand='bogus-stream-name'), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select([column("id")], None, table("zerver_message")) query = query.where(and_(*muting_conditions)) expected_query = ''' SELECT id FROM zerver_message WHERE recipient_id NOT IN (:recipient_id_1) AND NOT (recipient_id = :recipient_id_2 AND upper(subject) = upper(:upper_1) OR recipient_id = :recipient_id_3 AND upper(subject) = upper(:upper_2))''' self.assertEqual(fix_ws(query), fix_ws(expected_query)) params = get_sqlalchemy_query_params(query) self.assertEqual(params['recipient_id_1'], get_recipient_id_for_stream_name(realm, 'Verona')) self.assertEqual(params['recipient_id_2'], get_recipient_id_for_stream_name(realm, 'Scotland')) self.assertEqual(params['upper_1'], 'golf') self.assertEqual(params['recipient_id_3'], get_recipient_id_for_stream_name(realm, 'web stuff')) self.assertEqual(params['upper_2'], 'css') def test_get_messages_queries(self): # type: () -> None query_ids = self.get_query_ids() sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 100 ORDER BY message_id DESC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 100, 'num_before': 10, 'num_after': 0}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM ((SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 99 ORDER BY message_id DESC \n LIMIT 10) UNION ALL (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id >= 100 ORDER BY message_id ASC \n LIMIT 11)) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 100, 'num_before': 10, 'num_after': 10}, sql) def test_get_messages_with_narrow_queries(self): # type: () -> None query_ids = self.get_query_ids() sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (sender_id = {othello_id} AND recipient_id = {hamlet_recipient} OR sender_id = {hamlet_id} AND recipient_id = {othello_recipient}) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["pm-with", "%s"]]' % (self.example_email("othello"),)}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2) != 0 AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["is", "starred"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND sender_id = {othello_id} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["sender", "%s"]]' % (self.example_email("othello"),)}, sql) sql_template = 'SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"]]'}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND upper(subject) = upper('blah') AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["topic", "blah"]]'}, sql) sql_template = "SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND upper(subject) = upper('blah') AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["topic", "blah"]]'}, sql) # Narrow to pms with yourself sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND sender_id = {hamlet_id} AND recipient_id = {hamlet_recipient} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["pm-with", "%s"]]' % (self.example_email("hamlet"),)}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND recipient_id = {scotland_recipient} AND (flags & 2) != 0 AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["is", "starred"]]'}, sql) @override_settings(USING_PGROONGA=False) def test_get_messages_with_search_queries(self): # type: () -> None query_ids = self.get_query_ids() sql_template = "SELECT anon_1.message_id, anon_1.flags, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT message_id, flags, subject, rendered_content, ts_match_locs_array('zulip.english_us_search', rendered_content, plainto_tsquery('zulip.english_us_search', 'jumping')) AS content_matches, ts_match_locs_array('zulip.english_us_search', escape_html(subject), plainto_tsquery('zulip.english_us_search', 'jumping')) AS subject_matches \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (search_tsvector @@ plainto_tsquery('zulip.english_us_search', 'jumping')) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" # type: Text sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["search", "jumping"]]'}, sql) sql_template = "SELECT anon_1.message_id, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT id AS message_id, subject, rendered_content, ts_match_locs_array('zulip.english_us_search', rendered_content, plainto_tsquery('zulip.english_us_search', 'jumping')) AS content_matches, ts_match_locs_array('zulip.english_us_search', escape_html(subject), plainto_tsquery('zulip.english_us_search', 'jumping')) AS subject_matches \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND (search_tsvector @@ plainto_tsquery('zulip.english_us_search', 'jumping')) AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["search", "jumping"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT message_id, flags, subject, rendered_content, ts_match_locs_array(\'zulip.english_us_search\', rendered_content, plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AS content_matches, ts_match_locs_array(\'zulip.english_us_search\', escape_html(subject), plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AS subject_matches \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (content ILIKE \'%jumping%\' OR subject ILIKE \'%jumping%\') AND (search_tsvector @@ plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["search", "\\"jumping\\" quickly"]]'}, sql) @override_settings(USING_PGROONGA=False) def test_get_messages_with_search_using_email(self): # type: () -> None self.login(self.example_email("cordelia")) messages_to_search = [ ('say hello', 'How are you doing, @**Othello, the Moor of Venice**?'), ('lunch plans', 'I am hungry!'), ] next_message_id = self.get_last_message().id + 1 for topic, content in messages_to_search: self.send_message( sender_name=self.example_email("cordelia"), raw_recipients="Verona", message_type=Recipient.STREAM, content=content, subject=topic, ) self._update_tsvector_index() narrow = [ dict(operator='sender', operand=self.example_email("cordelia")), dict(operator='search', operand=self.example_email("othello")), ] result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_after=10, )) # type: Dict[str, Dict] self.assertEqual(len(result['messages']), 0) narrow = [ dict(operator='sender', operand=self.example_email("cordelia")), dict(operator='search', operand='othello'), ] result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=next_message_id, num_after=10, )) self.assertEqual(len(result['messages']), 1) messages = result['messages'] meeting_message = [m for m in messages if m['subject'] == 'say hello'][0] self.assertEqual( meeting_message['match_subject'], 'say hello') self.assertEqual( meeting_message['match_content'], ('How are you doing, ' + '@Othello, the Moor of Venice?
') % ( self.example_email("othello"),))