discuss lunch after ' + 'lunch
') meeting_message = [m for m in messages if m['subject'] == 'lunch plans'][0] self.assertEqual( meeting_message['match_subject'], 'lunch plans') self.assertEqual( meeting_message['match_content'], 'I am hungry!
') @override_settings(USING_PGROONGA=True) def test_get_old_messages_with_search_pgroonga(self): self.login("cordelia@zulip.com") messages_to_search = [ (u'日本語', u'こんにちは。今日はいい天気ですね。'), (u'日本語', u'今朝はごはんを食べました。'), (u'日本語', u'昨日、日本のお菓子を送りました。'), ('english', u'I want to go to 日本!'), ('english', 'Can you speak Japanese?'), ] for topic, content in messages_to_search: self.send_message( sender_name="cordelia@zulip.com", raw_recipients="Verona", message_type=Recipient.STREAM, content=content, subject=topic, ) # We use brute force here and update our text search index # for the entire zerver_message table (which is small in test # mode). In production there is an async process which keeps # the search index up to date. with connection.cursor() as cursor: cursor.execute(""" UPDATE zerver_message SET search_pgroonga = subject || ' ' || rendered_content """) narrow = [ dict(operator='search', operand=u'日本'), ] result = self.get_and_check_messages(dict( narrow=ujson.dumps(narrow), anchor=0, num_after=10, )) self.assertEqual(len(result['messages']), 4) messages = result['messages'] japanese_message = [m for m in messages if m['subject'] == u'日本語'][-1] self.assertEqual( japanese_message['match_subject'], u'日本語') self.assertEqual( japanese_message['match_content'], u'昨日、日本の' + u'お菓子を送りました。
') english_message = [m for m in messages if m['subject'] == 'english'][0] self.assertEqual( english_message['match_subject'], 'english') self.assertEqual( english_message['match_content'], u'I want to go to 日本!
') def test_get_old_messages_with_only_searching_anchor(self): """ Test that specifying an anchor but 0 for num_before and num_after returns at most 1 message. """ self.login("cordelia@zulip.com") anchor = self.send_message("cordelia@zulip.com", "Verona", Recipient.STREAM) narrow = [dict(operator='sender', operand='cordelia@zulip.com')] result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow), anchor=anchor, num_before=0, num_after=0)) self.assertEqual(len(result['messages']), 1) narrow = [dict(operator='is', operand='mentioned')] result = self.get_and_check_messages(dict(narrow=ujson.dumps(narrow), anchor=anchor, num_before=0, num_after=0)) self.assertEqual(len(result['messages']), 0) def test_missing_params(self): """ anchor, num_before, and num_after are all required POST parameters for get_old_messages. """ self.login("hamlet@zulip.com") required_args = (("anchor", 1), ("num_before", 1), ("num_after", 1)) for i in range(len(required_args)): post_params = dict(required_args[:i] + required_args[i + 1:]) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Missing '%s' argument" % (required_args[i][0],)) def test_bad_int_params(self): """ num_before, num_after, and narrow must all be non-negative integers or strings that can be converted to non-negative integers. """ self.login("hamlet@zulip.com") other_params = [("narrow", {}), ("anchor", 0)] int_params = ["num_before", "num_after"] bad_types = (False, "", "-1", -1) for idx, param in enumerate(int_params): for type in bad_types: # Rotate through every bad type for every integer # parameter, one at a time. post_params = dict(other_params + [(param, type)] + \ [(other_param, 0) for other_param in \ int_params[:idx] + int_params[idx + 1:]] ) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Bad value for '%s': %s" % (param, type)) def test_bad_narrow_type(self): """ narrow must be a list of string pairs. """ self.login("hamlet@zulip.com") other_params = [("anchor", 0), ("num_before", 0), ("num_after", 0)] bad_types = (False, 0, '', '{malformed json,', '{foo: 3}', '[1,2]', '[["x","y","z"]]') for type in bad_types: post_params = dict(other_params + [("narrow", type)]) result = self.client_get("/json/messages", post_params) self.assert_json_error(result, "Bad value for 'narrow': %s" % (type,)) def test_old_empty_narrow(self): """ '{}' is accepted to mean 'no narrow', for use by old mobile clients. """ self.login("hamlet@zulip.com") all_result = self.get_and_check_messages({}) narrow_result = self.get_and_check_messages({'narrow': '{}'}) self.assertEqual(message_ids(all_result), message_ids(narrow_result)) def test_bad_narrow_operator(self): """ Unrecognized narrow operators are rejected. """ self.login("hamlet@zulip.com") for operator in ['', 'foo', 'stream:verona', '__init__']: narrow = [dict(operator=operator, operand='')] params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, "Invalid narrow operator: unknown operator") def test_non_string_narrow_operand_in_dict(self): """ We expect search operands to be strings, not integers. """ self.login("hamlet@zulip.com") not_a_string = 42 narrow = [dict(operator='stream', operand=not_a_string)] params = dict(anchor=0, num_before=0, num_after=0, narrow=ujson.dumps(narrow)) result = self.client_get("/json/messages", params) self.assert_json_error_contains(result, 'elem["operand"] is not a string') def exercise_bad_narrow_operand(self, operator, operands, error_msg): other_params = [("anchor", 0), ("num_before", 0), ("num_after", 0)] for operand in operands: post_params = dict(other_params + [ ("narrow", ujson.dumps([[operator, operand]]))]) result = self.client_get("/json/messages", post_params) self.assert_json_error_contains(result, error_msg) def test_bad_narrow_stream_content(self): """ If an invalid stream name is requested in get_old_messages, an error is returned. """ self.login("hamlet@zulip.com") bad_stream_content = (0, [], ["x", "y"]) self.exercise_bad_narrow_operand("stream", bad_stream_content, "Bad value for 'narrow'") def test_bad_narrow_one_on_one_email_content(self): """ If an invalid 'pm-with' is requested in get_old_messages, an error is returned. """ self.login("hamlet@zulip.com") bad_stream_content = (0, [], ["x", "y"]) self.exercise_bad_narrow_operand("pm-with", bad_stream_content, "Bad value for 'narrow'") def test_bad_narrow_nonexistent_stream(self): self.login("hamlet@zulip.com") self.exercise_bad_narrow_operand("stream", ['non-existent stream'], "Invalid narrow operator: unknown stream") def test_bad_narrow_nonexistent_email(self): self.login("hamlet@zulip.com") self.exercise_bad_narrow_operand("pm-with", ['non-existent-user@zulip.com'], "Invalid narrow operator: unknown user") def test_message_without_rendered_content(self): """Older messages may not have rendered_content in the database""" m = self.get_last_message() m.rendered_content = m.rendered_content_version = None m.content = 'test content' # Use to_dict_uncached_helper directly to avoid having to deal with remote cache d = m.to_dict_uncached_helper(True) self.assertEqual(d['content'], 'test content
') def common_check_get_old_messages_query(self, query_params, expected): user_profile = get_user_profile_by_email("hamlet@zulip.com") request = POSTRequestMock(query_params, user_profile) with queries_captured() as queries: get_old_messages_backend(request, user_profile) for query in queries: if "/* get_old_messages */" in query['sql']: sql = query['sql'].replace(" /* get_old_messages */", '') self.assertEqual(sql, expected) return self.fail("get_old_messages query not found") def test_use_first_unread_anchor_with_some_unread_messages(self): realm = get_realm('zulip.com') create_stream_if_needed(realm, 'devel') user_profile = get_user_profile_by_email("hamlet@zulip.com") # Have Othello send messages to Hamlet that he hasn't read. self.send_message("othello@zulip.com", "Scotland", Recipient.STREAM) last_message_id_to_hamlet = self.send_message("othello@zulip.com", "hamlet@zulip.com", Recipient.PERSONAL) # Add a few messages that help us test that our query doesn't # look at messages that are irrelevant to Hamlet. self.send_message("othello@zulip.com", "cordelia@zulip.com", Recipient.PERSONAL) self.send_message("othello@zulip.com", "iago@zulip.com", Recipient.PERSONAL) query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=0, num_after=0, narrow='[]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_old_messages_backend(request, user_profile) # Verify the query for old messages looks correct. queries = [q for q in all_queries if '/* get_old_messages */' in q['sql']] self.assertEqual(len(queries), 1) sql = queries[0]['sql'] self.assertNotIn('AND message_id = 10000000000000000', sql) self.assertIn('ORDER BY message_id ASC', sql) cond = 'WHERE user_profile_id = %d AND message_id = %d' % (user_profile.id, last_message_id_to_hamlet) self.assertIn(cond, sql) def test_use_first_unread_anchor_with_no_unread_messages(self): realm = get_realm('zulip.com') create_stream_if_needed(realm, 'devel') user_profile = get_user_profile_by_email("hamlet@zulip.com") query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=0, num_after=0, narrow='[]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_old_messages_backend(request, user_profile) # Next, verify the use_first_unread_anchor setting invokes # the `message_id = 10000000000000000` hack. queries = [q for q in all_queries if '/* get_old_messages */' in q['sql']] self.assertEqual(len(queries), 1) self.assertIn('AND message_id = 10000000000000000', queries[0]['sql']) def test_use_first_unread_anchor_with_muted_topics(self): """ Test that our logic related to `use_first_unread_anchor` invokes the `message_id = 10000000000000000` hack for the `/* get_old_messages */` query when relevant muting is in effect. This is a very arcane test on arcane, but very heavily field-tested, logic in get_old_messages_backend(). If this test breaks, be absolutely sure you know what you're doing. """ realm = get_realm('zulip.com') create_stream_if_needed(realm, 'devel') user_profile = get_user_profile_by_email("hamlet@zulip.com") user_profile.muted_topics = ujson.dumps([['Scotland', 'golf'], ['devel', 'css'], ['bogus', 'bogus']]) user_profile.save() query_params = dict( use_first_unread_anchor='true', anchor=0, num_before=0, num_after=0, narrow='[["stream", "Scotland"]]' ) request = POSTRequestMock(query_params, user_profile) with queries_captured() as all_queries: get_old_messages_backend(request, user_profile) # Do some tests on the main query, to verify the muting logic # runs on this code path. queries = [q for q in all_queries if q['sql'].startswith("SELECT message_id, flags")] self.assertEqual(len(queries), 1) stream = get_stream('Scotland', realm) recipient_id = get_recipient(Recipient.STREAM, stream.id).id cond = '''AND NOT (recipient_id = {scotland} AND upper(subject) = upper('golf'))'''.format(scotland=recipient_id) self.assertIn(cond, queries[0]['sql']) # Next, verify the use_first_unread_anchor setting invokes # the `message_id = 10000000000000000` hack. queries = [q for q in all_queries if '/* get_old_messages */' in q['sql']] self.assertEqual(len(queries), 1) self.assertIn('AND message_id = 10000000000000000', queries[0]['sql']) def test_exclude_muting_conditions(self): realm = get_realm('zulip.com') create_stream_if_needed(realm, 'devel') user_profile = get_user_profile_by_email("hamlet@zulip.com") # Test the do-nothing case first. user_profile.muted_topics = ujson.dumps([['irrelevant_stream', 'irrelevant_topic']]) user_profile.save() # If nothing relevant is muted, then exclude_muting_conditions() # should return an empty list. narrow = [ dict(operator='stream', operand='Scotland'), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) self.assertEqual(muting_conditions, []) # Ok, now set up our muted topics to include a topic relevant to our narrow. user_profile.muted_topics = ujson.dumps([['Scotland', 'golf'], ['devel', 'css'], ['bogus', 'bogus']]) user_profile.save() # And verify that our query will exclude them. narrow = [ dict(operator='stream', operand='Scotland'), ] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select([column("id").label("message_id")], None, "zerver_message") query = query.where(*muting_conditions) expected_query = ''' SELECT id AS message_id FROM zerver_message WHERE NOT (recipient_id = :recipient_id_1 AND upper(subject) = upper(:upper_1)) ''' self.assertEqual(fix_ws(query), fix_ws(expected_query)) params = get_sqlalchemy_query_params(query) self.assertEqual(params['recipient_id_1'], get_recipient_id_for_stream_name(realm, 'Scotland')) self.assertEqual(params['upper_1'], 'golf') mute_stream(realm, user_profile, 'Verona') narrow = [] muting_conditions = exclude_muting_conditions(user_profile, narrow) query = select([column("id")], None, "zerver_message") query = query.where(and_(*muting_conditions)) expected_query = ''' SELECT id FROM zerver_message WHERE recipient_id NOT IN (:recipient_id_1) AND NOT (recipient_id = :recipient_id_2 AND upper(subject) = upper(:upper_1) OR recipient_id = :recipient_id_3 AND upper(subject) = upper(:upper_2))''' self.assertEqual(fix_ws(query), fix_ws(expected_query)) params = get_sqlalchemy_query_params(query) self.assertEqual(params['recipient_id_1'], get_recipient_id_for_stream_name(realm, 'Verona')) self.assertEqual(params['recipient_id_2'], get_recipient_id_for_stream_name(realm, 'Scotland')) self.assertEqual(params['upper_1'], 'golf') self.assertEqual(params['recipient_id_3'], get_recipient_id_for_stream_name(realm, 'devel')) self.assertEqual(params['upper_2'], 'css') def test_get_old_messages_queries(self): query_ids = self.get_query_ids() sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 100 ORDER BY message_id DESC \n LIMIT 11) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 100, 'num_before': 10, 'num_after': 0}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM ((SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id <= 99 ORDER BY message_id DESC \n LIMIT 10) UNION ALL (SELECT message_id, flags \nFROM zerver_usermessage \nWHERE user_profile_id = {hamlet_id} AND message_id >= 100 ORDER BY message_id ASC \n LIMIT 11)) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 100, 'num_before': 10, 'num_after': 10}, sql) def test_get_old_messages_with_narrow_queries(self): query_ids = self.get_query_ids() sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (sender_id = {othello_id} AND recipient_id = {hamlet_recipient} OR sender_id = {hamlet_id} AND recipient_id = {othello_recipient}) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["pm-with", "othello@zulip.com"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (flags & 2) != 0 AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["is", "starred"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND sender_id = {othello_id} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["sender", "othello@zulip.com"]]'}, sql) sql_template = 'SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"]]'}, sql) sql_template = "SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND upper(subject) = upper('blah') AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["topic", "blah"]]'}, sql) sql_template = "SELECT anon_1.message_id \nFROM (SELECT id AS message_id \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND upper(subject) = upper('blah') AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["topic", "blah"]]'}, sql) # Narrow to pms with yourself sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND sender_id = {hamlet_id} AND recipient_id = {hamlet_recipient} AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["pm-with", "hamlet@zulip.com"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags \nFROM (SELECT message_id, flags \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND recipient_id = {scotland_recipient} AND (flags & 2) != 0 AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["is", "starred"]]'}, sql) @override_settings(USING_PGROONGA=False) def test_get_old_messages_with_search_queries(self): query_ids = self.get_query_ids() sql_template = "SELECT anon_1.message_id, anon_1.flags, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT message_id, flags, subject, rendered_content, ts_match_locs_array('zulip.english_us_search', rendered_content, plainto_tsquery('zulip.english_us_search', 'jumping')) AS content_matches, ts_match_locs_array('zulip.english_us_search', escape_html(subject), plainto_tsquery('zulip.english_us_search', 'jumping')) AS subject_matches \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (search_tsvector @@ plainto_tsquery('zulip.english_us_search', 'jumping')) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["search", "jumping"]]'}, sql) sql_template = "SELECT anon_1.message_id, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT id AS message_id, subject, rendered_content, ts_match_locs_array('zulip.english_us_search', rendered_content, plainto_tsquery('zulip.english_us_search', 'jumping')) AS content_matches, ts_match_locs_array('zulip.english_us_search', escape_html(subject), plainto_tsquery('zulip.english_us_search', 'jumping')) AS subject_matches \nFROM zerver_message \nWHERE recipient_id = {scotland_recipient} AND (search_tsvector @@ plainto_tsquery('zulip.english_us_search', 'jumping')) AND zerver_message.id >= 0 ORDER BY zerver_message.id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC" sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["stream", "Scotland"], ["search", "jumping"]]'}, sql) sql_template = 'SELECT anon_1.message_id, anon_1.flags, anon_1.subject, anon_1.rendered_content, anon_1.content_matches, anon_1.subject_matches \nFROM (SELECT message_id, flags, subject, rendered_content, ts_match_locs_array(\'zulip.english_us_search\', rendered_content, plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AS content_matches, ts_match_locs_array(\'zulip.english_us_search\', escape_html(subject), plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AS subject_matches \nFROM zerver_usermessage JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id \nWHERE user_profile_id = {hamlet_id} AND (content ILIKE \'%jumping%\' OR subject ILIKE \'%jumping%\') AND (search_tsvector @@ plainto_tsquery(\'zulip.english_us_search\', \'"jumping" quickly\')) AND message_id >= 0 ORDER BY message_id ASC \n LIMIT 10) AS anon_1 ORDER BY message_id ASC' sql = sql_template.format(**query_ids) self.common_check_get_old_messages_query({'anchor': 0, 'num_before': 0, 'num_after': 10, 'narrow': '[["search", "\\"jumping\\" quickly"]]'}, sql)