2016-05-25 15:02:02 +02:00
|
|
|
from django.utils.translation import ugettext as _
|
2017-04-15 04:03:56 +02:00
|
|
|
from django.utils.timezone import now as timezone_now
|
2013-12-12 18:36:32 +01:00
|
|
|
from django.conf import settings
|
|
|
|
from django.core import validators
|
|
|
|
from django.core.exceptions import ValidationError
|
2019-01-24 19:14:25 +01:00
|
|
|
from django.db import connection, IntegrityError
|
2016-06-06 00:32:39 +02:00
|
|
|
from django.http import HttpRequest, HttpResponse
|
2019-02-02 23:53:22 +01:00
|
|
|
from typing import Dict, List, Set, Any, Iterable, \
|
2018-09-22 05:10:57 +02:00
|
|
|
Optional, Tuple, Union, Sequence, cast
|
2017-07-21 02:17:28 +02:00
|
|
|
from zerver.lib.exceptions import JsonableError, ErrorCode
|
2017-02-20 00:19:29 +01:00
|
|
|
from zerver.lib.html_diff import highlight_html_differences
|
2020-05-07 13:19:54 +02:00
|
|
|
from zerver.decorator import has_request_variables, REQ
|
2013-11-26 00:41:24 +01:00
|
|
|
from django.utils.html import escape as escape_html
|
2013-12-12 18:36:32 +01:00
|
|
|
from zerver.lib import bugdown
|
2018-06-15 20:56:36 +02:00
|
|
|
from zerver.lib.zcommand import process_zcommands
|
2019-06-28 21:05:58 +02:00
|
|
|
from zerver.lib.actions import recipient_for_user_profiles, do_update_message_flags, \
|
2018-08-11 16:26:46 +02:00
|
|
|
compute_irc_user_fullname, compute_jabber_user_fullname, \
|
2013-12-12 18:36:32 +01:00
|
|
|
create_mirror_user_if_needed, check_send_message, do_update_message, \
|
2020-03-28 07:46:04 +01:00
|
|
|
extract_private_recipients, render_incoming_message, do_delete_messages, \
|
2020-02-11 17:44:41 +01:00
|
|
|
do_mark_all_as_read, do_mark_stream_messages_as_read, extract_stream_indicator, \
|
2018-01-04 18:41:34 +01:00
|
|
|
get_user_info_for_message_updates, check_schedule_message
|
2019-06-28 21:05:58 +02:00
|
|
|
from zerver.lib.addressee import get_user_profiles, get_user_profiles_by_ids
|
2016-10-27 12:06:44 +02:00
|
|
|
from zerver.lib.queue import queue_json_publish
|
2016-10-04 15:52:26 +02:00
|
|
|
from zerver.lib.message import (
|
2016-10-12 02:14:08 +02:00
|
|
|
access_message,
|
2017-11-07 17:36:29 +01:00
|
|
|
messages_for_ids,
|
2016-11-07 20:40:40 +01:00
|
|
|
render_markdown,
|
2018-01-02 18:33:28 +01:00
|
|
|
get_first_visible_message_id,
|
2020-03-28 07:46:04 +01:00
|
|
|
truncate_body,
|
2016-10-04 15:52:26 +02:00
|
|
|
)
|
2013-12-12 18:36:32 +01:00
|
|
|
from zerver.lib.response import json_success, json_error
|
2016-07-19 08:12:35 +02:00
|
|
|
from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection
|
2019-08-13 20:20:36 +02:00
|
|
|
from zerver.lib.streams import access_stream_by_id, get_public_streams_queryset, \
|
|
|
|
can_access_stream_history_by_name, can_access_stream_history_by_id, \
|
2020-02-19 01:38:34 +01:00
|
|
|
get_stream_by_narrow_operand_access_unchecked, get_stream_by_id
|
2018-01-04 18:41:34 +01:00
|
|
|
from zerver.lib.timestamp import datetime_to_timestamp, convert_to_UTC
|
|
|
|
from zerver.lib.timezone import get_timezone
|
2018-11-01 22:15:43 +01:00
|
|
|
from zerver.lib.topic import (
|
2018-11-09 17:06:00 +01:00
|
|
|
topic_column_sa,
|
2018-11-01 22:15:43 +01:00
|
|
|
topic_match_sa,
|
2018-11-09 17:32:08 +01:00
|
|
|
user_message_exists_for_topic,
|
2018-11-09 17:19:17 +01:00
|
|
|
DB_TOPIC_NAME,
|
2018-11-09 17:53:59 +01:00
|
|
|
LEGACY_PREV_TOPIC,
|
2018-11-09 17:25:57 +01:00
|
|
|
MATCH_TOPIC,
|
2018-11-09 18:35:34 +01:00
|
|
|
REQ_topic,
|
2018-11-01 22:15:43 +01:00
|
|
|
)
|
2017-08-29 17:16:53 +02:00
|
|
|
from zerver.lib.topic_mutes import exclude_topic_mutes
|
2013-12-12 18:36:32 +01:00
|
|
|
from zerver.lib.utils import statsd
|
2014-02-14 15:48:42 +01:00
|
|
|
from zerver.lib.validator import \
|
2019-07-13 01:48:04 +02:00
|
|
|
check_list, check_int, check_dict, check_string, check_bool, \
|
2020-04-11 17:32:32 +02:00
|
|
|
check_string_or_int_list, check_string_or_int, check_string_in, \
|
2020-05-07 13:19:54 +02:00
|
|
|
check_required_string, to_non_negative_int
|
2018-08-11 16:26:46 +02:00
|
|
|
from zerver.lib.zephyr import compute_mit_user_fullname
|
2018-01-04 18:41:34 +01:00
|
|
|
from zerver.models import Message, UserProfile, Stream, Subscription, Client,\
|
2020-05-07 09:48:43 +02:00
|
|
|
Realm, RealmDomain, Recipient, UserMessage, UserActivity, \
|
2019-08-11 18:57:54 +02:00
|
|
|
email_to_domain, get_realm, get_active_streams, get_user_including_cross_realm, \
|
2019-12-05 23:26:24 +01:00
|
|
|
get_user_by_id_in_realm_including_cross_realm
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2013-12-10 23:32:29 +01:00
|
|
|
from sqlalchemy import func
|
2019-08-28 11:06:38 +02:00
|
|
|
from sqlalchemy.dialects import postgresql
|
2013-12-10 23:32:29 +01:00
|
|
|
from sqlalchemy.sql import select, join, column, literal_column, literal, and_, \
|
2019-02-02 23:53:22 +01:00
|
|
|
or_, not_, union_all, alias, Selectable, ColumnElement, table
|
2013-12-10 23:32:29 +01:00
|
|
|
|
2018-01-04 18:41:34 +01:00
|
|
|
from dateutil.parser import parse as dateparser
|
2013-12-12 18:36:32 +01:00
|
|
|
import re
|
|
|
|
import ujson
|
2016-06-21 21:34:41 +02:00
|
|
|
import datetime
|
2016-06-24 02:26:09 +02:00
|
|
|
|
2017-02-23 05:50:15 +01:00
|
|
|
LARGER_THAN_MAX_MESSAGE_ID = 10000000000000000
|
2018-09-09 14:54:52 +02:00
|
|
|
MAX_MESSAGES_PER_FETCH = 5000
|
2017-02-23 05:50:15 +01:00
|
|
|
|
2016-04-21 21:47:01 +02:00
|
|
|
class BadNarrowOperator(JsonableError):
|
2017-07-21 02:17:28 +02:00
|
|
|
code = ErrorCode.BAD_NARROW
|
|
|
|
data_fields = ['desc']
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def __init__(self, desc: str) -> None:
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
self.desc: str = desc
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-07-21 02:17:28 +02:00
|
|
|
@staticmethod
|
2017-11-27 09:28:57 +01:00
|
|
|
def msg_format() -> str:
|
2017-07-21 02:17:28 +02:00
|
|
|
return _('Invalid narrow operator: {desc}')
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-11-09 11:45:56 +01:00
|
|
|
# TODO: Should be Select, but sqlalchemy stubs are busted
|
|
|
|
Query = Any
|
|
|
|
|
|
|
|
# TODO: should be Callable[[ColumnElement], ColumnElement], but sqlalchemy stubs are busted
|
|
|
|
ConditionTransform = Any
|
2016-09-12 02:09:24 +02:00
|
|
|
|
2019-08-11 19:07:34 +02:00
|
|
|
OptionalNarrowListT = Optional[List[Dict[str, Any]]]
|
|
|
|
|
2019-08-28 11:06:38 +02:00
|
|
|
# These delimiters will not appear in rendered messages or HTML-escaped topics.
|
|
|
|
TS_START = "<ts-match>"
|
|
|
|
TS_STOP = "</ts-match>"
|
|
|
|
|
|
|
|
def ts_locs_array(
|
|
|
|
config: ColumnElement, text: ColumnElement, tsquery: ColumnElement
|
|
|
|
) -> ColumnElement:
|
|
|
|
options = "HighlightAll = TRUE, StartSel = %s, StopSel = %s" % (TS_START, TS_STOP)
|
|
|
|
delimited = func.ts_headline(config, text, tsquery, options)
|
|
|
|
parts = func.unnest(func.string_to_array(delimited, TS_START)).alias()
|
|
|
|
part = column(parts.name)
|
|
|
|
part_len = func.length(part) - len(TS_STOP)
|
|
|
|
match_pos = func.sum(part_len).over(rows=(None, -1)) + len(TS_STOP)
|
|
|
|
match_len = func.strpos(part, TS_STOP) - 1
|
|
|
|
return func.array(
|
|
|
|
select([postgresql.array([match_pos, match_len])])
|
|
|
|
.select_from(parts)
|
|
|
|
.offset(1)
|
|
|
|
.as_scalar()
|
|
|
|
)
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
# When you add a new operator to this, also update zerver/lib/narrow.py
|
2017-11-05 11:53:59 +01:00
|
|
|
class NarrowBuilder:
|
2017-06-30 01:46:57 +02:00
|
|
|
'''
|
|
|
|
Build up a SQLAlchemy query to find messages matching a narrow.
|
|
|
|
'''
|
|
|
|
|
|
|
|
# This class has an important security invariant:
|
|
|
|
#
|
|
|
|
# None of these methods ever *add* messages to a query's result.
|
|
|
|
#
|
|
|
|
# That is, the `add_term` method, and its helpers the `by_*` methods,
|
|
|
|
# are passed a Query object representing a query for messages; they may
|
|
|
|
# call some methods on it, and then they return a resulting Query
|
|
|
|
# object. Things these methods may do to the queries they handle
|
|
|
|
# include
|
|
|
|
# * add conditions to filter out rows (i.e., messages), with `query.where`
|
|
|
|
# * add columns for more information on the same message, with `query.column`
|
|
|
|
# * add a join for more information on the same message
|
|
|
|
#
|
|
|
|
# Things they may not do include
|
|
|
|
# * anything that would pull in additional rows, or information on
|
|
|
|
# other messages.
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def __init__(self, user_profile: UserProfile, msg_id_column: str) -> None:
|
2013-12-12 18:36:32 +01:00
|
|
|
self.user_profile = user_profile
|
2013-12-10 23:32:29 +01:00
|
|
|
self.msg_id_column = msg_id_column
|
2017-07-18 23:03:17 +02:00
|
|
|
self.user_realm = user_profile.realm
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def add_term(self, query: Query, term: Dict[str, Any]) -> Query:
|
2017-06-30 01:46:57 +02:00
|
|
|
"""
|
|
|
|
Extend the given query to one narrowed by the given term, and return the result.
|
|
|
|
|
|
|
|
This method satisfies an important security property: the returned
|
|
|
|
query never includes a message that the given query didn't. In
|
|
|
|
particular, if the given query will only find messages that a given
|
|
|
|
user can legitimately see, then so will the returned query.
|
|
|
|
"""
|
|
|
|
# To maintain the security property, we hold all the `by_*`
|
|
|
|
# methods to the same criterion. See the class's block comment
|
|
|
|
# for details.
|
2016-09-12 02:09:24 +02:00
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
# We have to be careful here because we're letting users call a method
|
|
|
|
# by name! The prefix 'by_' prevents it from colliding with builtin
|
|
|
|
# Python __magic__ stuff.
|
2014-02-10 21:45:53 +01:00
|
|
|
operator = term['operator']
|
|
|
|
operand = term['operand']
|
2014-02-11 21:36:59 +01:00
|
|
|
|
2014-02-12 19:09:11 +01:00
|
|
|
negated = term.get('negated', False)
|
2014-02-11 21:36:59 +01:00
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
method_name = 'by_' + operator.replace('-', '_')
|
|
|
|
method = getattr(self, method_name, None)
|
|
|
|
if method is None:
|
|
|
|
raise BadNarrowOperator('unknown operator ' + operator)
|
|
|
|
|
2014-02-12 19:09:11 +01:00
|
|
|
if negated:
|
|
|
|
maybe_negate = not_
|
|
|
|
else:
|
|
|
|
maybe_negate = lambda cond: cond
|
|
|
|
|
|
|
|
return method(query, operand, maybe_negate)
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_has(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2014-03-05 18:41:01 +01:00
|
|
|
if operand not in ['attachment', 'image', 'link']:
|
|
|
|
raise BadNarrowOperator("unknown 'has' operand " + operand)
|
|
|
|
col_name = 'has_' + operand
|
|
|
|
cond = column(col_name)
|
|
|
|
return query.where(maybe_negate(cond))
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_in(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2014-02-27 23:57:16 +01:00
|
|
|
if operand == 'home':
|
|
|
|
conditions = exclude_muting_conditions(self.user_profile, [])
|
|
|
|
return query.where(and_(*conditions))
|
|
|
|
elif operand == 'all':
|
|
|
|
return query
|
|
|
|
|
|
|
|
raise BadNarrowOperator("unknown 'in' operand " + operand)
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_is(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2013-12-12 18:36:32 +01:00
|
|
|
if operand == 'private':
|
2018-08-08 11:09:43 +02:00
|
|
|
cond = column("flags").op("&")(UserMessage.flags.is_private.mask) != 0
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
elif operand == 'starred':
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = column("flags").op("&")(UserMessage.flags.starred.mask) != 0
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2017-06-19 03:21:48 +02:00
|
|
|
elif operand == 'unread':
|
|
|
|
cond = column("flags").op("&")(UserMessage.flags.read.mask) == 0
|
|
|
|
return query.where(maybe_negate(cond))
|
2017-08-16 15:20:11 +02:00
|
|
|
elif operand == 'mentioned':
|
|
|
|
cond1 = column("flags").op("&")(UserMessage.flags.mentioned.mask) != 0
|
|
|
|
cond2 = column("flags").op("&")(UserMessage.flags.wildcard_mentioned.mask) != 0
|
|
|
|
cond = or_(cond1, cond2)
|
|
|
|
return query.where(maybe_negate(cond))
|
|
|
|
elif operand == 'alerted':
|
|
|
|
cond = column("flags").op("&")(UserMessage.flags.has_alert_word.mask) != 0
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
raise BadNarrowOperator("unknown 'is' operand " + operand)
|
|
|
|
|
2014-01-07 22:15:22 +01:00
|
|
|
_alphanum = frozenset(
|
|
|
|
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
|
|
|
|
|
2018-04-24 03:47:28 +02:00
|
|
|
def _pg_re_escape(self, pattern: str) -> str:
|
2014-01-07 22:15:22 +01:00
|
|
|
"""
|
|
|
|
Escape user input to place in a regex
|
|
|
|
|
|
|
|
Python's re.escape escapes unicode characters in a way which postgres
|
2017-11-04 05:23:22 +01:00
|
|
|
fails on, '\u03bb' to '\\\u03bb'. This function will correctly escape
|
|
|
|
them for postgres, '\u03bb' to '\\u03bb'.
|
2014-01-07 22:15:22 +01:00
|
|
|
"""
|
|
|
|
s = list(pattern)
|
|
|
|
for i, c in enumerate(s):
|
|
|
|
if c not in self._alphanum:
|
2017-04-26 01:28:22 +02:00
|
|
|
if ord(c) >= 128:
|
2014-01-07 22:15:22 +01:00
|
|
|
# convert the character to hex postgres regex will take
|
|
|
|
# \uXXXX
|
|
|
|
s[i] = '\\u{:0>4x}'.format(ord(c))
|
|
|
|
else:
|
|
|
|
s[i] = '\\' + c
|
|
|
|
return ''.join(s)
|
|
|
|
|
2019-08-07 17:32:19 +02:00
|
|
|
def by_stream(self, query: Query, operand: Union[str, int], maybe_negate: ConditionTransform) -> Query:
|
2017-03-23 07:22:28 +01:00
|
|
|
try:
|
2017-08-15 18:58:29 +02:00
|
|
|
# Because you can see your own message history for
|
|
|
|
# private streams you are no longer subscribed to, we
|
2019-08-11 18:57:54 +02:00
|
|
|
# need get_stream_by_narrow_operand_access_unchecked here.
|
|
|
|
stream = get_stream_by_narrow_operand_access_unchecked(operand, self.user_profile.realm)
|
2017-03-23 07:22:28 +01:00
|
|
|
except Stream.DoesNotExist:
|
2019-08-07 17:32:19 +02:00
|
|
|
raise BadNarrowOperator('unknown stream ' + str(operand))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2016-07-27 01:45:29 +02:00
|
|
|
if self.user_profile.realm.is_zephyr_mirror_realm:
|
2017-06-30 02:24:05 +02:00
|
|
|
# MIT users expect narrowing to "social" to also show messages to
|
|
|
|
# /^(un)*social(.d)*$/ (unsocial, ununsocial, social.d, ...).
|
|
|
|
|
|
|
|
# In `ok_to_include_history`, we assume that a non-negated
|
|
|
|
# `stream` term for a public stream will limit the query to
|
|
|
|
# that specific stream. So it would be a bug to hit this
|
|
|
|
# codepath after relying on this term there. But all streams in
|
|
|
|
# a Zephyr realm are private, so that doesn't happen.
|
|
|
|
assert(not stream.is_public())
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE)
|
2017-02-23 01:06:04 +01:00
|
|
|
# Since the regex has a `.+` in it and "" is invalid as a
|
|
|
|
# stream name, this will always match
|
|
|
|
assert(m is not None)
|
|
|
|
base_stream_name = m.group(1)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2014-01-24 23:30:53 +01:00
|
|
|
matching_streams = get_active_streams(self.user_profile.realm).filter(
|
|
|
|
name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name),))
|
2019-12-06 00:15:59 +01:00
|
|
|
recipient_ids = [matching_stream.recipient_id for matching_stream in matching_streams]
|
|
|
|
cond = column("recipient_id").in_(recipient_ids)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-12-05 23:26:24 +01:00
|
|
|
recipient = stream.recipient
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = column("recipient_id") == recipient.id
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-08-13 20:20:36 +02:00
|
|
|
def by_streams(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
|
|
|
if operand == 'public':
|
|
|
|
# Get all both subscribed and non subscribed public streams
|
|
|
|
# but exclude any private subscribed streams.
|
|
|
|
public_streams_queryset = get_public_streams_queryset(self.user_profile.realm)
|
|
|
|
recipient_ids = Recipient.objects.filter(
|
|
|
|
type=Recipient.STREAM,
|
2019-09-23 04:25:11 +02:00
|
|
|
type_id__in=public_streams_queryset).values_list('id', flat=True).order_by('id')
|
2019-08-13 20:20:36 +02:00
|
|
|
cond = column("recipient_id").in_(recipient_ids)
|
|
|
|
return query.where(maybe_negate(cond))
|
|
|
|
raise BadNarrowOperator('unknown streams operand ' + operand)
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_topic(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2016-07-27 01:45:29 +02:00
|
|
|
if self.user_profile.realm.is_zephyr_mirror_realm:
|
2013-12-12 18:36:32 +01:00
|
|
|
# MIT users expect narrowing to topic "foo" to also show messages to /^foo(.d)*$/
|
|
|
|
# (foo, foo.d, foo.d.d, etc)
|
|
|
|
m = re.search(r'^(.*?)(?:\.d)*$', operand, re.IGNORECASE)
|
2017-02-23 01:06:04 +01:00
|
|
|
# Since the regex has a `.*` in it, this will always match
|
|
|
|
assert(m is not None)
|
|
|
|
base_topic = m.group(1)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
|
|
|
# Additionally, MIT users expect the empty instance and
|
|
|
|
# instance "personal" to be the same.
|
|
|
|
if base_topic in ('', 'personal', '(instance "")'):
|
2017-02-22 21:23:22 +01:00
|
|
|
cond = or_(
|
2018-11-01 22:15:43 +01:00
|
|
|
topic_match_sa(""),
|
|
|
|
topic_match_sa(".d"),
|
|
|
|
topic_match_sa(".d.d"),
|
|
|
|
topic_match_sa(".d.d.d"),
|
|
|
|
topic_match_sa(".d.d.d.d"),
|
|
|
|
topic_match_sa("personal"),
|
|
|
|
topic_match_sa("personal.d"),
|
|
|
|
topic_match_sa("personal.d.d"),
|
|
|
|
topic_match_sa("personal.d.d.d"),
|
|
|
|
topic_match_sa("personal.d.d.d.d"),
|
|
|
|
topic_match_sa('(instance "")'),
|
|
|
|
topic_match_sa('(instance "").d'),
|
|
|
|
topic_match_sa('(instance "").d.d'),
|
|
|
|
topic_match_sa('(instance "").d.d.d'),
|
|
|
|
topic_match_sa('(instance "").d.d.d.d'),
|
2017-02-22 21:23:22 +01:00
|
|
|
)
|
2013-12-12 18:36:32 +01:00
|
|
|
else:
|
2017-02-22 21:23:22 +01:00
|
|
|
# We limit `.d` counts, since postgres has much better
|
|
|
|
# query planning for this than they do for a regular
|
|
|
|
# expression (which would sometimes table scan).
|
|
|
|
cond = or_(
|
2018-11-01 22:15:43 +01:00
|
|
|
topic_match_sa(base_topic),
|
|
|
|
topic_match_sa(base_topic + ".d"),
|
|
|
|
topic_match_sa(base_topic + ".d.d"),
|
|
|
|
topic_match_sa(base_topic + ".d.d.d"),
|
|
|
|
topic_match_sa(base_topic + ".d.d.d.d"),
|
2017-02-22 21:23:22 +01:00
|
|
|
)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-11-01 22:15:43 +01:00
|
|
|
cond = topic_match_sa(operand)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-07-13 01:48:04 +02:00
|
|
|
def by_sender(self, query: Query, operand: Union[str, int], maybe_negate: ConditionTransform) -> Query:
|
2013-12-12 18:36:32 +01:00
|
|
|
try:
|
2019-07-13 01:48:04 +02:00
|
|
|
if isinstance(operand, str):
|
|
|
|
sender = get_user_including_cross_realm(operand, self.user_realm)
|
|
|
|
else:
|
|
|
|
sender = get_user_by_id_in_realm_including_cross_realm(operand, self.user_realm)
|
2013-12-12 18:36:32 +01:00
|
|
|
except UserProfile.DoesNotExist:
|
2019-07-13 01:48:04 +02:00
|
|
|
raise BadNarrowOperator('unknown user ' + str(operand))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = column("sender_id") == literal(sender.id)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_near(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2013-12-10 23:32:29 +01:00
|
|
|
return query
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_id(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2018-11-07 00:53:02 +01:00
|
|
|
if not str(operand).isdigit():
|
|
|
|
raise BadNarrowOperator("Invalid message ID")
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = self.msg_id_column == literal(operand)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-06-08 23:21:01 +02:00
|
|
|
def by_pm_with(self, query: Query, operand: Union[str, Iterable[int]],
|
|
|
|
maybe_negate: ConditionTransform) -> Query:
|
2019-06-28 21:05:58 +02:00
|
|
|
|
|
|
|
try:
|
|
|
|
if isinstance(operand, str):
|
|
|
|
email_list = operand.split(",")
|
|
|
|
user_profiles = get_user_profiles(
|
|
|
|
emails=email_list,
|
|
|
|
realm=self.user_realm
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
"""
|
|
|
|
This is where we handle passing a list of user IDs for the narrow, which is the
|
|
|
|
preferred/cleaner API.
|
|
|
|
"""
|
|
|
|
user_profiles = get_user_profiles_by_ids(
|
|
|
|
user_ids=operand,
|
|
|
|
realm=self.user_realm
|
|
|
|
)
|
|
|
|
|
|
|
|
recipient = recipient_for_user_profiles(user_profiles=user_profiles,
|
|
|
|
forwarded_mirror_message=False,
|
|
|
|
forwarder_user_profile=None,
|
2019-06-29 15:09:35 +02:00
|
|
|
sender=self.user_profile,
|
|
|
|
allow_deactivated=True)
|
2019-06-28 21:05:58 +02:00
|
|
|
except (JsonableError, ValidationError):
|
|
|
|
raise BadNarrowOperator('unknown user in ' + str(operand))
|
|
|
|
|
|
|
|
# Group DM
|
|
|
|
if recipient.type == Recipient.HUDDLE:
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = column("recipient_id") == recipient.id
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
narrow: Handle spurious emails in pm-with searches.
If cordelia searches on pm-with:iago@zulip.com,cordelia@zulip.com,
we now properly treat that the same way as pm-with:iago@zulip.com.
Before this fix, the query would initially go through the
huddle code path. The symptom wasn't completely obvious, as
eventually a deeper function would return a recipient id
corresponding to a single PM with @iago@zulip.com, but we would
only get messages where iago was the recipient, and not any
messages where he was the sender to cordelia.
I put the helper function for this in zerver/lib/addressee, which
is somewhat speculative. Eventually, we'll want pm-with queries
to allow for user ids, and I imagine there will be some shared
logic with other Addressee code in terms of how we handle these
strings. The way we deal with lists of emails/users for various
endpoints is kind of haphazard in the current code, although
granted it's mostly just repeating the same simple patterns. It
would be nice for some of this code to converge a bit. This
affects new messages, typing indicators, search filters, etc.,
and some endpoints have strange legacy stuff like supporting
JSON-encoded lists, so it's not trivial to clean this up.
Tweaked by tabbott to add some additional tests.
2018-10-12 17:56:46 +02:00
|
|
|
|
2019-06-28 21:05:58 +02:00
|
|
|
# 1:1 PM
|
|
|
|
other_participant = None
|
|
|
|
|
|
|
|
# Find if another person is in PM
|
|
|
|
for user in user_profiles:
|
|
|
|
if user.id != self.user_profile.id:
|
|
|
|
other_participant = user
|
|
|
|
|
|
|
|
# PM with another person
|
|
|
|
if other_participant:
|
|
|
|
# We need bidirectional messages PM with another person.
|
|
|
|
# But Recipient.PERSONAL objects only encode the person who
|
|
|
|
# received the message, and not the other participant in
|
|
|
|
# the thread (the sender), we need to do a somewhat
|
|
|
|
# complex query to get messages between these two users
|
|
|
|
# with either of them as the sender.
|
2019-12-05 23:35:33 +01:00
|
|
|
self_recipient_id = self.user_profile.recipient_id
|
2019-06-28 21:05:58 +02:00
|
|
|
cond = or_(and_(column("sender_id") == other_participant.id,
|
2019-12-05 23:35:33 +01:00
|
|
|
column("recipient_id") == self_recipient_id),
|
2014-02-11 23:33:24 +01:00
|
|
|
and_(column("sender_id") == self.user_profile.id,
|
2019-06-28 21:05:58 +02:00
|
|
|
column("recipient_id") == recipient.id))
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-10 23:32:29 +01:00
|
|
|
|
2019-06-28 21:05:58 +02:00
|
|
|
# PM with self
|
|
|
|
cond = and_(column("sender_id") == self.user_profile.id,
|
|
|
|
column("recipient_id") == recipient.id)
|
|
|
|
return query.where(maybe_negate(cond))
|
|
|
|
|
2019-07-14 19:31:28 +02:00
|
|
|
def by_group_pm_with(self, query: Query, operand: Union[str, int],
|
2017-11-27 09:28:57 +01:00
|
|
|
maybe_negate: ConditionTransform) -> Query:
|
2017-03-23 23:35:37 +01:00
|
|
|
try:
|
2019-07-14 19:31:28 +02:00
|
|
|
if isinstance(operand, str):
|
|
|
|
narrow_profile = get_user_including_cross_realm(operand, self.user_realm)
|
|
|
|
else:
|
|
|
|
narrow_profile = get_user_by_id_in_realm_including_cross_realm(operand, self.user_realm)
|
2017-03-23 23:35:37 +01:00
|
|
|
except UserProfile.DoesNotExist:
|
2019-06-08 23:21:01 +02:00
|
|
|
raise BadNarrowOperator('unknown user ' + str(operand))
|
2017-03-23 23:35:37 +01:00
|
|
|
|
|
|
|
self_recipient_ids = [
|
|
|
|
recipient_tuple['recipient_id'] for recipient_tuple
|
|
|
|
in Subscription.objects.filter(
|
|
|
|
user_profile=self.user_profile,
|
|
|
|
recipient__type=Recipient.HUDDLE
|
|
|
|
).values("recipient_id")]
|
|
|
|
narrow_recipient_ids = [
|
|
|
|
recipient_tuple['recipient_id'] for recipient_tuple
|
|
|
|
in Subscription.objects.filter(
|
|
|
|
user_profile=narrow_profile,
|
|
|
|
recipient__type=Recipient.HUDDLE
|
|
|
|
).values("recipient_id")]
|
|
|
|
|
|
|
|
recipient_ids = set(self_recipient_ids) & set(narrow_recipient_ids)
|
|
|
|
cond = column("recipient_id").in_(recipient_ids)
|
|
|
|
return query.where(maybe_negate(cond))
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def by_search(self, query: Query, operand: str, maybe_negate: ConditionTransform) -> Query:
|
2016-04-24 17:08:51 +02:00
|
|
|
if settings.USING_PGROONGA:
|
|
|
|
return self._by_search_pgroonga(query, operand, maybe_negate)
|
|
|
|
else:
|
|
|
|
return self._by_search_tsearch(query, operand, maybe_negate)
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def _by_search_pgroonga(self, query: Query, operand: str,
|
|
|
|
maybe_negate: ConditionTransform) -> Query:
|
2018-05-31 04:53:47 +02:00
|
|
|
match_positions_character = func.pgroonga_match_positions_character
|
|
|
|
query_extract_keywords = func.pgroonga_query_extract_keywords
|
2018-05-19 05:39:13 +02:00
|
|
|
operand_escaped = func.escape_html(operand)
|
|
|
|
keywords = query_extract_keywords(operand_escaped)
|
2017-03-31 03:03:55 +02:00
|
|
|
query = query.column(match_positions_character(column("rendered_content"),
|
|
|
|
keywords).label("content_matches"))
|
2018-11-09 17:06:00 +01:00
|
|
|
query = query.column(match_positions_character(func.escape_html(topic_column_sa()),
|
2018-11-09 16:32:05 +01:00
|
|
|
keywords).label("topic_matches"))
|
2018-05-31 04:53:47 +02:00
|
|
|
condition = column("search_pgroonga").op("&@~")(operand_escaped)
|
2016-04-24 17:08:51 +02:00
|
|
|
return query.where(maybe_negate(condition))
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def _by_search_tsearch(self, query: Query, operand: str,
|
|
|
|
maybe_negate: ConditionTransform) -> Query:
|
2013-12-10 23:32:29 +01:00
|
|
|
tsquery = func.plainto_tsquery(literal("zulip.english_us_search"), literal(operand))
|
|
|
|
query = query.column(ts_locs_array(literal("zulip.english_us_search"),
|
|
|
|
column("rendered_content"),
|
|
|
|
tsquery).label("content_matches"))
|
2018-11-09 17:12:27 +01:00
|
|
|
# We HTML-escape the topic in Postgres to avoid doing a server round-trip
|
2013-12-10 23:32:29 +01:00
|
|
|
query = query.column(ts_locs_array(literal("zulip.english_us_search"),
|
2018-11-09 17:06:00 +01:00
|
|
|
func.escape_html(topic_column_sa()),
|
2018-11-09 16:32:05 +01:00
|
|
|
tsquery).label("topic_matches"))
|
2013-12-02 20:29:57 +01:00
|
|
|
|
|
|
|
# Do quoted string matching. We really want phrase
|
|
|
|
# search here so we can ignore punctuation and do
|
|
|
|
# stemming, but there isn't a standard phrase search
|
|
|
|
# mechanism in Postgres
|
2018-07-02 00:05:24 +02:00
|
|
|
for term in re.findall(r'"[^"]+"|\S+', operand):
|
2013-12-02 20:29:57 +01:00
|
|
|
if term[0] == '"' and term[-1] == '"':
|
|
|
|
term = term[1:-1]
|
2013-12-10 23:32:29 +01:00
|
|
|
term = '%' + connection.ops.prep_for_like_query(term) + '%'
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = or_(column("content").ilike(term),
|
2018-11-09 17:06:00 +01:00
|
|
|
topic_column_sa().ilike(term))
|
2014-02-12 19:09:11 +01:00
|
|
|
query = query.where(maybe_negate(cond))
|
2013-12-02 20:29:57 +01:00
|
|
|
|
2014-02-11 23:33:24 +01:00
|
|
|
cond = column("search_tsvector").op("@@")(tsquery)
|
2014-02-12 19:09:11 +01:00
|
|
|
return query.where(maybe_negate(cond))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-04-24 03:47:28 +02:00
|
|
|
def highlight_string(text: str, locs: Iterable[Tuple[int, int]]) -> str:
|
2017-11-04 05:23:22 +01:00
|
|
|
highlight_start = '<span class="highlight">'
|
|
|
|
highlight_stop = '</span>'
|
2013-11-26 00:41:24 +01:00
|
|
|
pos = 0
|
2017-10-31 19:03:12 +01:00
|
|
|
result = ''
|
2017-04-06 15:59:56 +02:00
|
|
|
in_tag = False
|
2017-10-31 19:03:12 +01:00
|
|
|
|
2013-11-26 00:41:24 +01:00
|
|
|
for loc in locs:
|
|
|
|
(offset, length) = loc
|
2017-10-31 19:03:12 +01:00
|
|
|
|
|
|
|
prefix_start = pos
|
|
|
|
prefix_end = offset
|
|
|
|
match_start = offset
|
|
|
|
match_end = offset + length
|
|
|
|
|
2019-08-28 11:06:38 +02:00
|
|
|
prefix = text[prefix_start:prefix_end]
|
|
|
|
match = text[match_start:match_end]
|
2017-10-31 19:03:12 +01:00
|
|
|
|
|
|
|
for character in (prefix + match):
|
|
|
|
if character == '<':
|
2017-04-06 15:59:56 +02:00
|
|
|
in_tag = True
|
2017-10-31 19:03:12 +01:00
|
|
|
elif character == '>':
|
2017-04-06 15:59:56 +02:00
|
|
|
in_tag = False
|
|
|
|
if in_tag:
|
2017-10-31 19:03:12 +01:00
|
|
|
result += prefix
|
|
|
|
result += match
|
2017-04-06 15:59:56 +02:00
|
|
|
else:
|
2017-10-31 19:03:12 +01:00
|
|
|
result += prefix
|
2017-04-06 15:59:56 +02:00
|
|
|
result += highlight_start
|
2017-10-31 19:03:12 +01:00
|
|
|
result += match
|
2017-04-06 15:59:56 +02:00
|
|
|
result += highlight_stop
|
2017-10-31 19:03:12 +01:00
|
|
|
pos = match_end
|
|
|
|
|
2019-08-28 11:06:38 +02:00
|
|
|
result += text[pos:]
|
2016-08-25 08:00:52 +02:00
|
|
|
return result
|
2013-11-26 00:41:24 +01:00
|
|
|
|
2018-11-09 17:19:17 +01:00
|
|
|
def get_search_fields(rendered_content: str, topic_name: str, content_matches: Iterable[Tuple[int, int]],
|
2018-11-09 16:32:05 +01:00
|
|
|
topic_matches: Iterable[Tuple[int, int]]) -> Dict[str, str]:
|
2018-11-09 17:25:57 +01:00
|
|
|
return {
|
|
|
|
'match_content': highlight_string(rendered_content, content_matches),
|
|
|
|
MATCH_TOPIC: highlight_string(escape_html(topic_name), topic_matches),
|
|
|
|
}
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-08-11 19:07:34 +02:00
|
|
|
def narrow_parameter(json: str) -> OptionalNarrowListT:
|
2016-07-30 01:27:56 +02:00
|
|
|
|
2014-02-10 21:45:53 +01:00
|
|
|
data = ujson.loads(json)
|
|
|
|
if not isinstance(data, list):
|
|
|
|
raise ValueError("argument is not a list")
|
2017-03-19 01:46:35 +01:00
|
|
|
if len(data) == 0:
|
|
|
|
# The "empty narrow" should be None, and not []
|
|
|
|
return None
|
2014-02-10 21:45:53 +01:00
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def convert_term(elem: Union[Dict[str, Any], List[str]]) -> Dict[str, Any]:
|
2016-09-12 02:09:24 +02:00
|
|
|
|
2014-02-11 00:31:26 +01:00
|
|
|
# We have to support a legacy tuple format.
|
|
|
|
if isinstance(elem, list):
|
2019-07-29 00:23:24 +02:00
|
|
|
if (len(elem) != 2 or any(not isinstance(x, str) for x in elem)):
|
2014-02-11 00:31:26 +01:00
|
|
|
raise ValueError("element is not a string pair")
|
|
|
|
return dict(operator=elem[0], operand=elem[1])
|
|
|
|
|
|
|
|
if isinstance(elem, dict):
|
2019-07-11 18:54:28 +02:00
|
|
|
# Make sure to sync this list to frontend also when adding a new operator.
|
|
|
|
# that supports user IDs. Relevant code is located in static/js/message_fetch.js
|
2019-08-10 18:14:22 +02:00
|
|
|
# in handle_operators_supporting_id_based_api function where you will need to update
|
2019-08-10 18:10:29 +02:00
|
|
|
# operators_supporting_id, or operators_supporting_ids array.
|
2019-08-07 17:32:19 +02:00
|
|
|
operators_supporting_id = ['sender', 'group-pm-with', 'stream']
|
2019-08-10 18:10:29 +02:00
|
|
|
operators_supporting_ids = ['pm-with']
|
2020-04-11 17:32:32 +02:00
|
|
|
operators_non_empty_operand = {'search'}
|
2019-07-13 01:48:04 +02:00
|
|
|
|
|
|
|
operator = elem.get('operator', '')
|
2019-08-10 18:10:29 +02:00
|
|
|
if operator in operators_supporting_id:
|
2019-07-13 01:48:04 +02:00
|
|
|
operand_validator = check_string_or_int
|
2019-08-10 18:10:29 +02:00
|
|
|
elif operator in operators_supporting_ids:
|
2019-06-08 23:21:01 +02:00
|
|
|
operand_validator = check_string_or_int_list
|
2020-04-11 17:32:32 +02:00
|
|
|
elif operator in operators_non_empty_operand:
|
|
|
|
operand_validator = check_required_string
|
2019-06-08 23:21:01 +02:00
|
|
|
else:
|
|
|
|
operand_validator = check_string
|
|
|
|
|
2014-02-11 00:31:26 +01:00
|
|
|
validator = check_dict([
|
|
|
|
('operator', check_string),
|
2019-06-08 23:21:01 +02:00
|
|
|
('operand', operand_validator),
|
2014-02-11 00:31:26 +01:00
|
|
|
])
|
|
|
|
|
|
|
|
error = validator('elem', elem)
|
|
|
|
if error:
|
|
|
|
raise JsonableError(error)
|
|
|
|
|
|
|
|
# whitelist the fields we care about for now
|
2014-02-11 21:36:59 +01:00
|
|
|
return dict(
|
|
|
|
operator=elem['operator'],
|
|
|
|
operand=elem['operand'],
|
|
|
|
negated=elem.get('negated', False),
|
|
|
|
)
|
2014-02-11 00:31:26 +01:00
|
|
|
|
|
|
|
raise ValueError("element is not a dictionary")
|
2014-02-10 21:45:53 +01:00
|
|
|
|
2015-11-01 17:14:53 +01:00
|
|
|
return list(map(convert_term, data))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2019-08-11 19:07:34 +02:00
|
|
|
def ok_to_include_history(narrow: OptionalNarrowListT, user_profile: UserProfile) -> bool:
|
2014-02-13 16:24:06 +01:00
|
|
|
# There are occasions where we need to find Message rows that
|
|
|
|
# have no corresponding UserMessage row, because the user is
|
|
|
|
# reading a public stream that might include messages that
|
|
|
|
# were sent while the user was not subscribed, but which they are
|
|
|
|
# allowed to see. We have to be very careful about constructing
|
|
|
|
# queries in those situations, so this function should return True
|
|
|
|
# only if we are 100% sure that we're gonna add a clause to the
|
|
|
|
# query that narrows to a particular public stream on the user's realm.
|
|
|
|
# If we screw this up, then we can get into a nasty situation of
|
|
|
|
# polluting our narrow results with messages from other realms.
|
2013-12-12 18:36:32 +01:00
|
|
|
include_history = False
|
|
|
|
if narrow is not None:
|
2014-02-10 21:45:53 +01:00
|
|
|
for term in narrow:
|
2014-02-13 18:53:51 +01:00
|
|
|
if term['operator'] == "stream" and not term.get('negated', False):
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
operand: Union[str, int] = term['operand']
|
2019-08-07 17:32:19 +02:00
|
|
|
if isinstance(operand, str):
|
|
|
|
include_history = can_access_stream_history_by_name(user_profile, operand)
|
|
|
|
else:
|
|
|
|
include_history = can_access_stream_history_by_id(user_profile, operand)
|
2019-08-13 20:20:36 +02:00
|
|
|
elif (term['operator'] == "streams" and term['operand'] == "public"
|
|
|
|
and not term.get('negated', False) and user_profile.can_access_public_streams()):
|
|
|
|
include_history = True
|
2014-01-14 22:53:28 +01:00
|
|
|
# Disable historical messages if the user is narrowing on anything
|
|
|
|
# that's a property on the UserMessage table. There cannot be
|
|
|
|
# historical messages in these cases anyway.
|
2014-02-10 21:45:53 +01:00
|
|
|
for term in narrow:
|
|
|
|
if term['operator'] == "is":
|
2013-12-12 18:36:32 +01:00
|
|
|
include_history = False
|
|
|
|
|
2014-02-13 16:24:06 +01:00
|
|
|
return include_history
|
|
|
|
|
2019-08-12 18:49:06 +02:00
|
|
|
def get_stream_from_narrow_access_unchecked(narrow: OptionalNarrowListT, realm: Realm) -> Optional[Stream]:
|
2017-03-19 01:46:35 +01:00
|
|
|
if narrow is not None:
|
|
|
|
for term in narrow:
|
|
|
|
if term['operator'] == 'stream':
|
2019-08-12 18:49:06 +02:00
|
|
|
return get_stream_by_narrow_operand_access_unchecked(term['operand'], realm)
|
2014-02-25 15:41:32 +01:00
|
|
|
return None
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def exclude_muting_conditions(user_profile: UserProfile,
|
2019-08-11 19:07:34 +02:00
|
|
|
narrow: OptionalNarrowListT) -> List[Selectable]:
|
2014-02-25 22:22:35 +01:00
|
|
|
conditions = []
|
2017-09-17 20:19:12 +02:00
|
|
|
stream_id = None
|
2019-08-12 18:49:06 +02:00
|
|
|
try:
|
|
|
|
# Note: It is okay here to not check access to stream
|
|
|
|
# because we are only using the stream id to exclude data,
|
|
|
|
# not to include results.
|
|
|
|
stream = get_stream_from_narrow_access_unchecked(narrow, user_profile.realm)
|
|
|
|
if stream is not None:
|
2019-08-11 18:57:54 +02:00
|
|
|
stream_id = stream.id
|
2019-08-12 18:49:06 +02:00
|
|
|
except Stream.DoesNotExist:
|
|
|
|
pass
|
2017-09-17 20:19:12 +02:00
|
|
|
|
|
|
|
if stream_id is None:
|
2014-02-25 22:22:35 +01:00
|
|
|
rows = Subscription.objects.filter(
|
|
|
|
user_profile=user_profile,
|
|
|
|
active=True,
|
2018-08-02 23:46:05 +02:00
|
|
|
is_muted=True,
|
2014-02-25 22:22:35 +01:00
|
|
|
recipient__type=Recipient.STREAM
|
|
|
|
).values('recipient_id')
|
2015-11-01 17:14:53 +01:00
|
|
|
muted_recipient_ids = [row['recipient_id'] for row in rows]
|
2017-02-22 23:43:22 +01:00
|
|
|
if len(muted_recipient_ids) > 0:
|
|
|
|
# Only add the condition if we have muted streams to simplify/avoid warnings.
|
|
|
|
condition = not_(column("recipient_id").in_(muted_recipient_ids))
|
|
|
|
conditions.append(condition)
|
2014-02-25 22:22:35 +01:00
|
|
|
|
2017-09-17 20:19:12 +02:00
|
|
|
conditions = exclude_topic_mutes(conditions, user_profile, stream_id)
|
2014-02-24 23:00:58 +01:00
|
|
|
|
2014-02-25 22:22:35 +01:00
|
|
|
return conditions
|
2014-02-24 23:00:58 +01:00
|
|
|
|
2018-04-05 21:56:27 +02:00
|
|
|
def get_base_query_for_search(user_profile: UserProfile,
|
|
|
|
need_message: bool,
|
|
|
|
need_user_message: bool) -> Tuple[Query, ColumnElement]:
|
|
|
|
if need_message and need_user_message:
|
|
|
|
query = select([column("message_id"), column("flags")],
|
|
|
|
column("user_profile_id") == literal(user_profile.id),
|
|
|
|
join(table("zerver_usermessage"), table("zerver_message"),
|
|
|
|
literal_column("zerver_usermessage.message_id") ==
|
|
|
|
literal_column("zerver_message.id")))
|
|
|
|
inner_msg_id_col = column("message_id")
|
|
|
|
return (query, inner_msg_id_col)
|
|
|
|
|
|
|
|
if need_user_message:
|
|
|
|
query = select([column("message_id"), column("flags")],
|
|
|
|
column("user_profile_id") == literal(user_profile.id),
|
|
|
|
table("zerver_usermessage"))
|
|
|
|
inner_msg_id_col = column("message_id")
|
|
|
|
return (query, inner_msg_id_col)
|
|
|
|
|
|
|
|
else:
|
|
|
|
assert(need_message)
|
|
|
|
query = select([column("id").label("message_id")],
|
|
|
|
None,
|
|
|
|
table("zerver_message"))
|
|
|
|
inner_msg_id_col = literal_column("zerver_message.id")
|
|
|
|
return (query, inner_msg_id_col)
|
|
|
|
|
2018-04-05 20:42:37 +02:00
|
|
|
def add_narrow_conditions(user_profile: UserProfile,
|
|
|
|
inner_msg_id_col: ColumnElement,
|
|
|
|
query: Query,
|
2019-08-11 19:07:34 +02:00
|
|
|
narrow: OptionalNarrowListT) -> Tuple[Query, bool]:
|
2018-04-05 21:17:21 +02:00
|
|
|
is_search = False # for now
|
2018-04-05 20:42:37 +02:00
|
|
|
|
|
|
|
if narrow is None:
|
|
|
|
return (query, is_search)
|
|
|
|
|
|
|
|
# Build the query for the narrow
|
|
|
|
builder = NarrowBuilder(user_profile, inner_msg_id_col)
|
2018-04-05 21:17:21 +02:00
|
|
|
search_operands = []
|
2018-04-05 20:42:37 +02:00
|
|
|
|
2018-04-05 21:17:21 +02:00
|
|
|
# As we loop through terms, builder does most of the work to extend
|
|
|
|
# our query, but we need to collect the search operands and handle
|
|
|
|
# them after the loop.
|
2018-04-05 20:42:37 +02:00
|
|
|
for term in narrow:
|
|
|
|
if term['operator'] == 'search':
|
2018-04-05 21:17:21 +02:00
|
|
|
search_operands.append(term['operand'])
|
2018-04-05 20:42:37 +02:00
|
|
|
else:
|
|
|
|
query = builder.add_term(query, term)
|
|
|
|
|
2018-04-05 21:17:21 +02:00
|
|
|
if search_operands:
|
|
|
|
is_search = True
|
2018-11-09 17:06:00 +01:00
|
|
|
query = query.column(topic_column_sa()).column(column("rendered_content"))
|
2018-04-05 21:17:21 +02:00
|
|
|
search_term = dict(
|
|
|
|
operator='search',
|
|
|
|
operand=' '.join(search_operands)
|
|
|
|
)
|
2018-04-05 20:42:37 +02:00
|
|
|
query = builder.add_term(query, search_term)
|
|
|
|
|
|
|
|
return (query, is_search)
|
|
|
|
|
2018-04-05 14:52:02 +02:00
|
|
|
def find_first_unread_anchor(sa_conn: Any,
|
|
|
|
user_profile: UserProfile,
|
2019-08-11 19:07:34 +02:00
|
|
|
narrow: OptionalNarrowListT) -> int:
|
2018-04-05 22:17:50 +02:00
|
|
|
# We always need UserMessage in our query, because it has the unread
|
|
|
|
# flag for the user.
|
|
|
|
need_user_message = True
|
|
|
|
|
2018-04-06 17:57:20 +02:00
|
|
|
# Because we will need to call exclude_muting_conditions, unless
|
|
|
|
# the user hasn't muted anything, we will need to include Message
|
|
|
|
# in our query. It may be worth eventually adding an optimization
|
|
|
|
# for the case of a user who hasn't muted anything to avoid the
|
|
|
|
# join in that case, but it's low priority.
|
2018-04-05 22:17:50 +02:00
|
|
|
need_message = True
|
|
|
|
|
|
|
|
query, inner_msg_id_col = get_base_query_for_search(
|
|
|
|
user_profile=user_profile,
|
|
|
|
need_message=need_message,
|
|
|
|
need_user_message=need_user_message,
|
|
|
|
)
|
|
|
|
|
|
|
|
query, is_search = add_narrow_conditions(
|
|
|
|
user_profile=user_profile,
|
|
|
|
inner_msg_id_col=inner_msg_id_col,
|
|
|
|
query=query,
|
|
|
|
narrow=narrow,
|
|
|
|
)
|
|
|
|
|
2018-04-05 14:52:02 +02:00
|
|
|
condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0
|
|
|
|
|
|
|
|
# We exclude messages on muted topics when finding the first unread
|
|
|
|
# message in this narrow
|
|
|
|
muting_conditions = exclude_muting_conditions(user_profile, narrow)
|
|
|
|
if muting_conditions:
|
|
|
|
condition = and_(condition, *muting_conditions)
|
|
|
|
|
|
|
|
first_unread_query = query.where(condition)
|
|
|
|
first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
|
|
|
|
first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
|
|
|
|
if len(first_unread_result) > 0:
|
|
|
|
anchor = first_unread_result[0][0]
|
|
|
|
else:
|
|
|
|
anchor = LARGER_THAN_MAX_MESSAGE_ID
|
|
|
|
|
|
|
|
return anchor
|
|
|
|
|
2018-06-02 13:59:02 +02:00
|
|
|
@has_request_variables
|
|
|
|
def zcommand_backend(request: HttpRequest, user_profile: UserProfile,
|
|
|
|
command: str=REQ('command')) -> HttpResponse:
|
2018-06-15 20:56:36 +02:00
|
|
|
return json_success(process_zcommands(command, user_profile))
|
2018-06-02 13:59:02 +02:00
|
|
|
|
2020-01-29 03:29:15 +01:00
|
|
|
def parse_anchor_value(anchor_val: Optional[str],
|
|
|
|
use_first_unread_anchor: bool) -> Optional[int]:
|
|
|
|
"""Given the anchor and use_first_unread_anchor parameters passed by
|
|
|
|
the client, computes what anchor value the client requested,
|
|
|
|
handling backwards-compatibility and the various string-valued
|
|
|
|
fields. We encode use_first_unread_anchor as anchor=None.
|
|
|
|
"""
|
|
|
|
if use_first_unread_anchor:
|
|
|
|
# Backwards-compatibility: Before we added support for the
|
|
|
|
# special string-typed anchor values, clients would pass
|
|
|
|
# anchor=None and use_first_unread_anchor=True to indicate
|
|
|
|
# what is now expressed as anchor="first_unread".
|
2020-01-28 06:37:25 +01:00
|
|
|
return None
|
2020-01-29 03:29:15 +01:00
|
|
|
if anchor_val is None:
|
|
|
|
# Throw an exception if neither an anchor argument not
|
|
|
|
# use_first_unread_anchor was specified.
|
|
|
|
raise JsonableError(_("Missing 'anchor' argument."))
|
2020-01-28 06:37:25 +01:00
|
|
|
if anchor_val == "oldest":
|
|
|
|
return 0
|
|
|
|
if anchor_val == "newest":
|
|
|
|
return LARGER_THAN_MAX_MESSAGE_ID
|
2020-01-29 03:29:15 +01:00
|
|
|
if anchor_val == "first_unread":
|
|
|
|
return None
|
2020-01-28 06:37:25 +01:00
|
|
|
try:
|
|
|
|
# We don't use `.isnumeric()` to support negative numbers for
|
|
|
|
# anchor. We don't recommend it in the API (if you want the
|
|
|
|
# very first message, use 0 or 1), but it used to be supported
|
|
|
|
# and was used by the webapp, so we need to continue
|
|
|
|
# supporting it for backwards-compatibility
|
|
|
|
anchor = int(anchor_val)
|
|
|
|
if anchor < 0:
|
|
|
|
return 0
|
|
|
|
return anchor
|
|
|
|
except ValueError:
|
|
|
|
raise JsonableError(_("Invalid anchor"))
|
|
|
|
|
2014-02-13 16:24:06 +01:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def get_messages_backend(request: HttpRequest, user_profile: UserProfile,
|
2020-01-28 06:37:25 +01:00
|
|
|
anchor_val: Optional[str]=REQ(
|
|
|
|
'anchor', str_validator=check_string, default=None),
|
2017-12-03 18:46:27 +01:00
|
|
|
num_before: int=REQ(converter=to_non_negative_int),
|
|
|
|
num_after: int=REQ(converter=to_non_negative_int),
|
2019-08-11 19:07:34 +02:00
|
|
|
narrow: OptionalNarrowListT=REQ('narrow', converter=narrow_parameter, default=None),
|
2020-01-29 03:29:15 +01:00
|
|
|
use_first_unread_anchor_val: bool=REQ('use_first_unread_anchor',
|
|
|
|
validator=check_bool, default=False),
|
2017-12-03 18:46:27 +01:00
|
|
|
client_gravatar: bool=REQ(validator=check_bool, default=False),
|
|
|
|
apply_markdown: bool=REQ(validator=check_bool, default=True)) -> HttpResponse:
|
2020-01-29 03:29:15 +01:00
|
|
|
anchor = parse_anchor_value(anchor_val, use_first_unread_anchor_val)
|
2018-09-09 14:54:52 +02:00
|
|
|
if num_before + num_after > MAX_MESSAGES_PER_FETCH:
|
2019-04-20 03:49:03 +02:00
|
|
|
return json_error(_("Too many messages requested (maximum %s).")
|
|
|
|
% (MAX_MESSAGES_PER_FETCH,))
|
2014-02-13 16:24:06 +01:00
|
|
|
|
2020-03-07 01:15:34 +01:00
|
|
|
if user_profile.realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
|
2019-02-05 07:12:37 +01:00
|
|
|
# If email addresses are only available to administrators,
|
|
|
|
# clients cannot compute gravatars, so we force-set it to false.
|
|
|
|
client_gravatar = False
|
|
|
|
|
|
|
|
include_history = ok_to_include_history(narrow, user_profile)
|
2018-04-05 22:32:30 +02:00
|
|
|
if include_history:
|
2017-06-30 02:24:05 +02:00
|
|
|
# The initial query in this case doesn't use `zerver_usermessage`,
|
|
|
|
# and isn't yet limited to messages the user is entitled to see!
|
|
|
|
#
|
|
|
|
# This is OK only because we've made sure this is a narrow that
|
|
|
|
# will cause us to limit the query appropriately later.
|
|
|
|
# See `ok_to_include_history` for details.
|
2018-04-05 21:56:27 +02:00
|
|
|
need_message = True
|
|
|
|
need_user_message = False
|
2018-04-05 22:32:30 +02:00
|
|
|
elif narrow is None:
|
2018-04-05 21:56:27 +02:00
|
|
|
# We need to limit to messages the user has received, but we don't actually
|
|
|
|
# need any fields from Message
|
|
|
|
need_message = False
|
|
|
|
need_user_message = True
|
2013-12-12 18:36:32 +01:00
|
|
|
else:
|
2018-04-05 21:56:27 +02:00
|
|
|
need_message = True
|
|
|
|
need_user_message = True
|
|
|
|
|
|
|
|
query, inner_msg_id_col = get_base_query_for_search(
|
|
|
|
user_profile=user_profile,
|
|
|
|
need_message=need_message,
|
|
|
|
need_user_message=need_user_message,
|
|
|
|
)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-04-05 20:42:37 +02:00
|
|
|
query, is_search = add_narrow_conditions(
|
|
|
|
user_profile=user_profile,
|
|
|
|
inner_msg_id_col=inner_msg_id_col,
|
|
|
|
query=query,
|
|
|
|
narrow=narrow,
|
|
|
|
)
|
2013-12-12 22:50:49 +01:00
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
if narrow is not None:
|
2013-12-12 22:50:49 +01:00
|
|
|
# Add some metadata to our logging data for narrows
|
2013-12-12 18:36:32 +01:00
|
|
|
verbose_operators = []
|
2014-02-10 21:45:53 +01:00
|
|
|
for term in narrow:
|
|
|
|
if term['operator'] == "is":
|
|
|
|
verbose_operators.append("is:" + term['operand'])
|
2013-12-12 18:36:32 +01:00
|
|
|
else:
|
2014-02-10 21:45:53 +01:00
|
|
|
verbose_operators.append(term['operator'])
|
2013-12-12 18:36:32 +01:00
|
|
|
request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)
|
|
|
|
|
2014-01-31 20:19:12 +01:00
|
|
|
sa_conn = get_sqlalchemy_connection()
|
2018-03-13 01:07:12 +01:00
|
|
|
|
2020-01-29 03:29:15 +01:00
|
|
|
if anchor is None:
|
|
|
|
# The use_first_unread_anchor code path
|
2018-04-05 13:31:22 +02:00
|
|
|
anchor = find_first_unread_anchor(
|
|
|
|
sa_conn,
|
|
|
|
user_profile,
|
|
|
|
narrow,
|
|
|
|
)
|
|
|
|
|
2019-07-29 00:23:24 +02:00
|
|
|
# Hint to mypy that anchor is now unconditionally an integer,
|
|
|
|
# since its inference engine can't figure that out.
|
|
|
|
assert anchor is not None
|
2018-03-13 01:07:12 +01:00
|
|
|
anchored_to_left = (anchor == 0)
|
|
|
|
|
2019-03-30 06:59:19 +01:00
|
|
|
# Set value that will be used to short circuit the after_query
|
2018-04-05 13:31:22 +02:00
|
|
|
# altogether and avoid needless conditions in the before_query.
|
2020-01-28 06:31:29 +01:00
|
|
|
anchored_to_right = (anchor >= LARGER_THAN_MAX_MESSAGE_ID)
|
2018-04-05 13:31:22 +02:00
|
|
|
if anchored_to_right:
|
2019-03-30 06:59:19 +01:00
|
|
|
num_after = 0
|
2018-04-05 13:31:22 +02:00
|
|
|
|
2018-09-19 14:23:02 +02:00
|
|
|
first_visible_message_id = get_first_visible_message_id(user_profile.realm)
|
2018-03-13 14:29:39 +01:00
|
|
|
query = limit_query_to_range(
|
|
|
|
query=query,
|
|
|
|
num_before=num_before,
|
|
|
|
num_after=num_after,
|
|
|
|
anchor=anchor,
|
|
|
|
anchored_to_left=anchored_to_left,
|
|
|
|
anchored_to_right=anchored_to_right,
|
|
|
|
id_col=inner_msg_id_col,
|
2018-09-19 14:23:02 +02:00
|
|
|
first_visible_message_id=first_visible_message_id,
|
2018-03-13 14:29:39 +01:00
|
|
|
)
|
2017-02-22 23:39:40 +01:00
|
|
|
|
2013-12-12 22:50:49 +01:00
|
|
|
main_query = alias(query)
|
|
|
|
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
|
2014-01-08 18:01:35 +01:00
|
|
|
# This is a hack to tag the query we use for testing
|
2017-03-24 07:51:46 +01:00
|
|
|
query = query.prefix_with("/* get_messages */")
|
2018-03-14 12:38:04 +01:00
|
|
|
rows = list(sa_conn.execute(query).fetchall())
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-03-15 11:21:36 +01:00
|
|
|
query_info = post_process_limited_query(
|
|
|
|
rows=rows,
|
|
|
|
num_before=num_before,
|
|
|
|
num_after=num_after,
|
|
|
|
anchor=anchor,
|
|
|
|
anchored_to_left=anchored_to_left,
|
|
|
|
anchored_to_right=anchored_to_right,
|
2018-09-19 14:23:02 +02:00
|
|
|
first_visible_message_id=first_visible_message_id,
|
2018-03-15 11:21:36 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
rows = query_info['rows']
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
# The following is a little messy, but ensures that the code paths
|
|
|
|
# are similar regardless of the value of include_history. The
|
|
|
|
# 'user_messages' dictionary maps each message to the user's
|
|
|
|
# UserMessage object for that message, which we will attach to the
|
|
|
|
# rendered message dict before returning it. We attempt to
|
2016-03-31 03:39:51 +02:00
|
|
|
# bulk-fetch rendered message dicts from remote cache using the
|
2013-12-12 18:36:32 +01:00
|
|
|
# 'messages' list.
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
message_ids: List[int] = []
|
|
|
|
user_message_flags: Dict[int, List[str]] = {}
|
2013-12-12 22:50:49 +01:00
|
|
|
if include_history:
|
2018-03-14 12:38:04 +01:00
|
|
|
message_ids = [row[0] for row in rows]
|
2013-12-10 23:32:29 +01:00
|
|
|
|
|
|
|
# TODO: This could be done with an outer join instead of two queries
|
2017-11-07 16:18:42 +01:00
|
|
|
um_rows = UserMessage.objects.filter(user_profile=user_profile,
|
|
|
|
message__id__in=message_ids)
|
|
|
|
user_message_flags = {um.message_id: um.flags_list() for um in um_rows}
|
|
|
|
|
2017-11-07 17:12:27 +01:00
|
|
|
for message_id in message_ids:
|
|
|
|
if message_id not in user_message_flags:
|
2013-12-10 23:32:29 +01:00
|
|
|
user_message_flags[message_id] = ["read", "historical"]
|
2013-12-12 18:36:32 +01:00
|
|
|
else:
|
2018-03-14 12:38:04 +01:00
|
|
|
for row in rows:
|
2013-12-10 23:32:29 +01:00
|
|
|
message_id = row[0]
|
|
|
|
flags = row[1]
|
2017-11-07 18:40:39 +01:00
|
|
|
user_message_flags[message_id] = UserMessage.flags_list_for_flags(flags)
|
2013-12-10 23:32:29 +01:00
|
|
|
message_ids.append(message_id)
|
|
|
|
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
search_fields: Dict[int, Dict[str, str]] = dict()
|
2017-11-07 17:12:27 +01:00
|
|
|
if is_search:
|
2018-03-14 12:38:04 +01:00
|
|
|
for row in rows:
|
2017-11-07 17:12:27 +01:00
|
|
|
message_id = row[0]
|
2018-11-09 17:19:17 +01:00
|
|
|
(topic_name, rendered_content, content_matches, topic_matches) = row[-4:]
|
2017-11-16 22:38:04 +01:00
|
|
|
|
|
|
|
try:
|
2018-11-09 17:19:17 +01:00
|
|
|
search_fields[message_id] = get_search_fields(rendered_content, topic_name,
|
2018-11-09 16:32:05 +01:00
|
|
|
content_matches, topic_matches)
|
2017-11-16 22:38:04 +01:00
|
|
|
except UnicodeDecodeError as err: # nocoverage
|
|
|
|
# No coverage for this block since it should be
|
|
|
|
# impossible, and we plan to remove it once we've
|
|
|
|
# debugged the case that makes it happen.
|
2018-04-05 21:00:07 +02:00
|
|
|
raise Exception(str(err), message_id, narrow)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-11-07 17:36:29 +01:00
|
|
|
message_list = messages_for_ids(
|
|
|
|
message_ids=message_ids,
|
|
|
|
user_message_flags=user_message_flags,
|
|
|
|
search_fields=search_fields,
|
|
|
|
apply_markdown=apply_markdown,
|
|
|
|
client_gravatar=client_gravatar,
|
|
|
|
allow_edit_history=user_profile.realm.allow_edit_history,
|
|
|
|
)
|
2017-10-10 09:22:21 +02:00
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
statsd.incr('loaded_old_messages', len(message_list))
|
2018-03-15 11:43:51 +01:00
|
|
|
|
|
|
|
ret = dict(
|
|
|
|
messages=message_list,
|
|
|
|
result='success',
|
|
|
|
msg='',
|
|
|
|
found_anchor=query_info['found_anchor'],
|
|
|
|
found_oldest=query_info['found_oldest'],
|
|
|
|
found_newest=query_info['found_newest'],
|
2018-09-19 14:23:02 +02:00
|
|
|
history_limited=query_info['history_limited'],
|
2018-03-15 11:43:51 +01:00
|
|
|
anchor=anchor,
|
|
|
|
)
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success(ret)
|
|
|
|
|
2018-03-13 14:29:39 +01:00
|
|
|
def limit_query_to_range(query: Query,
|
|
|
|
num_before: int,
|
|
|
|
num_after: int,
|
|
|
|
anchor: int,
|
|
|
|
anchored_to_left: bool,
|
|
|
|
anchored_to_right: bool,
|
2018-09-19 14:23:02 +02:00
|
|
|
id_col: ColumnElement,
|
|
|
|
first_visible_message_id: int) -> Query:
|
2018-03-13 14:29:39 +01:00
|
|
|
'''
|
|
|
|
This code is actually generic enough that we could move it to a
|
|
|
|
library, but our only caller for now is message search.
|
|
|
|
'''
|
|
|
|
need_before_query = (not anchored_to_left) and (num_before > 0)
|
|
|
|
need_after_query = (not anchored_to_right) and (num_after > 0)
|
|
|
|
|
|
|
|
need_both_sides = need_before_query and need_after_query
|
|
|
|
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
# The semantics of our flags are as follows:
|
|
|
|
#
|
|
|
|
# num_after = number of rows < anchor
|
|
|
|
# num_after = number of rows > anchor
|
|
|
|
#
|
|
|
|
# But we also want the row where id == anchor (if it exists),
|
|
|
|
# and we don't want to union up to 3 queries. So in some cases
|
|
|
|
# we do things like `after_limit = num_after + 1` to grab the
|
|
|
|
# anchor row in the "after" query.
|
|
|
|
#
|
|
|
|
# Note that in some cases, if the anchor row isn't found, we
|
|
|
|
# actually may fetch an extra row at one of the extremes.
|
2018-03-13 14:29:39 +01:00
|
|
|
if need_both_sides:
|
|
|
|
before_anchor = anchor - 1
|
2018-09-19 14:23:02 +02:00
|
|
|
after_anchor = max(anchor, first_visible_message_id)
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
before_limit = num_before
|
|
|
|
after_limit = num_after + 1
|
2018-03-13 14:29:39 +01:00
|
|
|
elif need_before_query:
|
|
|
|
before_anchor = anchor
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
before_limit = num_before
|
|
|
|
if not anchored_to_right:
|
|
|
|
before_limit += 1
|
|
|
|
elif need_after_query:
|
2018-09-19 14:23:02 +02:00
|
|
|
after_anchor = max(anchor, first_visible_message_id)
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
after_limit = num_after + 1
|
2018-03-13 14:29:39 +01:00
|
|
|
|
|
|
|
if need_before_query:
|
|
|
|
before_query = query
|
|
|
|
|
|
|
|
if not anchored_to_right:
|
|
|
|
before_query = before_query.where(id_col <= before_anchor)
|
|
|
|
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
before_query = before_query.order_by(id_col.desc())
|
|
|
|
before_query = before_query.limit(before_limit)
|
2018-03-13 14:29:39 +01:00
|
|
|
|
|
|
|
if need_after_query:
|
|
|
|
after_query = query
|
|
|
|
|
|
|
|
if not anchored_to_left:
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
after_query = after_query.where(id_col >= after_anchor)
|
2018-03-13 14:29:39 +01:00
|
|
|
|
search: Make `num_after`/`num_after` more consistent.
We now consistently set our query limits so that we get at
least `num_after` rows such that id > anchor. (Obviously, the
caveat is that if there aren't enough rows that fulfill the
query, we'll return the full set of rows, but that may be less
than `num_after`.) Likewise for `num_before`.
Before this change, we would sometimes return one too few rows
for narrow queries.
Now, we're still a bit broken, but in a more consistent way. If
we have a query that does not match the anchor row (which could
be true even for a non-narrow query), but which does match lots
of rows after the anchor, we'll return `num_after + 1` rows
on the right hand side, whether or not the query has narrow
parameters.
The off-by-one semantics here have probably been moot all along,
since our windows are approximate to begin with. If we set
num_after to 100, its just a rough performance optimization to
begin with, so it doesn't matter whether we return 99 or 101 rows,
as long as we set the anchor correctly on the subsequent query.
We will make the results more rigorous in a follow up commit.
2018-03-14 13:22:16 +01:00
|
|
|
after_query = after_query.order_by(id_col.asc())
|
|
|
|
after_query = after_query.limit(after_limit)
|
2018-03-13 14:29:39 +01:00
|
|
|
|
|
|
|
if need_both_sides:
|
|
|
|
query = union_all(before_query.self_group(), after_query.self_group())
|
|
|
|
elif need_before_query:
|
|
|
|
query = before_query
|
|
|
|
elif need_after_query:
|
|
|
|
query = after_query
|
|
|
|
else:
|
|
|
|
# If we don't have either a before_query or after_query, it's because
|
|
|
|
# some combination of num_before/num_after/anchor are zero or
|
|
|
|
# use_first_unread_anchor logic found no unread messages.
|
|
|
|
#
|
|
|
|
# The most likely reason is somebody is doing an id search, so searching
|
|
|
|
# for something like `message_id = 42` is exactly what we want. In other
|
|
|
|
# cases, which could possibly be buggy API clients, at least we will
|
|
|
|
# return at most one row here.
|
|
|
|
query = query.where(id_col == anchor)
|
|
|
|
|
|
|
|
return query
|
|
|
|
|
2018-03-15 11:20:55 +01:00
|
|
|
def post_process_limited_query(rows: List[Any],
|
|
|
|
num_before: int,
|
|
|
|
num_after: int,
|
|
|
|
anchor: int,
|
|
|
|
anchored_to_left: bool,
|
2018-09-19 14:23:02 +02:00
|
|
|
anchored_to_right: bool,
|
|
|
|
first_visible_message_id: int) -> Dict[str, Any]:
|
2018-03-15 11:20:55 +01:00
|
|
|
# Our queries may have fetched extra rows if they added
|
|
|
|
# "headroom" to the limits, but we want to truncate those
|
|
|
|
# rows.
|
|
|
|
#
|
|
|
|
# Also, in cases where we had non-zero values of num_before or
|
|
|
|
# num_after, we want to know found_oldest and found_newest, so
|
|
|
|
# that the clients will know that they got complete results.
|
|
|
|
|
2018-09-19 14:23:02 +02:00
|
|
|
if first_visible_message_id > 0:
|
|
|
|
visible_rows = [r for r in rows if r[0] >= first_visible_message_id]
|
|
|
|
else:
|
|
|
|
visible_rows = rows
|
|
|
|
|
|
|
|
rows_limited = len(visible_rows) != len(rows)
|
|
|
|
|
2018-03-15 11:20:55 +01:00
|
|
|
if anchored_to_right:
|
|
|
|
num_after = 0
|
2018-09-19 14:23:02 +02:00
|
|
|
before_rows = visible_rows[:]
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
anchor_rows: List[Any] = []
|
|
|
|
after_rows: List[Any] = []
|
2018-03-15 11:20:55 +01:00
|
|
|
else:
|
2018-09-19 14:23:02 +02:00
|
|
|
before_rows = [r for r in visible_rows if r[0] < anchor]
|
|
|
|
anchor_rows = [r for r in visible_rows if r[0] == anchor]
|
|
|
|
after_rows = [r for r in visible_rows if r[0] > anchor]
|
2018-03-15 11:20:55 +01:00
|
|
|
|
|
|
|
if num_before:
|
|
|
|
before_rows = before_rows[-1 * num_before:]
|
|
|
|
|
|
|
|
if num_after:
|
|
|
|
after_rows = after_rows[:num_after]
|
|
|
|
|
2018-09-19 14:23:02 +02:00
|
|
|
visible_rows = before_rows + anchor_rows + after_rows
|
2018-03-15 11:20:55 +01:00
|
|
|
|
|
|
|
found_anchor = len(anchor_rows) == 1
|
|
|
|
found_oldest = anchored_to_left or (len(before_rows) < num_before)
|
|
|
|
found_newest = anchored_to_right or (len(after_rows) < num_after)
|
2018-09-19 14:23:02 +02:00
|
|
|
# BUG: history_limited is incorrect False in the event that we had
|
|
|
|
# to bump `anchor` up due to first_visible_message_id, and there
|
|
|
|
# were actually older messages. This may be a rare event in the
|
|
|
|
# context where history_limited is relevant, because it can only
|
|
|
|
# happen in one-sided queries with no num_before (see tests tagged
|
|
|
|
# BUG in PostProcessTest for examples), and we don't generally do
|
|
|
|
# those from the UI, so this might be OK for now.
|
|
|
|
#
|
|
|
|
# The correct fix for this probably involves e.g. making a
|
|
|
|
# `before_query` when we increase `anchor` just to confirm whether
|
|
|
|
# messages were hidden.
|
|
|
|
history_limited = rows_limited and found_oldest
|
2018-03-15 11:20:55 +01:00
|
|
|
|
|
|
|
return dict(
|
2018-09-19 14:23:02 +02:00
|
|
|
rows=visible_rows,
|
2018-03-15 11:20:55 +01:00
|
|
|
found_anchor=found_anchor,
|
|
|
|
found_newest=found_newest,
|
|
|
|
found_oldest=found_oldest,
|
2018-09-19 14:23:02 +02:00
|
|
|
history_limited=history_limited,
|
2018-03-15 11:20:55 +01:00
|
|
|
)
|
|
|
|
|
2020-05-07 09:48:43 +02:00
|
|
|
def get_latest_update_message_flag_activity(user_profile: UserProfile) -> Optional[UserActivity]:
|
|
|
|
return UserActivity.objects.filter(user_profile=user_profile,
|
2020-05-14 20:23:17 +02:00
|
|
|
query='update_message_flags').order_by("last_visit").last()
|
2020-05-07 09:48:43 +02:00
|
|
|
|
|
|
|
# NOTE: If this function name is changed, add the new name to the
|
|
|
|
# query in get_latest_update_message_flag_activity
|
2013-12-12 18:36:32 +01:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def update_message_flags(request: HttpRequest, user_profile: UserProfile,
|
|
|
|
messages: List[int]=REQ(validator=check_list(check_int)),
|
2018-04-24 03:47:28 +02:00
|
|
|
operation: str=REQ('op'), flag: str=REQ()) -> HttpResponse:
|
2017-08-04 23:06:23 +02:00
|
|
|
|
2018-03-14 00:05:55 +01:00
|
|
|
count = do_update_message_flags(user_profile, request.client, operation, flag, messages)
|
2016-07-13 03:16:42 +02:00
|
|
|
|
2017-08-06 15:00:08 +02:00
|
|
|
target_count_str = str(len(messages))
|
|
|
|
log_data_str = "[%s %s/%s] actually %s" % (operation, flag, target_count_str, count)
|
2016-07-13 03:16:42 +02:00
|
|
|
request._log_data["extra"] = log_data_str
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success({'result': 'success',
|
|
|
|
'messages': messages,
|
|
|
|
'msg': ''})
|
|
|
|
|
2017-08-04 20:26:38 +02:00
|
|
|
@has_request_variables
|
2017-11-27 09:28:57 +01:00
|
|
|
def mark_all_as_read(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
|
2018-03-14 00:04:16 +01:00
|
|
|
count = do_mark_all_as_read(user_profile, request.client)
|
2017-08-04 20:26:38 +02:00
|
|
|
|
|
|
|
log_data_str = "[%s updated]" % (count,)
|
|
|
|
request._log_data["extra"] = log_data_str
|
|
|
|
|
|
|
|
return json_success({'result': 'success',
|
|
|
|
'msg': ''})
|
|
|
|
|
2017-08-08 16:11:45 +02:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def mark_stream_as_read(request: HttpRequest,
|
|
|
|
user_profile: UserProfile,
|
|
|
|
stream_id: int=REQ(validator=check_int)) -> HttpResponse:
|
2017-08-15 19:34:15 +02:00
|
|
|
stream, recipient, sub = access_stream_by_id(user_profile, stream_id)
|
2018-03-14 00:09:11 +01:00
|
|
|
count = do_mark_stream_messages_as_read(user_profile, request.client, stream)
|
2017-08-06 15:00:08 +02:00
|
|
|
|
2017-08-08 16:11:45 +02:00
|
|
|
log_data_str = "[%s updated]" % (count,)
|
|
|
|
request._log_data["extra"] = log_data_str
|
|
|
|
|
|
|
|
return json_success({'result': 'success',
|
|
|
|
'msg': ''})
|
|
|
|
|
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def mark_topic_as_read(request: HttpRequest,
|
|
|
|
user_profile: UserProfile,
|
|
|
|
stream_id: int=REQ(validator=check_int),
|
2018-04-24 03:47:28 +02:00
|
|
|
topic_name: str=REQ()) -> HttpResponse:
|
2017-08-15 19:28:32 +02:00
|
|
|
stream, recipient, sub = access_stream_by_id(user_profile, stream_id)
|
2017-08-04 23:06:23 +02:00
|
|
|
|
|
|
|
if topic_name:
|
2018-11-09 17:32:08 +01:00
|
|
|
topic_exists = user_message_exists_for_topic(
|
|
|
|
user_profile=user_profile,
|
|
|
|
recipient=recipient,
|
|
|
|
topic_name=topic_name,
|
|
|
|
)
|
|
|
|
|
2017-08-04 23:06:23 +02:00
|
|
|
if not topic_exists:
|
|
|
|
raise JsonableError(_('No such topic \'%s\'') % (topic_name,))
|
|
|
|
|
2018-03-14 00:09:11 +01:00
|
|
|
count = do_mark_stream_messages_as_read(user_profile, request.client, stream, topic_name)
|
2017-08-04 23:06:23 +02:00
|
|
|
|
2017-08-08 16:11:45 +02:00
|
|
|
log_data_str = "[%s updated]" % (count,)
|
2017-08-04 23:06:23 +02:00
|
|
|
request._log_data["extra"] = log_data_str
|
|
|
|
|
|
|
|
return json_success({'result': 'success',
|
|
|
|
'msg': ''})
|
|
|
|
|
2019-03-30 07:11:46 +01:00
|
|
|
class InvalidMirrorInput(Exception):
|
|
|
|
pass
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def create_mirrored_message_users(request: HttpRequest, user_profile: UserProfile,
|
2019-03-30 07:11:46 +01:00
|
|
|
recipients: Iterable[str]) -> UserProfile:
|
2013-12-12 18:36:32 +01:00
|
|
|
if "sender" not in request.POST:
|
2019-03-30 07:11:46 +01:00
|
|
|
raise InvalidMirrorInput("No sender")
|
2013-12-12 18:36:32 +01:00
|
|
|
|
|
|
|
sender_email = request.POST["sender"].strip().lower()
|
2020-04-09 21:51:58 +02:00
|
|
|
referenced_users = {sender_email}
|
2013-12-12 18:36:32 +01:00
|
|
|
if request.POST['type'] == 'private':
|
|
|
|
for email in recipients:
|
|
|
|
referenced_users.add(email.lower())
|
|
|
|
|
|
|
|
if request.client.name == "zephyr_mirror":
|
2014-03-05 17:54:37 +01:00
|
|
|
user_check = same_realm_zephyr_user
|
2013-12-12 18:36:32 +01:00
|
|
|
fullname_function = compute_mit_user_fullname
|
|
|
|
elif request.client.name == "irc_mirror":
|
|
|
|
user_check = same_realm_irc_user
|
|
|
|
fullname_function = compute_irc_user_fullname
|
2014-02-28 20:53:54 +01:00
|
|
|
elif request.client.name in ("jabber_mirror", "JabberMirror"):
|
2014-03-05 17:51:35 +01:00
|
|
|
user_check = same_realm_jabber_user
|
2013-12-12 18:36:32 +01:00
|
|
|
fullname_function = compute_jabber_user_fullname
|
|
|
|
else:
|
2019-03-30 07:11:46 +01:00
|
|
|
raise InvalidMirrorInput("Unrecognized mirroring client")
|
2013-12-12 18:36:32 +01:00
|
|
|
|
|
|
|
for email in referenced_users:
|
|
|
|
# Check that all referenced users are in our realm:
|
|
|
|
if not user_check(user_profile, email):
|
2019-03-30 07:11:46 +01:00
|
|
|
raise InvalidMirrorInput("At least one user cannot be mirrored")
|
2013-12-12 18:36:32 +01:00
|
|
|
|
|
|
|
# Create users for the referenced users, if needed.
|
|
|
|
for email in referenced_users:
|
|
|
|
create_mirror_user_if_needed(user_profile.realm, email, fullname_function)
|
|
|
|
|
2017-07-18 23:31:27 +02:00
|
|
|
sender = get_user_including_cross_realm(sender_email, user_profile.realm)
|
2019-03-30 07:11:46 +01:00
|
|
|
return sender
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-04-24 03:47:28 +02:00
|
|
|
def same_realm_zephyr_user(user_profile: UserProfile, email: str) -> bool:
|
2016-07-27 01:45:29 +02:00
|
|
|
#
|
|
|
|
# Are the sender and recipient both addresses in the same Zephyr
|
|
|
|
# mirroring realm? We have to handle this specially, inferring
|
|
|
|
# the domain from the e-mail address, because the recipient may
|
|
|
|
# not existing in Zulip and we may need to make a stub Zephyr
|
|
|
|
# mirroring user on the fly.
|
2013-12-12 18:36:32 +01:00
|
|
|
try:
|
|
|
|
validators.validate_email(email)
|
|
|
|
except ValidationError:
|
|
|
|
return False
|
|
|
|
|
2016-11-11 21:13:30 +01:00
|
|
|
domain = email_to_domain(email)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-03-31 16:20:07 +02:00
|
|
|
# Assumes allow_subdomains=False for all RealmDomain's corresponding to
|
2017-01-21 08:19:03 +01:00
|
|
|
# these realms.
|
2016-11-09 02:40:54 +01:00
|
|
|
return user_profile.realm.is_zephyr_mirror_realm and \
|
2017-03-31 16:20:07 +02:00
|
|
|
RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-04-24 03:47:28 +02:00
|
|
|
def same_realm_irc_user(user_profile: UserProfile, email: str) -> bool:
|
2013-12-12 18:36:32 +01:00
|
|
|
# Check whether the target email address is an IRC user in the
|
|
|
|
# same realm as user_profile, i.e. if the domain were example.com,
|
|
|
|
# the IRC user would need to be username@irc.example.com
|
|
|
|
try:
|
|
|
|
validators.validate_email(email)
|
|
|
|
except ValidationError:
|
|
|
|
return False
|
|
|
|
|
2016-11-11 21:13:30 +01:00
|
|
|
domain = email_to_domain(email).replace("irc.", "")
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2017-03-31 16:20:07 +02:00
|
|
|
# Assumes allow_subdomains=False for all RealmDomain's corresponding to
|
2017-01-21 08:19:03 +01:00
|
|
|
# these realms.
|
2017-03-31 16:20:07 +02:00
|
|
|
return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-04-24 03:47:28 +02:00
|
|
|
def same_realm_jabber_user(user_profile: UserProfile, email: str) -> bool:
|
2013-12-12 18:36:32 +01:00
|
|
|
try:
|
|
|
|
validators.validate_email(email)
|
|
|
|
except ValidationError:
|
|
|
|
return False
|
|
|
|
|
2016-11-11 05:50:34 +01:00
|
|
|
# If your Jabber users have a different email domain than the
|
|
|
|
# Zulip users, this is where you would do any translation.
|
2016-11-11 21:13:30 +01:00
|
|
|
domain = email_to_domain(email)
|
2016-11-09 02:40:54 +01:00
|
|
|
|
2017-03-31 16:20:07 +02:00
|
|
|
# Assumes allow_subdomains=False for all RealmDomain's corresponding to
|
2017-01-21 08:19:03 +01:00
|
|
|
# these realms.
|
2017-03-31 16:20:07 +02:00
|
|
|
return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2018-01-04 18:41:34 +01:00
|
|
|
def handle_deferred_message(sender: UserProfile, client: Client,
|
2018-09-19 23:12:03 +02:00
|
|
|
message_type_name: str,
|
|
|
|
message_to: Union[Sequence[str], Sequence[int]],
|
2018-04-24 03:47:28 +02:00
|
|
|
topic_name: Optional[str],
|
|
|
|
message_content: str, delivery_type: str,
|
2019-03-30 06:59:19 +01:00
|
|
|
defer_until: str, tz_guess: Optional[str],
|
2018-01-04 18:41:34 +01:00
|
|
|
forwarder_user_profile: UserProfile,
|
|
|
|
realm: Optional[Realm]) -> HttpResponse:
|
|
|
|
deliver_at = None
|
|
|
|
local_tz = 'UTC'
|
|
|
|
if tz_guess:
|
|
|
|
local_tz = tz_guess
|
|
|
|
elif sender.timezone:
|
|
|
|
local_tz = sender.timezone
|
|
|
|
try:
|
|
|
|
deliver_at = dateparser(defer_until)
|
|
|
|
except ValueError:
|
2018-02-07 01:13:11 +01:00
|
|
|
return json_error(_("Invalid time format"))
|
2018-01-04 18:41:34 +01:00
|
|
|
|
|
|
|
deliver_at_usertz = deliver_at
|
|
|
|
if deliver_at_usertz.tzinfo is None:
|
|
|
|
user_tz = get_timezone(local_tz)
|
|
|
|
# Since mypy is not able to recognize localize and normalize as attributes of tzinfo we use ignore.
|
2020-04-22 04:13:37 +02:00
|
|
|
deliver_at_usertz = user_tz.normalize(user_tz.localize(deliver_at)) # type: ignore[attr-defined] # Reason in comment on previous line.
|
2018-01-04 18:41:34 +01:00
|
|
|
deliver_at = convert_to_UTC(deliver_at_usertz)
|
|
|
|
|
|
|
|
if deliver_at <= timezone_now():
|
2018-02-07 01:13:11 +01:00
|
|
|
return json_error(_("Time must be in the future."))
|
2018-01-04 18:41:34 +01:00
|
|
|
|
|
|
|
check_schedule_message(sender, client, message_type_name, message_to,
|
2018-01-12 12:38:45 +01:00
|
|
|
topic_name, message_content, delivery_type,
|
2018-01-04 18:41:34 +01:00
|
|
|
deliver_at, realm=realm,
|
|
|
|
forwarder_user_profile=forwarder_user_profile)
|
|
|
|
return json_success({"deliver_at": str(deliver_at_usertz)})
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def send_message_backend(request: HttpRequest, user_profile: UserProfile,
|
2018-04-24 03:47:28 +02:00
|
|
|
message_type_name: str=REQ('type'),
|
2020-02-11 17:44:41 +01:00
|
|
|
req_to: Optional[str]=REQ('to', default=None),
|
2019-11-13 08:17:49 +01:00
|
|
|
forged_str: Optional[str]=REQ("forged",
|
|
|
|
default=None,
|
|
|
|
documentation_pending=True),
|
2018-11-09 18:35:34 +01:00
|
|
|
topic_name: Optional[str]=REQ_topic(),
|
2018-04-24 03:47:28 +02:00
|
|
|
message_content: str=REQ('content'),
|
2019-07-04 08:15:10 +02:00
|
|
|
widget_content: Optional[str]=REQ(default=None,
|
|
|
|
documentation_pending=True),
|
|
|
|
realm_str: Optional[str]=REQ('realm_str', default=None,
|
|
|
|
documentation_pending=True),
|
|
|
|
local_id: Optional[str]=REQ(default=None,
|
|
|
|
documentation_pending=True),
|
|
|
|
queue_id: Optional[str]=REQ(default=None,
|
|
|
|
documentation_pending=True),
|
|
|
|
delivery_type: Optional[str]=REQ('delivery_type', default='send_now',
|
|
|
|
documentation_pending=True),
|
|
|
|
defer_until: Optional[str]=REQ('deliver_at', default=None,
|
|
|
|
documentation_pending=True),
|
|
|
|
tz_guess: Optional[str]=REQ('tz_guess', default=None,
|
|
|
|
documentation_pending=True)
|
|
|
|
) -> HttpResponse:
|
2020-02-11 17:44:41 +01:00
|
|
|
|
|
|
|
# If req_to is None, then we default to an
|
|
|
|
# empty list of recipients.
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
message_to: Union[Sequence[int], Sequence[str]] = []
|
2020-02-11 17:44:41 +01:00
|
|
|
|
|
|
|
if req_to is not None:
|
|
|
|
if message_type_name == 'stream':
|
|
|
|
stream_indicator = extract_stream_indicator(req_to)
|
|
|
|
|
|
|
|
# For legacy reasons check_send_message expects
|
|
|
|
# a list of streams, instead of a single stream.
|
|
|
|
#
|
|
|
|
# Also, mypy can't detect that a single-item
|
|
|
|
# list populated from a Union[int, str] is actually
|
|
|
|
# a Union[Sequence[int], Sequence[str]].
|
|
|
|
message_to = cast(
|
|
|
|
Union[Sequence[int], Sequence[str]],
|
|
|
|
[stream_indicator]
|
|
|
|
)
|
|
|
|
else:
|
2020-02-11 21:28:14 +01:00
|
|
|
message_to = extract_private_recipients(req_to)
|
2020-02-11 17:44:41 +01:00
|
|
|
|
2019-11-13 08:17:49 +01:00
|
|
|
# Temporary hack: We're transitioning `forged` from accepting
|
|
|
|
# `yes` to accepting `true` like all of our normal booleans.
|
|
|
|
forged = forged_str is not None and forged_str in ["yes", "true"]
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
client = request.client
|
2016-02-08 03:59:38 +01:00
|
|
|
is_super_user = request.user.is_api_super_user
|
2013-12-12 18:36:32 +01:00
|
|
|
if forged and not is_super_user:
|
2016-05-25 15:02:02 +02:00
|
|
|
return json_error(_("User not authorized for this query"))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
|
|
|
realm = None
|
2017-01-03 06:57:03 +01:00
|
|
|
if realm_str and realm_str != user_profile.realm.string_id:
|
2013-12-12 18:36:32 +01:00
|
|
|
if not is_super_user:
|
|
|
|
# The email gateway bot needs to be able to send messages in
|
|
|
|
# any realm.
|
2016-05-25 15:02:02 +02:00
|
|
|
return json_error(_("User not authorized for this query"))
|
2019-05-04 04:47:44 +02:00
|
|
|
try:
|
|
|
|
realm = get_realm(realm_str)
|
|
|
|
except Realm.DoesNotExist:
|
2018-03-08 01:53:24 +01:00
|
|
|
return json_error(_("Unknown organization '%s'") % (realm_str,))
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2014-02-28 20:53:54 +01:00
|
|
|
if client.name in ["zephyr_mirror", "irc_mirror", "jabber_mirror", "JabberMirror"]:
|
2013-12-12 18:36:32 +01:00
|
|
|
# Here's how security works for mirroring:
|
|
|
|
#
|
|
|
|
# For private messages, the message must be (1) both sent and
|
|
|
|
# received exclusively by users in your realm, and (2)
|
|
|
|
# received by the forwarding user.
|
|
|
|
#
|
|
|
|
# For stream messages, the message must be (1) being forwarded
|
|
|
|
# by an API superuser for your realm and (2) being sent to a
|
2017-11-25 09:52:04 +01:00
|
|
|
# mirrored stream.
|
2013-12-12 18:36:32 +01:00
|
|
|
#
|
2020-02-22 13:25:25 +01:00
|
|
|
# The most important security checks are in
|
|
|
|
# `create_mirrored_message_users` below, which checks the
|
|
|
|
# same-realm constraint.
|
2013-12-12 18:36:32 +01:00
|
|
|
if "sender" not in request.POST:
|
2016-05-25 15:02:02 +02:00
|
|
|
return json_error(_("Missing sender"))
|
2013-12-12 18:36:32 +01:00
|
|
|
if message_type_name != "private" and not is_super_user:
|
2016-05-25 15:02:02 +02:00
|
|
|
return json_error(_("User not authorized for this query"))
|
2018-09-22 05:10:57 +02:00
|
|
|
|
|
|
|
# For now, mirroring only works with recipient emails, not for
|
|
|
|
# recipient user IDs.
|
|
|
|
if not all(isinstance(to_item, str) for to_item in message_to):
|
|
|
|
return json_error(_("Mirroring not allowed with recipient user IDs"))
|
|
|
|
|
|
|
|
# We need this manual cast so that mypy doesn't complain about
|
|
|
|
# create_mirrored_message_users not being able to accept a Sequence[int]
|
|
|
|
# type parameter.
|
|
|
|
message_to = cast(Sequence[str], message_to)
|
|
|
|
|
2019-03-30 07:11:46 +01:00
|
|
|
try:
|
|
|
|
mirror_sender = create_mirrored_message_users(request, user_profile, message_to)
|
|
|
|
except InvalidMirrorInput:
|
2016-05-25 15:02:02 +02:00
|
|
|
return json_error(_("Invalid mirrored message"))
|
2019-03-30 07:11:46 +01:00
|
|
|
|
2016-07-27 01:45:29 +02:00
|
|
|
if client.name == "zephyr_mirror" and not user_profile.realm.is_zephyr_mirror_realm:
|
2018-03-08 01:53:24 +01:00
|
|
|
return json_error(_("Zephyr mirroring is not allowed in this organization"))
|
2013-12-12 18:36:32 +01:00
|
|
|
sender = mirror_sender
|
|
|
|
else:
|
2020-03-08 21:47:51 +01:00
|
|
|
if "sender" in request.POST:
|
|
|
|
return json_error(_("Invalid mirrored message"))
|
2013-12-12 18:36:32 +01:00
|
|
|
sender = user_profile
|
|
|
|
|
2018-01-12 12:38:45 +01:00
|
|
|
if (delivery_type == 'send_later' or delivery_type == 'remind') and defer_until is None:
|
|
|
|
return json_error(_("Missing deliver_at in a request for delayed message delivery"))
|
|
|
|
|
2018-02-19 06:03:27 +01:00
|
|
|
if (delivery_type == 'send_later' or delivery_type == 'remind') and defer_until is not None:
|
2018-01-04 18:41:34 +01:00
|
|
|
return handle_deferred_message(sender, client, message_type_name,
|
|
|
|
message_to, topic_name, message_content,
|
2018-01-12 12:38:45 +01:00
|
|
|
delivery_type, defer_until, tz_guess,
|
2018-01-04 18:41:34 +01:00
|
|
|
forwarder_user_profile=user_profile,
|
|
|
|
realm=realm)
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
ret = check_send_message(sender, client, message_type_name, message_to,
|
2017-10-27 14:53:28 +02:00
|
|
|
topic_name, message_content, forged=forged,
|
2013-12-12 18:36:32 +01:00
|
|
|
forged_timestamp = request.POST.get('time'),
|
|
|
|
forwarder_user_profile=user_profile, realm=realm,
|
2018-05-21 15:23:46 +02:00
|
|
|
local_id=local_id, sender_queue_id=queue_id,
|
|
|
|
widget_content=widget_content)
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success({"id": ret})
|
|
|
|
|
2017-11-27 09:28:57 +01:00
|
|
|
def fill_edit_history_entries(message_history: List[Dict[str, Any]], message: Message) -> None:
|
2017-02-20 00:19:29 +01:00
|
|
|
"""This fills out the message edit history entries from the database,
|
|
|
|
which are designed to have the minimum data possible, to instead
|
|
|
|
have the current topic + content as of that time, plus data on
|
|
|
|
whatever changed. This makes it much simpler to do future
|
|
|
|
processing.
|
|
|
|
|
|
|
|
Note that this mutates what is passed to it, which is sorta a bad pattern.
|
|
|
|
"""
|
|
|
|
prev_content = message.content
|
|
|
|
prev_rendered_content = message.rendered_content
|
2018-11-01 16:18:20 +01:00
|
|
|
prev_topic = message.topic_name()
|
2018-06-27 12:51:44 +02:00
|
|
|
|
|
|
|
# Make sure that the latest entry in the history corresponds to the
|
|
|
|
# message's last edit time
|
|
|
|
if len(message_history) > 0:
|
2019-03-30 06:59:19 +01:00
|
|
|
assert message.last_edit_time is not None
|
2018-06-27 12:51:44 +02:00
|
|
|
assert(datetime_to_timestamp(message.last_edit_time) ==
|
|
|
|
message_history[0]['timestamp'])
|
2017-02-20 01:44:12 +01:00
|
|
|
|
2017-02-20 00:19:29 +01:00
|
|
|
for entry in message_history:
|
|
|
|
entry['topic'] = prev_topic
|
2018-11-09 17:53:59 +01:00
|
|
|
if LEGACY_PREV_TOPIC in entry:
|
|
|
|
prev_topic = entry[LEGACY_PREV_TOPIC]
|
2017-02-20 00:19:29 +01:00
|
|
|
entry['prev_topic'] = prev_topic
|
2018-11-09 17:53:59 +01:00
|
|
|
del entry[LEGACY_PREV_TOPIC]
|
2017-02-20 00:19:29 +01:00
|
|
|
|
|
|
|
entry['content'] = prev_content
|
|
|
|
entry['rendered_content'] = prev_rendered_content
|
|
|
|
if 'prev_content' in entry:
|
|
|
|
del entry['prev_rendered_content_version']
|
|
|
|
prev_content = entry['prev_content']
|
|
|
|
prev_rendered_content = entry['prev_rendered_content']
|
2019-03-30 06:59:19 +01:00
|
|
|
assert prev_rendered_content is not None
|
2017-02-20 00:19:29 +01:00
|
|
|
entry['content_html_diff'] = highlight_html_differences(
|
|
|
|
prev_rendered_content,
|
2017-10-12 08:38:02 +02:00
|
|
|
entry['rendered_content'],
|
|
|
|
message.id)
|
2017-02-20 00:19:29 +01:00
|
|
|
|
2017-02-20 01:44:12 +01:00
|
|
|
message_history.append(dict(
|
|
|
|
topic = prev_topic,
|
|
|
|
content = prev_content,
|
|
|
|
rendered_content = prev_rendered_content,
|
2019-08-28 02:43:19 +02:00
|
|
|
timestamp = datetime_to_timestamp(message.date_sent),
|
2017-02-20 01:44:12 +01:00
|
|
|
user_id = message.sender_id,
|
|
|
|
))
|
2017-02-20 00:19:29 +01:00
|
|
|
|
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def get_message_edit_history(request: HttpRequest, user_profile: UserProfile,
|
2019-08-17 01:21:08 +02:00
|
|
|
message_id: int=REQ(converter=to_non_negative_int,
|
|
|
|
path_only=True)) -> HttpResponse:
|
2017-07-16 11:00:44 +02:00
|
|
|
if not user_profile.realm.allow_edit_history:
|
|
|
|
return json_error(_("Message edit history is disabled in this organization"))
|
2017-02-20 00:19:29 +01:00
|
|
|
message, ignored_user_message = access_message(user_profile, message_id)
|
|
|
|
|
|
|
|
# Extract the message edit history from the message
|
2018-06-27 12:51:44 +02:00
|
|
|
if message.edit_history is not None:
|
|
|
|
message_edit_history = ujson.loads(message.edit_history)
|
|
|
|
else:
|
|
|
|
message_edit_history = []
|
2017-02-20 00:19:29 +01:00
|
|
|
|
|
|
|
# Fill in all the extra data that will make it usable
|
|
|
|
fill_edit_history_entries(message_edit_history, message)
|
2017-02-20 01:46:07 +01:00
|
|
|
return json_success({"message_history": reversed(message_edit_history)})
|
2017-02-20 00:19:29 +01:00
|
|
|
|
2020-03-24 20:36:45 +01:00
|
|
|
PROPAGATE_MODE_VALUES = ["change_later", "change_one", "change_all"]
|
2013-12-12 18:36:32 +01:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def update_message_backend(request: HttpRequest, user_profile: UserMessage,
|
2019-10-07 13:23:05 +02:00
|
|
|
message_id: int=REQ(converter=to_non_negative_int, path_only=True),
|
2020-02-19 01:38:34 +01:00
|
|
|
stream_id: Optional[int]=REQ(converter=to_non_negative_int, default=None),
|
2018-11-09 18:45:12 +01:00
|
|
|
topic_name: Optional[str]=REQ_topic(),
|
2020-03-24 20:36:45 +01:00
|
|
|
propagate_mode: Optional[str]=REQ(
|
|
|
|
default="change_one",
|
|
|
|
str_validator=check_string_in(PROPAGATE_MODE_VALUES)),
|
2018-04-24 03:47:28 +02:00
|
|
|
content: Optional[str]=REQ(default=None)) -> HttpResponse:
|
2016-06-21 21:34:41 +02:00
|
|
|
if not user_profile.realm.allow_message_editing:
|
2017-05-23 22:02:43 +02:00
|
|
|
return json_error(_("Your organization has turned off message editing"))
|
2016-07-11 03:01:03 +02:00
|
|
|
|
2020-02-19 01:38:34 +01:00
|
|
|
if propagate_mode != "change_one" and topic_name is None and stream_id is None:
|
2020-03-24 20:36:45 +01:00
|
|
|
return json_error(_("Invalid propagate_mode without topic edit"))
|
|
|
|
|
2017-02-23 00:24:11 +01:00
|
|
|
message, ignored_user_message = access_message(user_profile, message_id)
|
2018-05-24 14:08:50 +02:00
|
|
|
is_no_topic_msg = (message.topic_name() == "(no topic)")
|
2016-06-21 21:34:41 +02:00
|
|
|
|
|
|
|
# You only have permission to edit a message if:
|
2017-03-13 14:42:03 +01:00
|
|
|
# you change this value also change those two parameters in message_edit.js.
|
2016-06-21 21:34:41 +02:00
|
|
|
# 1. You sent it, OR:
|
|
|
|
# 2. This is a topic-only edit for a (no topic) message, OR:
|
2017-12-03 00:56:17 +01:00
|
|
|
# 3. This is a topic-only edit and you are an admin, OR:
|
|
|
|
# 4. This is a topic-only edit and your realm allows users to edit topics.
|
2016-06-21 21:34:41 +02:00
|
|
|
if message.sender == user_profile:
|
|
|
|
pass
|
2018-05-24 14:08:50 +02:00
|
|
|
elif (content is None) and (is_no_topic_msg or
|
2017-12-03 00:56:17 +01:00
|
|
|
user_profile.is_realm_admin or
|
|
|
|
user_profile.realm.allow_community_topic_editing):
|
2016-06-21 21:34:41 +02:00
|
|
|
pass
|
|
|
|
else:
|
|
|
|
raise JsonableError(_("You don't have permission to edit this message"))
|
|
|
|
|
|
|
|
# If there is a change to the content, check that it hasn't been too long
|
|
|
|
# Allow an extra 20 seconds since we potentially allow editing 15 seconds
|
|
|
|
# past the limit, and in case there are network issues, etc. The 15 comes
|
|
|
|
# from (min_seconds_to_edit + seconds_left_buffer) in message_edit.js; if
|
|
|
|
# you change this value also change those two parameters in message_edit.js.
|
|
|
|
edit_limit_buffer = 20
|
2016-07-17 19:40:54 +02:00
|
|
|
if content is not None and user_profile.realm.message_content_edit_limit_seconds > 0:
|
|
|
|
deadline_seconds = user_profile.realm.message_content_edit_limit_seconds + edit_limit_buffer
|
2019-08-28 02:43:19 +02:00
|
|
|
if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds):
|
2018-04-28 04:24:14 +02:00
|
|
|
raise JsonableError(_("The time limit for editing this message has passed"))
|
2016-06-21 21:34:41 +02:00
|
|
|
|
2017-12-03 00:56:17 +01:00
|
|
|
# If there is a change to the topic, check that the user is allowed to
|
|
|
|
# edit it and that it has not been too long. If this is not the user who
|
|
|
|
# sent the message, they are not the admin, and the time limit for editing
|
|
|
|
# topics is passed, raise an error.
|
2018-05-24 14:08:50 +02:00
|
|
|
if content is None and message.sender != user_profile and not user_profile.is_realm_admin and \
|
|
|
|
not is_no_topic_msg:
|
2017-12-03 00:56:17 +01:00
|
|
|
deadline_seconds = Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS + edit_limit_buffer
|
2019-08-28 02:43:19 +02:00
|
|
|
if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds):
|
2018-04-28 04:24:14 +02:00
|
|
|
raise JsonableError(_("The time limit for editing this message has passed"))
|
2017-12-03 00:56:17 +01:00
|
|
|
|
2020-02-19 01:38:34 +01:00
|
|
|
if topic_name is None and content is None and stream_id is None:
|
2016-07-11 03:01:03 +02:00
|
|
|
return json_error(_("Nothing to change"))
|
2018-11-09 18:42:46 +01:00
|
|
|
if topic_name is not None:
|
|
|
|
topic_name = topic_name.strip()
|
|
|
|
if topic_name == "":
|
2016-07-11 02:52:06 +02:00
|
|
|
raise JsonableError(_("Topic can't be empty"))
|
2016-06-21 21:34:41 +02:00
|
|
|
rendered_content = None
|
python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.
We can likely further refine the remaining pieces with some testing.
Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:
- invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+ invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None
- notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+ signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
- author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+ author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
- bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+ bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
- default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+ default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}
-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}
-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 01:09:50 +02:00
|
|
|
links_for_embed: Set[str] = set()
|
|
|
|
prior_mention_user_ids: Set[int] = set()
|
|
|
|
mention_user_ids: Set[int] = set()
|
|
|
|
mention_data: Optional[bugdown.MentionData] = None
|
2016-07-11 02:58:23 +02:00
|
|
|
if content is not None:
|
|
|
|
content = content.strip()
|
|
|
|
if content == "":
|
2016-11-09 22:16:12 +01:00
|
|
|
content = "(deleted)"
|
2016-06-21 21:34:41 +02:00
|
|
|
content = truncate_body(content)
|
2016-09-14 21:58:44 +02:00
|
|
|
|
2019-11-28 11:26:57 +01:00
|
|
|
mention_data = bugdown.MentionData(
|
|
|
|
realm_id=user_profile.realm.id,
|
|
|
|
content=content,
|
|
|
|
)
|
2017-09-27 15:06:03 +02:00
|
|
|
user_info = get_user_info_for_message_updates(message.id)
|
Notify offline users about edited stream messages.
We now do push notifications and missed message emails
for offline users who are subscribed to the stream for
a message that has been edited, but we short circuit
the offline-notification logic for any user who presumably
would have already received a notification on the original
message.
This effectively boils down to sending notifications to newly
mentioned users. The motivating use case here is that you
forget to mention somebody in a message, and then you edit
the message to mention the person. If they are offline, they
will now get pushed notifications and missed message emails,
with some minor caveats.
We try to mostly use the same techniques here as the
send-message code path, and we share common code with the
send-message path once we get to the Tornado layer and call
maybe_enqueue_notifications.
The major places where we differ are in a function called
maybe_enqueue_notifications_for_message_update, and the top
of that function short circuits a bunch of cases where we
can mostly assume that the original message had an offline
notification.
We can expect a couple changes in the future:
* Requirements may change here, and it might make sense
to send offline notifications on the update side even
in circumstances where the original message had a
notification.
* We may track more notifications in a DB model, which
may simplify our short-circuit logic.
In the view/action layer, we already had two separate codepaths
for send-message and update-message, but this mostly echoes
what the send-message path does in terms of collecting data
about recipients.
2017-10-03 16:25:12 +02:00
|
|
|
prior_mention_user_ids = user_info['mention_user_ids']
|
2017-01-24 02:07:51 +01:00
|
|
|
|
2017-01-22 05:55:30 +01:00
|
|
|
# We render the message using the current user's realm; since
|
|
|
|
# the cross-realm bots never edit messages, this should be
|
|
|
|
# always correct.
|
|
|
|
# Note: If rendering fails, the called code will raise a JsonableError.
|
2016-09-15 00:24:44 +02:00
|
|
|
rendered_content = render_incoming_message(message,
|
2017-01-22 05:55:30 +01:00
|
|
|
content,
|
2017-09-27 15:06:03 +02:00
|
|
|
user_info['message_user_ids'],
|
2019-11-28 11:26:57 +01:00
|
|
|
user_profile.realm,
|
|
|
|
mention_data=mention_data)
|
2016-10-27 12:06:44 +02:00
|
|
|
links_for_embed |= message.links_for_preview
|
2016-07-11 03:01:03 +02:00
|
|
|
|
Notify offline users about edited stream messages.
We now do push notifications and missed message emails
for offline users who are subscribed to the stream for
a message that has been edited, but we short circuit
the offline-notification logic for any user who presumably
would have already received a notification on the original
message.
This effectively boils down to sending notifications to newly
mentioned users. The motivating use case here is that you
forget to mention somebody in a message, and then you edit
the message to mention the person. If they are offline, they
will now get pushed notifications and missed message emails,
with some minor caveats.
We try to mostly use the same techniques here as the
send-message code path, and we share common code with the
send-message path once we get to the Tornado layer and call
maybe_enqueue_notifications.
The major places where we differ are in a function called
maybe_enqueue_notifications_for_message_update, and the top
of that function short circuits a bunch of cases where we
can mostly assume that the original message had an offline
notification.
We can expect a couple changes in the future:
* Requirements may change here, and it might make sense
to send offline notifications on the update side even
in circumstances where the original message had a
notification.
* We may track more notifications in a DB model, which
may simplify our short-circuit logic.
In the view/action layer, we already had two separate codepaths
for send-message and update-message, but this mostly echoes
what the send-message path does in terms of collecting data
about recipients.
2017-10-03 16:25:12 +02:00
|
|
|
mention_user_ids = message.mentions_user_ids
|
|
|
|
|
2020-02-19 01:38:34 +01:00
|
|
|
new_stream = None
|
|
|
|
old_stream = None
|
|
|
|
number_changed = 0
|
|
|
|
|
|
|
|
if stream_id is not None:
|
|
|
|
if not user_profile.is_realm_admin:
|
|
|
|
raise JsonableError(_("You don't have permission to move this message"))
|
|
|
|
if content is not None:
|
|
|
|
raise JsonableError(_("Cannot change message content while changing stream"))
|
|
|
|
|
|
|
|
old_stream = get_stream_by_id(message.recipient.type_id)
|
|
|
|
new_stream = get_stream_by_id(stream_id)
|
|
|
|
|
|
|
|
if not (old_stream.is_public() and new_stream.is_public()):
|
|
|
|
# We'll likely decide to relax this condition in the
|
|
|
|
# future; it just requires more care with details like the
|
|
|
|
# breadcrumb messages.
|
|
|
|
raise JsonableError(_("Streams must be public"))
|
|
|
|
|
|
|
|
number_changed = do_update_message(user_profile, message, new_stream,
|
|
|
|
topic_name, propagate_mode,
|
|
|
|
content, rendered_content,
|
Notify offline users about edited stream messages.
We now do push notifications and missed message emails
for offline users who are subscribed to the stream for
a message that has been edited, but we short circuit
the offline-notification logic for any user who presumably
would have already received a notification on the original
message.
This effectively boils down to sending notifications to newly
mentioned users. The motivating use case here is that you
forget to mention somebody in a message, and then you edit
the message to mention the person. If they are offline, they
will now get pushed notifications and missed message emails,
with some minor caveats.
We try to mostly use the same techniques here as the
send-message code path, and we share common code with the
send-message path once we get to the Tornado layer and call
maybe_enqueue_notifications.
The major places where we differ are in a function called
maybe_enqueue_notifications_for_message_update, and the top
of that function short circuits a bunch of cases where we
can mostly assume that the original message had an offline
notification.
We can expect a couple changes in the future:
* Requirements may change here, and it might make sense
to send offline notifications on the update side even
in circumstances where the original message had a
notification.
* We may track more notifications in a DB model, which
may simplify our short-circuit logic.
In the view/action layer, we already had two separate codepaths
for send-message and update-message, but this mostly echoes
what the send-message path does in terms of collecting data
about recipients.
2017-10-03 16:25:12 +02:00
|
|
|
prior_mention_user_ids,
|
2019-11-28 11:26:57 +01:00
|
|
|
mention_user_ids, mention_data)
|
Notify offline users about edited stream messages.
We now do push notifications and missed message emails
for offline users who are subscribed to the stream for
a message that has been edited, but we short circuit
the offline-notification logic for any user who presumably
would have already received a notification on the original
message.
This effectively boils down to sending notifications to newly
mentioned users. The motivating use case here is that you
forget to mention somebody in a message, and then you edit
the message to mention the person. If they are offline, they
will now get pushed notifications and missed message emails,
with some minor caveats.
We try to mostly use the same techniques here as the
send-message code path, and we share common code with the
send-message path once we get to the Tornado layer and call
maybe_enqueue_notifications.
The major places where we differ are in a function called
maybe_enqueue_notifications_for_message_update, and the top
of that function short circuits a bunch of cases where we
can mostly assume that the original message had an offline
notification.
We can expect a couple changes in the future:
* Requirements may change here, and it might make sense
to send offline notifications on the update side even
in circumstances where the original message had a
notification.
* We may track more notifications in a DB model, which
may simplify our short-circuit logic.
In the view/action layer, we already had two separate codepaths
for send-message and update-message, but this mostly echoes
what the send-message path does in terms of collecting data
about recipients.
2017-10-03 16:25:12 +02:00
|
|
|
|
2017-01-24 02:07:12 +01:00
|
|
|
# Include the number of messages changed in the logs
|
|
|
|
request._log_data['extra'] = "[%s]" % (number_changed,)
|
2020-04-26 13:30:57 +02:00
|
|
|
if links_for_embed:
|
2016-10-27 12:06:44 +02:00
|
|
|
event_data = {
|
|
|
|
'message_id': message.id,
|
|
|
|
'message_content': message.content,
|
2017-02-23 06:20:01 +01:00
|
|
|
# The choice of `user_profile.realm_id` rather than
|
|
|
|
# `sender.realm_id` must match the decision made in the
|
|
|
|
# `render_incoming_message` call earlier in this function.
|
|
|
|
'message_realm_id': user_profile.realm_id,
|
2016-10-27 12:06:44 +02:00
|
|
|
'urls': links_for_embed}
|
2017-11-24 13:18:46 +01:00
|
|
|
queue_json_publish('embed_links', event_data)
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success()
|
|
|
|
|
2017-05-14 21:14:26 +02:00
|
|
|
|
2017-11-26 09:12:10 +01:00
|
|
|
def validate_can_delete_message(user_profile: UserProfile, message: Message) -> None:
|
|
|
|
if user_profile.is_realm_admin:
|
|
|
|
# Admin can delete any message, any time.
|
|
|
|
return
|
|
|
|
if message.sender != user_profile:
|
|
|
|
# Users can only delete messages sent by them.
|
|
|
|
raise JsonableError(_("You don't have permission to delete this message"))
|
|
|
|
if not user_profile.realm.allow_message_deleting:
|
|
|
|
# User can not delete message, if message deleting is not allowed in realm.
|
|
|
|
raise JsonableError(_("You don't have permission to delete this message"))
|
|
|
|
|
|
|
|
deadline_seconds = user_profile.realm.message_content_delete_limit_seconds
|
|
|
|
if deadline_seconds == 0:
|
|
|
|
# 0 for no time limit to delete message
|
|
|
|
return
|
2019-08-28 02:43:19 +02:00
|
|
|
if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds):
|
2017-11-26 09:12:10 +01:00
|
|
|
# User can not delete message after deadline time of realm
|
|
|
|
raise JsonableError(_("The time limit for deleting this message has passed"))
|
|
|
|
return
|
|
|
|
|
2017-05-14 21:14:26 +02:00
|
|
|
@has_request_variables
|
2017-11-27 09:28:57 +01:00
|
|
|
def delete_message_backend(request: HttpRequest, user_profile: UserProfile,
|
2019-08-17 01:21:08 +02:00
|
|
|
message_id: int=REQ(converter=to_non_negative_int,
|
|
|
|
path_only=True)) -> HttpResponse:
|
2017-05-14 21:14:26 +02:00
|
|
|
message, ignored_user_message = access_message(user_profile, message_id)
|
2017-11-26 09:12:10 +01:00
|
|
|
validate_can_delete_message(user_profile, message)
|
2019-01-24 19:14:25 +01:00
|
|
|
try:
|
2019-11-12 21:20:31 +01:00
|
|
|
do_delete_messages(user_profile.realm, [message])
|
2019-01-24 19:14:25 +01:00
|
|
|
except (Message.DoesNotExist, IntegrityError):
|
|
|
|
raise JsonableError(_("Message already deleted"))
|
2017-05-14 21:14:26 +02:00
|
|
|
return json_success()
|
|
|
|
|
2013-12-12 18:36:32 +01:00
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def json_fetch_raw_message(request: HttpRequest, user_profile: UserProfile,
|
2019-10-07 13:23:05 +02:00
|
|
|
message_id: int=REQ(converter=to_non_negative_int,
|
|
|
|
path_only=True)) -> HttpResponse:
|
2016-10-12 02:14:08 +02:00
|
|
|
(message, user_message) = access_message(user_profile, message_id)
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success({"raw_content": message.content})
|
|
|
|
|
|
|
|
@has_request_variables
|
2017-11-27 09:28:57 +01:00
|
|
|
def render_message_backend(request: HttpRequest, user_profile: UserProfile,
|
2018-04-24 03:47:28 +02:00
|
|
|
content: str=REQ()) -> HttpResponse:
|
2016-11-07 20:40:40 +01:00
|
|
|
message = Message()
|
|
|
|
message.sender = user_profile
|
|
|
|
message.content = content
|
|
|
|
message.sending_client = request.client
|
|
|
|
|
2017-01-18 23:19:18 +01:00
|
|
|
rendered_content = render_markdown(message, content, realm=user_profile.realm)
|
2013-12-12 18:36:32 +01:00
|
|
|
return json_success({"rendered": rendered_content})
|
|
|
|
|
|
|
|
@has_request_variables
|
2017-12-03 18:46:27 +01:00
|
|
|
def messages_in_narrow_backend(request: HttpRequest, user_profile: UserProfile,
|
|
|
|
msg_ids: List[int]=REQ(validator=check_list(check_int)),
|
2019-08-11 19:07:34 +02:00
|
|
|
narrow: OptionalNarrowListT=REQ(converter=narrow_parameter)
|
2017-12-03 18:46:27 +01:00
|
|
|
) -> HttpResponse:
|
2016-07-30 01:27:56 +02:00
|
|
|
|
2018-01-02 18:33:28 +01:00
|
|
|
first_visible_message_id = get_first_visible_message_id(user_profile.realm)
|
|
|
|
msg_ids = [message_id for message_id in msg_ids if message_id >= first_visible_message_id]
|
2017-06-30 02:24:05 +02:00
|
|
|
# This query is limited to messages the user has access to because they
|
|
|
|
# actually received them, as reflected in `zerver_usermessage`.
|
2018-11-09 17:06:00 +01:00
|
|
|
query = select([column("message_id"), topic_column_sa(), column("rendered_content")],
|
2013-12-10 23:32:29 +01:00
|
|
|
and_(column("user_profile_id") == literal(user_profile.id),
|
|
|
|
column("message_id").in_(msg_ids)),
|
2017-02-22 22:13:57 +01:00
|
|
|
join(table("zerver_usermessage"), table("zerver_message"),
|
2013-12-10 23:32:29 +01:00
|
|
|
literal_column("zerver_usermessage.message_id") ==
|
|
|
|
literal_column("zerver_message.id")))
|
|
|
|
|
2014-02-27 20:09:59 +01:00
|
|
|
builder = NarrowBuilder(user_profile, column("message_id"))
|
2017-03-19 01:46:35 +01:00
|
|
|
if narrow is not None:
|
|
|
|
for term in narrow:
|
|
|
|
query = builder.add_term(query, term)
|
2013-12-12 18:36:32 +01:00
|
|
|
|
2013-12-10 23:32:29 +01:00
|
|
|
sa_conn = get_sqlalchemy_connection()
|
|
|
|
query_result = list(sa_conn.execute(query).fetchall())
|
|
|
|
|
|
|
|
search_fields = dict()
|
|
|
|
for row in query_result:
|
2017-08-18 15:50:54 +02:00
|
|
|
message_id = row['message_id']
|
2018-11-09 17:19:17 +01:00
|
|
|
topic_name = row[DB_TOPIC_NAME]
|
2017-08-18 15:50:54 +02:00
|
|
|
rendered_content = row['rendered_content']
|
|
|
|
|
|
|
|
if 'content_matches' in row:
|
|
|
|
content_matches = row['content_matches']
|
2018-11-09 16:32:05 +01:00
|
|
|
topic_matches = row['topic_matches']
|
2018-11-09 17:19:17 +01:00
|
|
|
search_fields[message_id] = get_search_fields(rendered_content, topic_name,
|
2018-11-09 16:32:05 +01:00
|
|
|
content_matches, topic_matches)
|
2017-08-18 15:50:54 +02:00
|
|
|
else:
|
2018-11-09 17:25:57 +01:00
|
|
|
search_fields[message_id] = {
|
|
|
|
'match_content': rendered_content,
|
|
|
|
MATCH_TOPIC: escape_html(topic_name),
|
|
|
|
}
|
2013-12-10 23:32:29 +01:00
|
|
|
|
|
|
|
return json_success({"messages": search_fields})
|