Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Use text_type as type of cache keys and update users. 

This changes the type annotations for the cache keys in Zulip to be
consistently text_type, and updates the annotations for values that
are used as cache keys across the codebase.
This commit is contained in:
Eklavya Sharma 2016-06-10 03:42:34 +05:30 committed by Tim Abbott
parent d3b80d94a2
commit 53084fe03c
5 changed files with 49 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
zerver/lib/cache.py
View File

@ -87,7 +87,7 @@ def bounce_key_prefix_for_testing(test_name):
KEY_PREFIX = test_name + u':' + text_type(os.getpid()) + u':'
def get_cache_backend(cache_name):
# type: (str) -> get_cache
# type: (Optional[str]) -> get_cache
if cache_name is None:
return djcache
return get_cache(cache_name)
@ -138,14 +138,14 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None)
return decorator
def cache_set(key, val, cache_name=None, timeout=None):
# type: (str, Any, Optional[str], Optional[int]) -> None
# type: (text_type, Any, Optional[str], Optional[int]) -> None
remote_cache_stats_start()
cache_backend = get_cache_backend(cache_name)
cache_backend.set(KEY_PREFIX + key, (val,), timeout=timeout)
remote_cache_stats_finish()
def cache_get(key, cache_name=None):
# type: (str, Optional[str]) -> Any
# type: (text_type, Optional[str]) -> Any
remote_cache_stats_start()
cache_backend = get_cache_backend(cache_name)
ret = cache_backend.get(KEY_PREFIX + key)
@ -153,31 +153,31 @@ def cache_get(key, cache_name=None):
return ret
def cache_get_many(keys, cache_name=None):
# type: (List[str], Optional[str]) -> Dict[str, Any]
keys = [KEY_PREFIX + key for key in keys] # type: ignore # temporary
# type: (List[text_type], Optional[str]) -> Dict[text_type, Any]
keys = [KEY_PREFIX + key for key in keys]
remote_cache_stats_start()
ret = get_cache_backend(cache_name).get_many(keys)
remote_cache_stats_finish()
return dict([(key[len(KEY_PREFIX):], value) for key, value in ret.items()])
def cache_set_many(items, cache_name=None, timeout=None):
# type: (Dict[str, Any], Optional[str], Optional[int]) -> None
# type: (Dict[text_type, Any], Optional[str], Optional[int]) -> None
new_items = {}
for key in items:
new_items[KEY_PREFIX + key] = items[key]
items = new_items # type: ignore # temporary
items = new_items
remote_cache_stats_start()
get_cache_backend(cache_name).set_many(items, timeout=timeout)
remote_cache_stats_finish()
def cache_delete(key, cache_name=None):
# type: (str, Optional[str]) -> None
# type: (text_type, Optional[str]) -> None
remote_cache_stats_start()
get_cache_backend(cache_name).delete(KEY_PREFIX + key)
remote_cache_stats_finish()
def cache_delete_many(items, cache_name=None):
# type: (Iterable[str], Optional[str]) -> None
# type: (Iterable[text_type], Optional[str]) -> None
remote_cache_stats_start()
get_cache_backend(cache_name).delete_many(
KEY_PREFIX + item for item in items)
@ -202,8 +202,8 @@ def generic_bulk_cached_fetch(cache_key_function, query_function, object_ids,
setter=lambda obj: obj,
id_fetcher=lambda obj: obj.id,
cache_transformer=lambda obj: obj):
# type: (Callable[[Any], str], Callable[[List[Any]], List[Any]], List[Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any]) -> Dict[Any, Any]
cache_keys = {} # type: Dict[int, str]
# type: (Callable[[Any], text_type], Callable[[List[Any]], Iterable[Any]], Iterable[Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any]) -> Dict[Any, Any]
cache_keys = {} # type: Dict[Any, text_type]
for object_id in object_ids:
cache_keys[object_id] = cache_key_function(object_id)
cached_objects = cache_get_many([cache_keys[object_id]
@ -214,7 +214,7 @@ def generic_bulk_cached_fetch(cache_key_function, query_function, object_ids,
cache_keys[object_id] not in cached_objects]
db_objects = query_function(needed_ids)
items_for_remote_cache = {} # type: Dict[str, Any]
items_for_remote_cache = {} # type: Dict[text_type, Any]
for obj in db_objects:
key = cache_keys[id_fetcher(obj)]
item = cache_transformer(obj)
@ -244,23 +244,23 @@ def cache(func):
return cache_with_key(keyfunc)(func)
def message_cache_key(message_id):
# type: (int) -> str
return "message:%d" % (message_id,)
# type: (int) -> text_type
return u"message:%d" % (message_id,)
def display_recipient_cache_key(recipient_id):
# type: (int) -> str
return "display_recipient_dict:%d" % (recipient_id,)
# type: (int) -> text_type
return u"display_recipient_dict:%d" % (recipient_id,)
def user_profile_by_email_cache_key(email):
# type: (str) -> str
# type: (text_type) -> text_type
# See the comment in zerver/lib/avatar.py:gravatar_hash for why we
# are proactively encoding email addresses even though they will
# with high likelihood be ASCII-only for the foreseeable future.
return 'user_profile_by_email:%s' % (make_safe_digest(email.strip()),)
return u'user_profile_by_email:%s' % (make_safe_digest(email.strip()),)
def user_profile_by_id_cache_key(user_profile_id):
# type: (int) -> str
return "user_profile_by_id:%s" % (user_profile_id,)
# type: (int) -> text_type
return u"user_profile_by_id:%s" % (user_profile_id,)
# TODO: Refactor these cache helpers into another file that can import
# models.py so that we can replace many of these type: Anys
@ -271,8 +271,8 @@ def cache_save_user_profile(user_profile):
active_user_dict_fields = ['id', 'full_name', 'short_name', 'email', 'is_realm_admin', 'is_bot'] # type: List[str]
def active_user_dicts_in_realm_cache_key(realm):
# type: (Any) -> str
return "active_user_dicts_in_realm:%s" % (realm.id,)
# type: (Any) -> text_type
return u"active_user_dicts_in_realm:%s" % (realm.id,)
active_bot_dict_fields = ['id', 'full_name', 'short_name',
'email', 'default_sending_stream__name',
@ -280,17 +280,17 @@ active_bot_dict_fields = ['id', 'full_name', 'short_name',
'default_all_public_streams', 'api_key',
'bot_owner__email', 'avatar_source'] # type: List[str]
def active_bot_dicts_in_realm_cache_key(realm):
# type: (Any) -> str
return "active_bot_dicts_in_realm:%s" % (realm.id,)
# type: (Any) -> text_type
return u"active_bot_dicts_in_realm:%s" % (realm.id,)
def get_stream_cache_key(stream_name, realm):
# type: (six.text_type, Any) -> str
# type: (text_type, Any) -> text_type
from zerver.models import Realm
if isinstance(realm, Realm):
realm_id = realm.id
else:
realm_id = realm
return "stream_by_realm_and_name:%s:%s" % (
return u"stream_by_realm_and_name:%s:%s" % (
realm_id, make_safe_digest(stream_name.strip().lower()))
def update_user_profile_caches(user_profiles):
@ -341,8 +341,8 @@ def flush_realm(sender, **kwargs):
cache_delete(realm_alert_words_cache_key(realm))
def realm_alert_words_cache_key(realm):
# type: (Any) -> str
return "realm_alert_words:%s" % (realm.domain,)
# type: (Any) -> text_type
return u"realm_alert_words:%s" % (realm.domain,)
# Called by models.py to flush the stream cache whenever we save a stream
# object.

20
zerver/lib/cache_helpers.py
View File

@ -1,4 +1,6 @@
from __future__ import absolute_import
from six import text_type
from typing import Any, Dict, Callable, Tuple
# This file needs to be different from cache.py because cache.py
@ -37,33 +39,33 @@ def message_fetch_objects():
id__gt=max_id - MESSAGE_CACHE_SIZE)
def message_cache_items(items_for_remote_cache, message):
# type: (Dict[str, Tuple[Message]], Message) -> None
# type: (Dict[text_type, Tuple[Message]], Message) -> None
items_for_remote_cache[message_cache_key(message.id)] = (message,)
def user_cache_items(items_for_remote_cache, user_profile):
# type: (Dict[str, Tuple[UserProfile]], UserProfile) -> None
# type: (Dict[text_type, Tuple[UserProfile]], UserProfile) -> None
items_for_remote_cache[user_profile_by_email_cache_key(user_profile.email)] = (user_profile,)
items_for_remote_cache[user_profile_by_id_cache_key(user_profile.id)] = (user_profile,)
def stream_cache_items(items_for_remote_cache, stream):
# type: (Dict[str, Tuple[Stream]], Stream) -> None
# type: (Dict[text_type, Tuple[Stream]], Stream) -> None
items_for_remote_cache[get_stream_cache_key(stream.name, stream.realm_id)] = (stream,)
def client_cache_items(items_for_remote_cache, client):
# type: (Dict[str, Tuple[Client]], Client) -> None
# type: (Dict[text_type, Tuple[Client]], Client) -> None
items_for_remote_cache[get_client_cache_key(client.name)] = (client,)
def huddle_cache_items(items_for_remote_cache, huddle):
# type: (Dict[str, Tuple[Huddle]], Huddle) -> None
# type: (Dict[text_type, Tuple[Huddle]], Huddle) -> None
items_for_remote_cache[huddle_hash_cache_key(huddle.huddle_hash)] = (huddle,)
def recipient_cache_items(items_for_remote_cache, recipient):
# type: (Dict[str, Tuple[Recipient]], Recipient) -> None
# type: (Dict[text_type, Tuple[Recipient]], Recipient) -> None
items_for_remote_cache[get_recipient_cache_key(recipient.type, recipient.type_id)] = (recipient,)
session_engine = import_module(settings.SESSION_ENGINE)
def session_cache_items(items_for_remote_cache, session):
# type: (Dict[str, str], Session) -> None
# type: (Dict[text_type, text_type], Session) -> None
store = session_engine.SessionStore(session_key=session.session_key)
items_for_remote_cache[store.cache_key] = store.decode(session.session_data)
@ -81,13 +83,13 @@ cache_fillers = {
'message': (message_fetch_objects, message_cache_items, 3600 * 24, 1000),
'huddle': (lambda: Huddle.objects.select_related().all(), huddle_cache_items, 3600*24*7, 10000),
'session': (lambda: Session.objects.all(), session_cache_items, 3600*24*7, 10000),
} # type: Dict[str, Tuple[Callable[[], List[Any]], Callable[[Dict[str, Any], Any], None], int, int]]
} # type: Dict[str, Tuple[Callable[[], List[Any]], Callable[[Dict[text_type, Any], Any], None], int, int]]
def fill_remote_cache(cache):
# type: (str) -> None
remote_cache_time_start = get_remote_cache_time()
remote_cache_requests_start = get_remote_cache_requests()
items_for_remote_cache = {} # type: Dict[str, Any]
items_for_remote_cache = {} # type: Dict[text_type, Any]
(objects, items_filler, timeout, batch_size) = cache_fillers[cache]
count = 0
for obj in objects():

8
zerver/lib/test_helpers.py
View File

@ -71,15 +71,15 @@ def tornado_redirected_to_list(lst):
@contextmanager
def simulated_empty_cache():
# type: () -> Generator[List[Tuple[str, Union[str, List[str]], str]], None, None]
cache_queries = [] # type: List[Tuple[str, Union[str, List[str]], str]]
# type: () -> Generator[List[Tuple[str, Union[text_type, List[text_type]], text_type]], None, None]
cache_queries = [] # type: List[Tuple[str, Union[text_type, List[text_type]], text_type]]
def my_cache_get(key, cache_name=None):
# type: (str, Optional[str]) -> Any
# type: (text_type, Optional[str]) -> Any
cache_queries.append(('get', key, cache_name))
return None
def my_cache_get_many(keys, cache_name=None):
# type: (List[str], Optional[str]) -> Dict[str, Any]
# type: (List[text_type], Optional[str]) -> Dict[text_type, Any]
cache_queries.append(('getmany', keys, cache_name))
return None

10
zerver/models.py
View File

@ -45,7 +45,7 @@ bugdown = None # type: Any
MAX_SUBJECT_LENGTH = 60
MAX_MESSAGE_LENGTH = 10000
STREAM_NAMES = TypeVar('STREAM_NAMES', Sequence[str], AbstractSet[str])
STREAM_NAMES = TypeVar('STREAM_NAMES', Sequence[text_type], AbstractSet[text_type])
# Doing 1000 remote cache requests to get_display_recipient is quite slow,
# so add a local cache as well as the remote cache cache.
@ -701,7 +701,7 @@ def bulk_get_streams(realm, stream_names):
realm_id = realm
def fetch_streams_by_name(stream_names):
# type: (List[str]) -> List[str]
# type: (List[text_type]) -> Sequence[Stream]
#
# This should be just
#
@ -758,11 +758,11 @@ def stringify_message_dict(message_dict):
return zlib.compress(force_bytes(ujson.dumps(message_dict)))
def to_dict_cache_key_id(message_id, apply_markdown):
# type: (int, bool) -> str
return 'message_dict:%d:%d' % (message_id, apply_markdown)
# type: (int, bool) -> text_type
return u'message_dict:%d:%d' % (message_id, apply_markdown)
def to_dict_cache_key(message, apply_markdown):
# type: (Message, bool) -> str
# type: (Message, bool) -> text_type
return to_dict_cache_key_id(message.id, apply_markdown)
class Message(models.Model):

2
zerver/views/messages.py
View File

@ -552,7 +552,7 @@ def get_old_messages_backend(request, user_profile,
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
search_fields = dict() # type: Dict[int, Dict[str, str]]
search_fields = dict() # type: Dict[int, Dict[str, text_type]]
message_ids = [] # type: List[int]
user_message_flags = {} # type: Dict[int, List[str]]
if include_history: