Use text_type as type of cache keys and update users.

This changes the type annotations for the cache keys in Zulip to be
consistently text_type, and updates the annotations for values that
are used as cache keys across the codebase.

zerver/lib/cache.py

@@ -87,7 +87,7 @@ def bounce_key_prefix_for_testing(test_name):
     KEY_PREFIX = test_name + u':' + text_type(os.getpid()) + u':'
 
 def get_cache_backend(cache_name):
-    # type: (str) -> get_cache
+    # type: (Optional[str]) -> get_cache
     if cache_name is None:
         return djcache
     return get_cache(cache_name)
@@ -138,14 +138,14 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None)
     return decorator
 
 def cache_set(key, val, cache_name=None, timeout=None):
-    # type: (str, Any, Optional[str], Optional[int]) -> None
+    # type: (text_type, Any, Optional[str], Optional[int]) -> None
     remote_cache_stats_start()
     cache_backend = get_cache_backend(cache_name)
     cache_backend.set(KEY_PREFIX + key, (val,), timeout=timeout)
     remote_cache_stats_finish()
 
 def cache_get(key, cache_name=None):
-    # type: (str, Optional[str]) -> Any
+    # type: (text_type, Optional[str]) -> Any
     remote_cache_stats_start()
     cache_backend = get_cache_backend(cache_name)
     ret = cache_backend.get(KEY_PREFIX + key)
@@ -153,31 +153,31 @@ def cache_get(key, cache_name=None):
     return ret
 
 def cache_get_many(keys, cache_name=None):
-    # type: (List[str], Optional[str]) -> Dict[str, Any]
+    # type: (List[text_type], Optional[str]) -> Dict[text_type, Any]
-    keys = [KEY_PREFIX + key for key in keys] # type: ignore # temporary
+    keys = [KEY_PREFIX + key for key in keys]
     remote_cache_stats_start()
     ret = get_cache_backend(cache_name).get_many(keys)
     remote_cache_stats_finish()
     return dict([(key[len(KEY_PREFIX):], value) for key, value in ret.items()])
 
 def cache_set_many(items, cache_name=None, timeout=None):
-    # type: (Dict[str, Any], Optional[str], Optional[int]) -> None
+    # type: (Dict[text_type, Any], Optional[str], Optional[int]) -> None
     new_items = {}
     for key in items:
         new_items[KEY_PREFIX + key] = items[key]
-    items = new_items # type: ignore # temporary
+    items = new_items
     remote_cache_stats_start()
     get_cache_backend(cache_name).set_many(items, timeout=timeout)
     remote_cache_stats_finish()
 
 def cache_delete(key, cache_name=None):
-    # type: (str, Optional[str]) -> None
+    # type: (text_type, Optional[str]) -> None
     remote_cache_stats_start()
     get_cache_backend(cache_name).delete(KEY_PREFIX + key)
     remote_cache_stats_finish()
 
 def cache_delete_many(items, cache_name=None):
-    # type: (Iterable[str], Optional[str]) -> None
+    # type: (Iterable[text_type], Optional[str]) -> None
     remote_cache_stats_start()
     get_cache_backend(cache_name).delete_many(
         KEY_PREFIX + item for item in items)
@@ -202,8 +202,8 @@ def generic_bulk_cached_fetch(cache_key_function, query_function, object_ids,
                               setter=lambda obj: obj,
                               id_fetcher=lambda obj: obj.id,
                               cache_transformer=lambda obj: obj):
-    # type: (Callable[[Any], str], Callable[[List[Any]], List[Any]], List[Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any]) -> Dict[Any, Any]
+    # type: (Callable[[Any], text_type], Callable[[List[Any]], Iterable[Any]], Iterable[Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any], Callable[[Any], Any]) -> Dict[Any, Any]
-    cache_keys = {} # type: Dict[int, str]
+    cache_keys = {} # type: Dict[Any, text_type]
     for object_id in object_ids:
         cache_keys[object_id] = cache_key_function(object_id)
     cached_objects = cache_get_many([cache_keys[object_id]
@@ -214,7 +214,7 @@ def generic_bulk_cached_fetch(cache_key_function, query_function, object_ids,
                   cache_keys[object_id] not in cached_objects]
     db_objects = query_function(needed_ids)
 
-    items_for_remote_cache = {} # type: Dict[str, Any]
+    items_for_remote_cache = {} # type: Dict[text_type, Any]
     for obj in db_objects:
         key = cache_keys[id_fetcher(obj)]
         item = cache_transformer(obj)
@@ -244,23 +244,23 @@ def cache(func):
     return cache_with_key(keyfunc)(func)
 
 def message_cache_key(message_id):
-    # type: (int) -> str
+    # type: (int) -> text_type
-    return "message:%d" % (message_id,)
+    return u"message:%d" % (message_id,)
 
 def display_recipient_cache_key(recipient_id):
-    # type: (int) -> str
+    # type: (int) -> text_type
-    return "display_recipient_dict:%d" % (recipient_id,)
+    return u"display_recipient_dict:%d" % (recipient_id,)
 
 def user_profile_by_email_cache_key(email):
-    # type: (str) -> str
+    # type: (text_type) -> text_type
     # See the comment in zerver/lib/avatar.py:gravatar_hash for why we
     # are proactively encoding email addresses even though they will
     # with high likelihood be ASCII-only for the foreseeable future.
-    return 'user_profile_by_email:%s' % (make_safe_digest(email.strip()),)
+    return u'user_profile_by_email:%s' % (make_safe_digest(email.strip()),)
 
 def user_profile_by_id_cache_key(user_profile_id):
-    # type: (int) -> str
+    # type: (int) -> text_type
-    return "user_profile_by_id:%s" % (user_profile_id,)
+    return u"user_profile_by_id:%s" % (user_profile_id,)
 
 # TODO: Refactor these cache helpers into another file that can import
 # models.py so that we can replace many of these type: Anys
@@ -271,8 +271,8 @@ def cache_save_user_profile(user_profile):
 
 active_user_dict_fields = ['id', 'full_name', 'short_name', 'email', 'is_realm_admin', 'is_bot'] # type: List[str]
 def active_user_dicts_in_realm_cache_key(realm):
-    # type: (Any) -> str
+    # type: (Any) -> text_type
-    return "active_user_dicts_in_realm:%s" % (realm.id,)
+    return u"active_user_dicts_in_realm:%s" % (realm.id,)
 
 active_bot_dict_fields = ['id', 'full_name', 'short_name',
                           'email', 'default_sending_stream__name',
@@ -280,17 +280,17 @@ active_bot_dict_fields = ['id', 'full_name', 'short_name',
                           'default_all_public_streams', 'api_key',
                           'bot_owner__email', 'avatar_source'] # type: List[str]
 def active_bot_dicts_in_realm_cache_key(realm):
-    # type: (Any) -> str
+    # type: (Any) -> text_type
-    return "active_bot_dicts_in_realm:%s" % (realm.id,)
+    return u"active_bot_dicts_in_realm:%s" % (realm.id,)
 
 def get_stream_cache_key(stream_name, realm):
-    # type: (six.text_type, Any) -> str
+    # type: (text_type, Any) -> text_type
     from zerver.models import Realm
     if isinstance(realm, Realm):
         realm_id = realm.id
     else:
         realm_id = realm
-    return "stream_by_realm_and_name:%s:%s" % (
+    return u"stream_by_realm_and_name:%s:%s" % (
         realm_id, make_safe_digest(stream_name.strip().lower()))
 
 def update_user_profile_caches(user_profiles):
@@ -341,8 +341,8 @@ def flush_realm(sender, **kwargs):
         cache_delete(realm_alert_words_cache_key(realm))
 
 def realm_alert_words_cache_key(realm):
-    # type: (Any) -> str
+    # type: (Any) -> text_type
-    return "realm_alert_words:%s" % (realm.domain,)
+    return u"realm_alert_words:%s" % (realm.domain,)
 
 # Called by models.py to flush the stream cache whenever we save a stream
 # object.

zerver/lib/cache_helpers.py

@@ -1,4 +1,6 @@
 from __future__ import absolute_import
+
+from six import text_type
 from typing import Any, Dict, Callable, Tuple
 
 # This file needs to be different from cache.py because cache.py
@@ -37,33 +39,33 @@ def message_fetch_objects():
                                                     id__gt=max_id - MESSAGE_CACHE_SIZE)
 
 def message_cache_items(items_for_remote_cache, message):
-    # type: (Dict[str, Tuple[Message]], Message) -> None
+    # type: (Dict[text_type, Tuple[Message]], Message) -> None
     items_for_remote_cache[message_cache_key(message.id)] = (message,)
 
 def user_cache_items(items_for_remote_cache, user_profile):
-    # type: (Dict[str, Tuple[UserProfile]], UserProfile) -> None
+    # type: (Dict[text_type, Tuple[UserProfile]], UserProfile) -> None
     items_for_remote_cache[user_profile_by_email_cache_key(user_profile.email)] = (user_profile,)
     items_for_remote_cache[user_profile_by_id_cache_key(user_profile.id)] = (user_profile,)
 
 def stream_cache_items(items_for_remote_cache, stream):
-    # type: (Dict[str, Tuple[Stream]], Stream) -> None
+    # type: (Dict[text_type, Tuple[Stream]], Stream) -> None
     items_for_remote_cache[get_stream_cache_key(stream.name, stream.realm_id)] = (stream,)
 
 def client_cache_items(items_for_remote_cache, client):
-    # type: (Dict[str, Tuple[Client]], Client) -> None
+    # type: (Dict[text_type, Tuple[Client]], Client) -> None
     items_for_remote_cache[get_client_cache_key(client.name)] = (client,)
 
 def huddle_cache_items(items_for_remote_cache, huddle):
-    # type: (Dict[str, Tuple[Huddle]], Huddle) -> None
+    # type: (Dict[text_type, Tuple[Huddle]], Huddle) -> None
     items_for_remote_cache[huddle_hash_cache_key(huddle.huddle_hash)] = (huddle,)
 
 def recipient_cache_items(items_for_remote_cache, recipient):
-    # type: (Dict[str, Tuple[Recipient]], Recipient) -> None
+    # type: (Dict[text_type, Tuple[Recipient]], Recipient) -> None
     items_for_remote_cache[get_recipient_cache_key(recipient.type, recipient.type_id)] = (recipient,)
 
 session_engine = import_module(settings.SESSION_ENGINE)
 def session_cache_items(items_for_remote_cache, session):
-    # type: (Dict[str, str], Session) -> None
+    # type: (Dict[text_type, text_type], Session) -> None
     store = session_engine.SessionStore(session_key=session.session_key)
     items_for_remote_cache[store.cache_key] = store.decode(session.session_data)
 
@@ -81,13 +83,13 @@ cache_fillers = {
     'message': (message_fetch_objects, message_cache_items, 3600 * 24, 1000),
     'huddle': (lambda: Huddle.objects.select_related().all(), huddle_cache_items, 3600*24*7, 10000),
     'session': (lambda: Session.objects.all(), session_cache_items, 3600*24*7, 10000),
-    } # type: Dict[str, Tuple[Callable[[], List[Any]], Callable[[Dict[str, Any], Any], None], int, int]]
+    } # type: Dict[str, Tuple[Callable[[], List[Any]], Callable[[Dict[text_type, Any], Any], None], int, int]]
 
 def fill_remote_cache(cache):
     # type: (str) -> None
     remote_cache_time_start = get_remote_cache_time()
     remote_cache_requests_start = get_remote_cache_requests()
-    items_for_remote_cache = {} # type: Dict[str, Any]
+    items_for_remote_cache = {} # type: Dict[text_type, Any]
     (objects, items_filler, timeout, batch_size) = cache_fillers[cache]
     count = 0
     for obj in objects():

zerver/lib/test_helpers.py

@@ -71,15 +71,15 @@ def tornado_redirected_to_list(lst):
 
 @contextmanager
 def simulated_empty_cache():
-    # type: () -> Generator[List[Tuple[str, Union[str, List[str]], str]], None, None]
+    # type: () -> Generator[List[Tuple[str, Union[text_type, List[text_type]], text_type]], None, None]
-    cache_queries = [] # type: List[Tuple[str, Union[str, List[str]], str]]
+    cache_queries = [] # type: List[Tuple[str, Union[text_type, List[text_type]], text_type]]
     def my_cache_get(key, cache_name=None):
-        # type: (str, Optional[str]) -> Any
+        # type: (text_type, Optional[str]) -> Any
         cache_queries.append(('get', key, cache_name))
         return None
 
     def my_cache_get_many(keys, cache_name=None):
-        # type: (List[str], Optional[str]) -> Dict[str, Any]
+        # type: (List[text_type], Optional[str]) -> Dict[text_type, Any]
         cache_queries.append(('getmany', keys, cache_name))
         return None
 

zerver/models.py

@@ -45,7 +45,7 @@ bugdown = None # type: Any
 MAX_SUBJECT_LENGTH = 60
 MAX_MESSAGE_LENGTH = 10000
 
-STREAM_NAMES = TypeVar('STREAM_NAMES', Sequence[str], AbstractSet[str])
+STREAM_NAMES = TypeVar('STREAM_NAMES', Sequence[text_type], AbstractSet[text_type])
 
 # Doing 1000 remote cache requests to get_display_recipient is quite slow,
 # so add a local cache as well as the remote cache cache.
@@ -701,7 +701,7 @@ def bulk_get_streams(realm, stream_names):
         realm_id = realm
 
     def fetch_streams_by_name(stream_names):
-        # type: (List[str]) -> List[str]
+        # type: (List[text_type]) -> Sequence[Stream]
         #
         # This should be just
         #
@@ -758,11 +758,11 @@ def stringify_message_dict(message_dict):
     return zlib.compress(force_bytes(ujson.dumps(message_dict)))
 
 def to_dict_cache_key_id(message_id, apply_markdown):
-    # type: (int, bool) -> str
+    # type: (int, bool) -> text_type
-    return 'message_dict:%d:%d' % (message_id, apply_markdown)
+    return u'message_dict:%d:%d' % (message_id, apply_markdown)
 
 def to_dict_cache_key(message, apply_markdown):
-    # type: (Message, bool) -> str
+    # type: (Message, bool) -> text_type
     return to_dict_cache_key_id(message.id, apply_markdown)
 
 class Message(models.Model):

zerver/views/messages.py

@@ -552,7 +552,7 @@ def get_old_messages_backend(request, user_profile,
     # rendered message dict before returning it.  We attempt to
     # bulk-fetch rendered message dicts from remote cache using the
     # 'messages' list.
-    search_fields = dict() # type: Dict[int, Dict[str, str]]
+    search_fields = dict() # type: Dict[int, Dict[str, text_type]]
     message_ids = [] # type: List[int]
     user_message_flags = {} # type: Dict[int, List[str]]
     if include_history: