From c9faefd50e96aa460e77485e83c0a0f6fc4a8672 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Mon, 4 Apr 2022 18:27:33 -0700 Subject: [PATCH] cache: Instantiate only one BMemcached cache backend. Signed-off-by: Anders Kaseorg --- pyproject.toml | 1 + tools/test-backend | 1 + zerver/lib/singleton_bmemcached.py | 19 +++++++++++++++++++ zilencer/management/commands/populate_db.py | 2 +- zproject/computed_settings.py | 2 +- 5 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 zerver/lib/singleton_bmemcached.py diff --git a/pyproject.toml b/pyproject.toml index 72e61d60d1..5e26a0d322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ module = [ "disposable_email_domains.*", "django.*", "django_auth_ldap.*", + "django_bmemcached.*", "django_cte.*", "django_otp.*", "django_scim.*", diff --git a/tools/test-backend b/tools/test-backend index fb10db2041..97491d206e 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -140,6 +140,7 @@ not_yet_fully_covered = [ "zerver/webhooks/zapier/view.py", # Cannot have coverage, as tests run in a transaction "zerver/lib/safe_session_cached_db.py", + "zerver/lib/singleton_bmemcached.py", ] enforce_fully_covered = sorted( diff --git a/zerver/lib/singleton_bmemcached.py b/zerver/lib/singleton_bmemcached.py new file mode 100644 index 0000000000..79362f3317 --- /dev/null +++ b/zerver/lib/singleton_bmemcached.py @@ -0,0 +1,19 @@ +import pickle +from functools import lru_cache +from typing import Any, Dict + +from django_bmemcached.memcached import BMemcached + + +@lru_cache(None) +def _get_bmemcached(location: str, params: bytes) -> BMemcached: + return BMemcached(location, pickle.loads(params)) + + +def SingletonBMemcached(location: str, params: Dict[str, Any]) -> BMemcached: + # Django instantiates the cache backend per-task to guard against + # thread safety issues, but BMemcached is already thread-safe and + # does its own per-thread pooling, so make sure we instantiate only + # one to avoid extra connections. + + return _get_bmemcached(location, pickle.dumps(params)) diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 8f0d6868f1..3ddfaddd92 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -96,7 +96,7 @@ def clear_database() -> None: # With `zproject.test_settings`, we aren't using real memcached # and; we only need to flush memcached if we're populating a # database that would be used with it (i.e. zproject.dev_settings). - if default_cache["BACKEND"] == "django_bmemcached.memcached.BMemcached": + if default_cache["BACKEND"] == "zerver.lib.singleton_bmemcached.SingletonBMemcached": bmemcached.Client( (default_cache["LOCATION"],), **default_cache["OPTIONS"], diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 60f372dc81..81bd937992 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -340,7 +340,7 @@ MEMCACHED_PASSWORD = get_secret("memcached_password") CACHES = { "default": { - "BACKEND": "django_bmemcached.memcached.BMemcached", + "BACKEND": "zerver.lib.singleton_bmemcached.SingletonBMemcached", "LOCATION": MEMCACHED_LOCATION, "OPTIONS": { "socket_timeout": 3600,