memcached: Switch from pylibmc to python-binary-memcached.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

docs/development/setup-advanced.md

@@ -177,7 +177,7 @@ git remote add -f upstream https://github.com/zulip/zulip.git
 
 ```
 doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \
-    memcached libmemcached py-Pillow py-cryptography py-cffi
+    memcached py-Pillow py-cryptography py-cffi
 
 # Point environment to custom include locations and use newer GCC
 # (needed for Node modules):

puppet/zulip/manifests/app_frontend_base.pp

@@ -167,15 +167,4 @@ class zulip::app_frontend_base {
     mode    => '0755',
     source  => 'puppet:///modules/zulip/nagios_plugins/zulip_app_frontend',
   }
-
-  if $::osfamily == 'debian' {
-    # The pylibmc wheel looks for SASL plugins in the wrong place.
-    file { '/usr/lib64':
-      ensure => directory,
-    }
-    file { '/usr/lib64/sasl2':
-      ensure => link,
-      target => "/usr/lib/${::rubyplatform}/sasl2",
-    }
-  }
 }

requirements/common.in

@@ -79,10 +79,11 @@ pika
 psycopg2 --no-binary psycopg2
 
 # Needed for memcached usage
-pylibmc
+# https://github.com/jaysonsantos/python-binary-memcached/pull/230, https://github.com/jaysonsantos/python-binary-memcached/pull/231
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git
 
-# Needed for compression support in memcached via pylibmc
+# Needed for compression support in memcached via python-binary-memcached
-django-pylibmc
+django-bmemcached
 
 # Needed for zerver/tests/test_timestamp.py
 python-dateutil

requirements/dev.txt

@@ -277,6 +277,9 @@ django-bitfield==2.0.1 \
     --hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
     --hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
     # via -r requirements/common.in
+django-bmemcached==0.3.0 \
+    --hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
+    # via -r requirements/common.in
 django-cookies-samesite==0.6.6 \
     --hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
     # via -r requirements/common.in
@@ -292,10 +295,6 @@ django-phonenumber-field==3.0.1 \
     --hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
     --hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
     # via django-two-factor-auth
-django-pylibmc==0.6.1 \
-    --hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
-    --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
-    # via -r requirements/common.in
 django-sendfile2==0.6.0 \
     --hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
     # via -r requirements/common.in
@@ -787,13 +786,6 @@ pyjwt==1.7.1 \
     --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
     --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
     # via -r requirements/common.in, apns2, social-auth-core, twilio
-pylibmc==1.6.1 \
-    --hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
-    --hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
-    --hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
-    --hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
-    --hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
-    # via -r requirements/common.in, django-pylibmc
 pyoembed==0.1.2 \
     --hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
     # via -r requirements/common.in
@@ -808,6 +800,9 @@ pyparsing==2.4.7 \
 pyrsistent==0.16.0 \
     --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
     # via jsonschema
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
+    --hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
+    # via -r requirements/common.in, django-bmemcached
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@@ -971,7 +966,7 @@ sh==1.12.14 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
+    # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-binary-memcached, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
 snakeviz==2.1.0 \
     --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
     --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
@@ -1163,6 +1158,9 @@ ua-parser==0.10.0 \
     --hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
     --hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
     # via django-cookies-samesite
+uhashring==1.2 \
+    --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
+    # via python-binary-memcached
 https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
     --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
     # via -r requirements/common.in

requirements/prod.txt

@@ -185,6 +185,9 @@ django-bitfield==2.0.1 \
     --hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
     --hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
     # via -r requirements/common.in
+django-bmemcached==0.3.0 \
+    --hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
+    # via -r requirements/common.in
 django-cookies-samesite==0.6.6 \
     --hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
     # via -r requirements/common.in
@@ -200,10 +203,6 @@ django-phonenumber-field==3.0.1 \
     --hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
     --hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
     # via django-two-factor-auth
-django-pylibmc==0.6.1 \
-    --hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
-    --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
-    # via -r requirements/common.in
 django-sendfile2==0.6.0 \
     --hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
     # via -r requirements/common.in
@@ -562,13 +561,6 @@ pyjwt==1.7.1 \
     --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
     --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
     # via -r requirements/common.in, apns2, social-auth-core, twilio
-pylibmc==1.6.1 \
-    --hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
-    --hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
-    --hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
-    --hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
-    --hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
-    # via -r requirements/common.in, django-pylibmc
 pyoembed==0.1.2 \
     --hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
     # via -r requirements/common.in
@@ -579,6 +571,9 @@ pyopenssl==19.1.0 \
 pyrsistent==0.16.0 \
     --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
     # via jsonschema
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
+    --hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
+    # via -r requirements/common.in, django-bmemcached
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@@ -673,7 +668,7 @@ sentry-sdk==0.16.2 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
+    # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-binary-memcached, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
 social-auth-app-django==4.0.0 \
     --hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \
     --hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \
@@ -761,6 +756,9 @@ ua-parser==0.10.0 \
     --hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
     --hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
     # via django-cookies-samesite
+uhashring==1.2 \
+    --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
+    # via python-binary-memcached
 https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
     --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
     # via -r requirements/common.in

scripts/lib/setup-yum-repo

@@ -14,7 +14,6 @@ done
 
 is_centos=false
 is_rhel=false
-is_rhel_registered=false
 if [ -e /etc/centos-release ]; then
     is_centos=true
     yum install -y epel-release
@@ -27,12 +26,6 @@ if [ -e /etc/centos-release ]; then
 elif grep -q "Red Hat" /etc/redhat-release; then
     is_rhel=true
     yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    if subscription-manager status; then
-        # See https://access.redhat.com/discussions/2217891#comment-1032701
-        is_rhel_registered=true
-        # libmemcached-devel can be installed directly if the machine is registered
-        subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"
-    fi
 fi
 
 yum update -y
@@ -51,10 +44,6 @@ if [ "$is_centos" = true ]; then
     # https://pgroonga.github.io/install/centos.html
     yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
 elif [ "$is_rhel" = true ]; then
-    if [ "$is_rhel_registered" = false ]; then
-        echo "This machine is unregistered; installing libmemcached-devel from a CentOS mirror ..."
-        yum localinstall -y http://mirror.centos.org/centos/7/os/x86_64/Packages/libmemcached-devel-1.0.16-5.el7.x86_64.rpm
-    fi
     yum localinstall -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-latest-x86_64/pgdg-redhat10-10-2.noarch.rpm
     yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
 else

scripts/lib/setup_venv.py

@@ -17,7 +17,6 @@ VENV_DEPENDENCIES = [
     "zlib1g-dev",             # Needed to handle compressed PNGs with Pillow
     "libjpeg-dev",          # Needed to handle JPEGs with Pillow
     "libldap2-dev",
-    "libmemcached-dev",
     "python3-dev",          # Needed to install typed-ast dependency of mypy
     "python3-pip",
     "virtualenv",
@@ -43,7 +42,6 @@ COMMON_YUM_VENV_DEPENDENCIES = [
     "zlib-devel",
     "libjpeg-turbo-devel",
     "openldap-devel",
-    "libmemcached-devel",
     # Needed by python-xmlsec:
     "gcc"
     "python3-devel",

scripts/setup/flush-memcached

@@ -9,15 +9,10 @@ from scripts.lib.setup_path import setup_path
 
 setup_path()
 
-import pylibmc
+import bmemcached
 
 from zproject import settings
 
-assert isinstance(settings.CACHES["default"], dict)  # for mypy
+cache = settings.CACHES["default"]
-pylibmc.Client(
+assert isinstance(cache, dict)  # for mypy
-    [settings.MEMCACHED_LOCATION],
+bmemcached.Client((cache["LOCATION"],), **cache["OPTIONS"]).flush_all()
-    binary=True,
-    username=settings.MEMCACHED_USERNAME,
-    password=settings.MEMCACHED_PASSWORD,
-    behaviors=settings.CACHES["default"]["OPTIONS"],
-).flush_all()

tools/ci/Dockerfile.template

@@ -92,7 +92,7 @@ RUN apt-get update \
     memcached rabbitmq-server redis-server \
     hunspell-en-us supervisor libssl-dev puppet \
     gettext libffi-dev libfreetype6-dev zlib1g-dev \
-    libjpeg-dev libldap2-dev libmemcached-dev \
+    libjpeg-dev libldap2-dev \
     libxml2-dev libxslt1-dev libpq-dev moreutils
 
 # Upgrade git if it is less than v2.18 because GitHub Actions'

tools/test-install/prepare-base

@@ -52,7 +52,7 @@ run apt-get install -y --no-install-recommends \
   memcached redis-server \
   hunspell-en-us supervisor libssl-dev puppet \
   gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
-  libldap2-dev libmemcached-dev \
+  libldap2-dev \
   libxml2-dev libxslt1-dev libpq-dev \
   virtualenv \
   "${extra_packages[@]}"

version.py

@@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '95.3'
+PROVISION_VERSION = '96.0'

zilencer/management/commands/populate_db.py

@@ -4,7 +4,7 @@ import random
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple
 
-import pylibmc
+import bmemcached
 import ujson
 from django.conf import settings
 from django.contrib.sessions.models import Session
@@ -79,13 +79,9 @@ def clear_database() -> None:
     # With `zproject.test_settings`, we aren't using real memcached
     # and; we only need to flush memcached if we're populating a
     # database that would be used with it (i.e. zproject.dev_settings).
-    if default_cache['BACKEND'] == 'django_pylibmc.memcached.PyLibMCCache':
+    if default_cache['BACKEND'] == 'django_bmemcached.memcached.BMemcached':
-        pylibmc.Client(
+        bmemcached.Client(
-            [default_cache['LOCATION']],
+            (default_cache['LOCATION'],), **default_cache['OPTIONS'],
-            binary=True,
-            username=default_cache["USERNAME"],
-            password=default_cache["PASSWORD"],
-            behaviors=default_cache["OPTIONS"],
         ).flush_all()
 
     model: Any = None  # Hack because mypy doesn't know these are model classes

zproject/computed_settings.py

@@ -327,24 +327,17 @@ RABBITMQ_PASSWORD = get_secret("rabbitmq_password")
 
 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
 
-# Compress large values being stored in memcached; this is important
-# for at least the realm_users cache.
-PYLIBMC_MIN_COMPRESS_LEN = 100 * 1024
-PYLIBMC_COMPRESS_LEVEL = 1
-
 MEMCACHED_PASSWORD = get_secret("memcached_password")
 
 CACHES = {
     'default': {
-        'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
+        'BACKEND': 'django_bmemcached.memcached.BMemcached',
         'LOCATION': MEMCACHED_LOCATION,
-        'TIMEOUT': 3600,
-        'BINARY': True,
-        'USERNAME': MEMCACHED_USERNAME,
-        'PASSWORD': MEMCACHED_PASSWORD,
         'OPTIONS': {
-            'tcp_nodelay': True,
+            'socket_timeout': 3600,
-            'retry_timeout': 1,
+            'username': MEMCACHED_USERNAME,
+            'password': MEMCACHED_PASSWORD,
+            'pickle_protocol': 4,
         },
     },
     'database': {