Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

memcached: Switch from pylibmc to python-binary-memcached. 

Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg 2020-05-09 13:23:33 -07:00 committed by Tim Abbott
parent 15ffd2b666
commit dbdf67301b
13 changed files with 41 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/development/setup-advanced.md
View File

@ -177,7 +177,7 @@ git remote add -f upstream https://github.com/zulip/zulip.git
```
doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \
memcached libmemcached py-Pillow py-cryptography py-cffi
memcached py-Pillow py-cryptography py-cffi
# Point environment to custom include locations and use newer GCC
# (needed for Node modules):

11
puppet/zulip/manifests/app_frontend_base.pp
View File

@ -167,15 +167,4 @@ class zulip::app_frontend_base {
mode => '0755',
source => 'puppet:///modules/zulip/nagios_plugins/zulip_app_frontend',
}
if $::osfamily == 'debian' {
# The pylibmc wheel looks for SASL plugins in the wrong place.
file { '/usr/lib64':
ensure => directory,
}
file { '/usr/lib64/sasl2':
ensure => link,
target => "/usr/lib/${::rubyplatform}/sasl2",
}
}
}

7
requirements/common.in
View File

@ -79,10 +79,11 @@ pika
psycopg2 --no-binary psycopg2
# Needed for memcached usage
pylibmc
# https://github.com/jaysonsantos/python-binary-memcached/pull/230, https://github.com/jaysonsantos/python-binary-memcached/pull/231
https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git
# Needed for compression support in memcached via pylibmc
django-pylibmc
# Needed for compression support in memcached via python-binary-memcached
django-bmemcached
# Needed for zerver/tests/test_timestamp.py
python-dateutil

22
requirements/dev.txt
View File

@ -277,6 +277,9 @@ django-bitfield==2.0.1 \
--hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
--hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
# via -r requirements/common.in
django-bmemcached==0.3.0 \
--hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
# via -r requirements/common.in
django-cookies-samesite==0.6.6 \
--hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
# via -r requirements/common.in
@ -292,10 +295,6 @@ django-phonenumber-field==3.0.1 \
--hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
--hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
# via django-two-factor-auth
django-pylibmc==0.6.1 \
--hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
--hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
# via -r requirements/common.in
django-sendfile2==0.6.0 \
--hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
# via -r requirements/common.in
@ -787,13 +786,6 @@ pyjwt==1.7.1 \
--hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
--hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
# via -r requirements/common.in, apns2, social-auth-core, twilio
pylibmc==1.6.1 \
--hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
--hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
--hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
--hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
--hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
# via -r requirements/common.in, django-pylibmc
pyoembed==0.1.2 \
--hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
# via -r requirements/common.in
@ -808,6 +800,9 @@ pyparsing==2.4.7 \
pyrsistent==0.16.0 \
--hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
# via jsonschema
https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
--hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
# via -r requirements/common.in, django-bmemcached
python-dateutil==2.8.1 \
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@ -971,7 +966,7 @@ sh==1.12.14 \
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
# via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-binary-memcached, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
snakeviz==2.1.0 \
--hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
--hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
@ -1163,6 +1158,9 @@ ua-parser==0.10.0 \
--hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
--hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
# via django-cookies-samesite
uhashring==1.2 \
--hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
# via python-binary-memcached
https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
--hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
# via -r requirements/common.in

22
requirements/prod.txt
View File

@ -185,6 +185,9 @@ django-bitfield==2.0.1 \
--hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
--hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
# via -r requirements/common.in
django-bmemcached==0.3.0 \
--hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
# via -r requirements/common.in
django-cookies-samesite==0.6.6 \
--hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
# via -r requirements/common.in
@ -200,10 +203,6 @@ django-phonenumber-field==3.0.1 \
--hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
--hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
# via django-two-factor-auth
django-pylibmc==0.6.1 \
--hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
--hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
# via -r requirements/common.in
django-sendfile2==0.6.0 \
--hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
# via -r requirements/common.in
@ -562,13 +561,6 @@ pyjwt==1.7.1 \
--hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
--hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
# via -r requirements/common.in, apns2, social-auth-core, twilio
pylibmc==1.6.1 \
--hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
--hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
--hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
--hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
--hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
# via -r requirements/common.in, django-pylibmc
pyoembed==0.1.2 \
--hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
# via -r requirements/common.in
@ -579,6 +571,9 @@ pyopenssl==19.1.0 \
pyrsistent==0.16.0 \
--hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
# via jsonschema
https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
--hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
# via -r requirements/common.in, django-bmemcached
python-dateutil==2.8.1 \
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@ -673,7 +668,7 @@ sentry-sdk==0.16.2 \
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
# via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-binary-memcached, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
social-auth-app-django==4.0.0 \
--hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \
--hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \
@ -761,6 +756,9 @@ ua-parser==0.10.0 \
--hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
--hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
# via django-cookies-samesite
uhashring==1.2 \
--hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
# via python-binary-memcached
https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
--hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
# via -r requirements/common.in

11
scripts/lib/setup-yum-repo
View File

@ -14,7 +14,6 @@ done
is_centos=false
is_rhel=false
is_rhel_registered=false
if [ -e /etc/centos-release ]; then
is_centos=true
yum install -y epel-release
@ -27,12 +26,6 @@ if [ -e /etc/centos-release ]; then
elif grep -q "Red Hat" /etc/redhat-release; then
is_rhel=true
yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
if subscription-manager status; then
# See https://access.redhat.com/discussions/2217891#comment-1032701
is_rhel_registered=true
# libmemcached-devel can be installed directly if the machine is registered
subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"
fi
fi
yum update -y
@ -51,10 +44,6 @@ if [ "$is_centos" = true ]; then
# https://pgroonga.github.io/install/centos.html
yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
elif [ "$is_rhel" = true ]; then
if [ "$is_rhel_registered" = false ]; then
echo "This machine is unregistered; installing libmemcached-devel from a CentOS mirror ..."
yum localinstall -y http://mirror.centos.org/centos/7/os/x86_64/Packages/libmemcached-devel-1.0.16-5.el7.x86_64.rpm
fi
yum localinstall -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-latest-x86_64/pgdg-redhat10-10-2.noarch.rpm
yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
else

2
scripts/lib/setup_venv.py
View File

@ -17,7 +17,6 @@ VENV_DEPENDENCIES = [
"zlib1g-dev", # Needed to handle compressed PNGs with Pillow
"libjpeg-dev", # Needed to handle JPEGs with Pillow
"libldap2-dev",
"libmemcached-dev",
"python3-dev", # Needed to install typed-ast dependency of mypy
"python3-pip",
"virtualenv",
@ -43,7 +42,6 @@ COMMON_YUM_VENV_DEPENDENCIES = [
"zlib-devel",
"libjpeg-turbo-devel",
"openldap-devel",
"libmemcached-devel",
# Needed by python-xmlsec:
"gcc"
"python3-devel",

13
scripts/setup/flush-memcached
View File

@ -9,15 +9,10 @@ from scripts.lib.setup_path import setup_path
setup_path()
import pylibmc
import bmemcached
from zproject import settings
assert isinstance(settings.CACHES["default"], dict) # for mypy
pylibmc.Client(
[settings.MEMCACHED_LOCATION],
binary=True,
username=settings.MEMCACHED_USERNAME,
password=settings.MEMCACHED_PASSWORD,
behaviors=settings.CACHES["default"]["OPTIONS"],
).flush_all()
cache = settings.CACHES["default"]
assert isinstance(cache, dict) # for mypy
bmemcached.Client((cache["LOCATION"],), **cache["OPTIONS"]).flush_all()

2
tools/ci/Dockerfile.template
View File

@ -92,7 +92,7 @@ RUN apt-get update \
memcached rabbitmq-server redis-server \
hunspell-en-us supervisor libssl-dev puppet \
gettext libffi-dev libfreetype6-dev zlib1g-dev \
libjpeg-dev libldap2-dev libmemcached-dev \
libjpeg-dev libldap2-dev \
libxml2-dev libxslt1-dev libpq-dev moreutils
# Upgrade git if it is less than v2.18 because GitHub Actions'

2
tools/test-install/prepare-base
View File

@ -52,7 +52,7 @@ run apt-get install -y --no-install-recommends \
memcached redis-server \
hunspell-en-us supervisor libssl-dev puppet \
gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
libldap2-dev libmemcached-dev \
libldap2-dev \
libxml2-dev libxslt1-dev libpq-dev \
virtualenv \
"${extra_packages[@]}"

2
version.py
View File

@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27
# historical commits sharing the same major version, in which case a
# minor version bump suffices.
PROVISION_VERSION = '95.3'
PROVISION_VERSION = '96.0'

12
zilencer/management/commands/populate_db.py
View File

@ -4,7 +4,7 @@ import random
from datetime import datetime
from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple
import pylibmc
import bmemcached
import ujson
from django.conf import settings
from django.contrib.sessions.models import Session
@ -79,13 +79,9 @@ def clear_database() -> None:
# With `zproject.test_settings`, we aren't using real memcached
# and; we only need to flush memcached if we're populating a
# database that would be used with it (i.e. zproject.dev_settings).
if default_cache['BACKEND'] == 'django_pylibmc.memcached.PyLibMCCache':
pylibmc.Client(
[default_cache['LOCATION']],
binary=True,
username=default_cache["USERNAME"],
password=default_cache["PASSWORD"],
behaviors=default_cache["OPTIONS"],
if default_cache['BACKEND'] == 'django_bmemcached.memcached.BMemcached':
bmemcached.Client(
(default_cache['LOCATION'],), **default_cache['OPTIONS'],
).flush_all()
model: Any = None # Hack because mypy doesn't know these are model classes

17
zproject/computed_settings.py
View File

@ -327,24 +327,17 @@ RABBITMQ_PASSWORD = get_secret("rabbitmq_password")
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
# Compress large values being stored in memcached; this is important
# for at least the realm_users cache.
PYLIBMC_MIN_COMPRESS_LEN = 100 * 1024
PYLIBMC_COMPRESS_LEVEL = 1
MEMCACHED_PASSWORD = get_secret("memcached_password")
CACHES = {
'default': {
'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
'BACKEND': 'django_bmemcached.memcached.BMemcached',
'LOCATION': MEMCACHED_LOCATION,
'TIMEOUT': 3600,
'BINARY': True,
'USERNAME': MEMCACHED_USERNAME,
'PASSWORD': MEMCACHED_PASSWORD,
'OPTIONS': {
'tcp_nodelay': True,
'retry_timeout': 1,
'socket_timeout': 3600,
'username': MEMCACHED_USERNAME,
'password': MEMCACHED_PASSWORD,
'pickle_protocol': 4,
},
},
'database': {