From eda9ce23640bb8ef73c6d6640d4fe1279cff93fa Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Fri, 30 Apr 2021 12:57:25 -0700 Subject: [PATCH] locale: Use `C.UTF-8` rather than `en_US.UTF-8`. The `en_US.UTF-8` locale may not be configured or generated on all installs; it also requires that the `locales` package be installed. If users generate the `en_US.UTF-8` locale without adding it to the permanent set of system locales, the generated `en_US.UTF-8` stops working when the `locales` package is updated. Switch to using `C.UTF-8` in all cases, which is guaranteed to be installed. Fixes #15819. --- Vagrantfile | 4 ++-- .../postgresql/13/postgresql.conf.template.erb | 8 ++++---- .../postgresql/9.3/postgresql.conf.template.erb | 8 ++++---- .../postgresql/9.5/postgresql.conf.template.erb | 8 ++++---- .../postgresql/9.6/postgresql.conf.template.erb | 8 ++++---- puppet/zulip/templates/uwsgi.ini.template.erb | 2 +- scripts/lib/install | 7 +++---- scripts/lib/upgrade-zulip-stage-2 | 6 +++--- scripts/setup/create-db.sql | 4 ++-- tools/ci/Dockerfile.template | 13 ++----------- tools/setup/dev-vagrant-docker/Dockerfile | 4 ++-- tools/test-install/prepare-base | 4 ++-- tools/test-js-with-puppeteer | 2 +- 13 files changed, 34 insertions(+), 44 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index e4d938ce49..14cc0af078 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -166,8 +166,8 @@ sudo dpkg-divert --add --rename /etc/default/motd-news sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news' # Set default locale, this prevents errors if the user has another locale set. -if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then - echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale +if ! grep -q 'LC_ALL=C.UTF-8' /etc/default/locale; then + echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale fi # Set an environment variable, so that we won't print the virtualenv diff --git a/puppet/zulip/templates/postgresql/13/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/13/postgresql.conf.template.erb index f565159f58..1b3aef7b36 100644 --- a/puppet/zulip/templates/postgresql/13/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/13/postgresql.conf.template.erb @@ -691,11 +691,11 @@ timezone = 'Etc/UTC' # encoding # These settings are initialized by initdb, but they can be changed. -lc_messages = 'en_US.UTF-8' # locale for system error message +lc_messages = 'C.UTF-8' # locale for system error message # strings -lc_monetary = 'en_US.UTF-8' # locale for monetary formatting -lc_numeric = 'en_US.UTF-8' # locale for number formatting -lc_time = 'en_US.UTF-8' # locale for time formatting +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting # default configuration for text search default_text_search_config = 'pg_catalog.english' diff --git a/puppet/zulip/templates/postgresql/9.3/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/9.3/postgresql.conf.template.erb index 0d76992c63..b9b83c9975 100644 --- a/puppet/zulip/templates/postgresql/9.3/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/9.3/postgresql.conf.template.erb @@ -492,11 +492,11 @@ datestyle = 'iso, mdy' # encoding # These settings are initialized by initdb, but they can be changed. -lc_messages = 'en_US.UTF-8' # locale for system error message +lc_messages = 'C.UTF-8' # locale for system error message # strings -lc_monetary = 'en_US.UTF-8' # locale for monetary formatting -lc_numeric = 'en_US.UTF-8' # locale for number formatting -lc_time = 'en_US.UTF-8' # locale for time formatting +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting # default configuration for text search #default_text_search_config = 'pg_catalog.english' diff --git a/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb index 500e78cc46..5c786cd63a 100644 --- a/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb @@ -549,11 +549,11 @@ timezone = 'localtime' # encoding # These settings are initialized by initdb, but they can be changed. -lc_messages = 'en_US.UTF-8' # locale for system error message +lc_messages = 'C.UTF-8' # locale for system error message # strings -lc_monetary = 'en_US.UTF-8' # locale for monetary formatting -lc_numeric = 'en_US.UTF-8' # locale for number formatting -lc_time = 'en_US.UTF-8' # locale for time formatting +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting # default configuration for text search default_text_search_config = 'pg_catalog.english' diff --git a/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb index b6aa655f27..acc36cec9b 100644 --- a/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb @@ -565,11 +565,11 @@ timezone = 'UTC' # encoding # These settings are initialized by initdb, but they can be changed. -lc_messages = 'en_US.UTF-8' # locale for system error message +lc_messages = 'C.UTF-8' # locale for system error message # strings -lc_monetary = 'en_US.UTF-8' # locale for monetary formatting -lc_numeric = 'en_US.UTF-8' # locale for number formatting -lc_time = 'en_US.UTF-8' # locale for time formatting +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting # default configuration for text search default_text_search_config = 'pg_catalog.english' diff --git a/puppet/zulip/templates/uwsgi.ini.template.erb b/puppet/zulip/templates/uwsgi.ini.template.erb index da9b415c80..acea04cca9 100644 --- a/puppet/zulip/templates/uwsgi.ini.template.erb +++ b/puppet/zulip/templates/uwsgi.ini.template.erb @@ -10,7 +10,7 @@ harakiri=20 buffer-size=<%= @uwsgi_buffer_size %> listen=<%= @uwsgi_listen_backlog_limit %> post-buffering=4096 -env= LANG=en_US.UTF-8 +env= LANG=C.UTF-8 uid=zulip gid=zulip diff --git a/scripts/lib/install b/scripts/lib/install index 2afbe02fd7..4bb77e2b85 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -168,10 +168,9 @@ set -x ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)" # Force a known locale. Some packages on PyPI fail to install in some locales. -localedef -i en_US -f UTF-8 en_US.UTF-8 -export LC_ALL="en_US.UTF-8" -export LANG="en_US.UTF-8" -export LANGUAGE="en_US.UTF-8" +export LC_ALL="C.UTF-8" +export LANG="C.UTF-8" +export LANGUAGE="C.UTF-8" # Force a known path; this fixes problems on Debian where `su` from # non-root may not adjust `$PATH` to root's. diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index 50b85379e1..668d2334b6 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -21,9 +21,9 @@ if TYPE_CHECKING: os.environ["PYTHONUNBUFFERED"] = "y" # Force a known locale. Some packages on PyPI fail to install in some locales. -os.environ["LC_ALL"] = "en_US.UTF-8" -os.environ["LANG"] = "en_US.UTF-8" -os.environ["LANGUAGE"] = "en_US.UTF-8" +os.environ["LC_ALL"] = "C.UTF-8" +os.environ["LANG"] = "C.UTF-8" +os.environ["LANGUAGE"] = "C.UTF-8" sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from scripts.lib.zulip_tools import ( diff --git a/scripts/setup/create-db.sql b/scripts/setup/create-db.sql index 734a721ea5..7b63f10f6c 100644 --- a/scripts/setup/create-db.sql +++ b/scripts/setup/create-db.sql @@ -9,8 +9,8 @@ ALTER ROLE zulip SET search_path TO zulip,public; CREATE DATABASE zulip OWNER=zulip ENCODING=UTF8 - LC_COLLATE='en_US.UTF-8' - LC_CTYPE='en_US.UTF-8' + LC_COLLATE='C.UTF-8' + LC_CTYPE='C.UTF-8' TEMPLATE=template0; \connect zulip CREATE SCHEMA zulip AUTHORIZATION zulip; diff --git a/tools/ci/Dockerfile.template b/tools/ci/Dockerfile.template index 3d11137949..f18af72d69 100644 --- a/tools/ci/Dockerfile.template +++ b/tools/ci/Dockerfile.template @@ -40,19 +40,10 @@ RUN apt-get update \ unzip zip jq \ python3-pip \ && ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime \ - && {{ locale-gen en_US.UTF-8 || true; }} \ - && echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale + && echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale # Set the locale, together with the locale-related steps above. -# It's not entirely clear why, but alternatives that don't work include -# * using `C.UTF-8` instead of `en_US.UTF-8`, here and above -# (causes mysterious failures in zerver.tests.test_message_fetch) -# * skipping the /etc/default/locale step above (ditto) -# * skipping this ENV instruction (causes provision to fail, -# because Python tries to use the `ascii` codec) -# Details in https://github.com/zulip/zulip/pull/7762#issuecomment-353197289 -# and particularly https://circleci.com/gh/hackerkid/zulip/80 . -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL C.UTF-8 # Install Docker. This logic comes from Circle's Dockerfile; it's probably # faster than the upstream-recommended approach of using their apt repo, diff --git a/tools/setup/dev-vagrant-docker/Dockerfile b/tools/setup/dev-vagrant-docker/Dockerfile index 990ceec96a..0f5776b2fd 100644 --- a/tools/setup/dev-vagrant-docker/Dockerfile +++ b/tools/setup/dev-vagrant-docker/Dockerfile @@ -3,8 +3,8 @@ FROM ubuntu:18.04 ARG UBUNTU_MIRROR # Basic packages and dependencies of docker-systemctl-replacement -RUN echo locales locales/default_environment_locale select en_US.UTF-8 | debconf-set-selections \ - && echo locales locales/locales_to_be_generated select "en_US.UTF-8 UTF-8" | debconf-set-selections \ +RUN echo locales locales/default_environment_locale select C.UTF-8 | debconf-set-selections \ + && echo locales locales/locales_to_be_generated select "C.UTF-8 UTF-8" | debconf-set-selections \ && { [ ! "$UBUNTU_MIRROR" ] || sed -i "s|http://\(\w*\.\)*archive\.ubuntu\.com/ubuntu/\? |$UBUNTU_MIRROR |" /etc/apt/sources.list; } \ # This restores man pages and other documentation that have been # stripped from the default Ubuntu cloud image and installs diff --git a/tools/test-install/prepare-base b/tools/test-install/prepare-base index c2f1f5f9a3..d72c689931 100755 --- a/tools/test-install/prepare-base +++ b/tools/test-install/prepare-base @@ -60,8 +60,8 @@ run apt-get install -y --no-install-recommends \ "${extra_packages[@]}" run ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime -run locale-gen en_US.UTF-8 || true -echo "LC_ALL=en_US.UTF-8" | run tee /etc/default/locale +run locale-gen C.UTF-8 || true +echo "LC_ALL=C.UTF-8" | run tee /etc/default/locale # TODO: on failure, either stop or print message lxc-stop -n "$CONTAINER_NAME" diff --git a/tools/test-js-with-puppeteer b/tools/test-js-with-puppeteer index 682b97b5dd..49764c9c31 100755 --- a/tools/test-js-with-puppeteer +++ b/tools/test-js-with-puppeteer @@ -20,7 +20,7 @@ os.environ["PUPPETEER_TESTS"] = "1" # The locale can have impact how Firefox does locale-aware sorting, # which we do verify in some tests. -os.environ["LC_ALL"] = "en_US.UTF-8" +os.environ["LC_ALL"] = "C.UTF-8" os.environ["CHROMIUM_EXECUTABLE"] = os.path.join(ZULIP_PATH, "node_modules/.bin/chromium") os.environ.pop("http_proxy", "")