locale: Use `C.UTF-8` rather than `en_US.UTF-8`.

The `en_US.UTF-8` locale may not be configured or generated on all
installs; it also requires that the `locales` package be installed.
If users generate the `en_US.UTF-8` locale without adding it to the
permanent set of system locales, the generated `en_US.UTF-8` stops
working when the `locales` package is updated.

Switch to using `C.UTF-8` in all cases, which is guaranteed to be
installed.

Fixes #15819.

Vagrantfile

@@ -166,8 +166,8 @@ sudo dpkg-divert --add --rename /etc/default/motd-news
 sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
 
 # Set default locale, this prevents errors if the user has another locale set.
-if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
-    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
+if ! grep -q 'LC_ALL=C.UTF-8' /etc/default/locale; then
+    echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale
 fi
 
 # Set an environment variable, so that we won't print the virtualenv

puppet/zulip/templates/postgresql/13/postgresql.conf.template.erb

@@ -691,11 +691,11 @@ timezone = 'Etc/UTC'
 					# encoding
 
 # These settings are initialized by initdb, but they can be changed.
-lc_messages = 'en_US.UTF-8'			# locale for system error message
+lc_messages = 'C.UTF-8'			# locale for system error message
 					# strings
-lc_monetary = 'en_US.UTF-8'			# locale for monetary formatting
-lc_numeric = 'en_US.UTF-8'			# locale for number formatting
-lc_time = 'en_US.UTF-8'				# locale for time formatting
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
 
 # default configuration for text search
 default_text_search_config = 'pg_catalog.english'

puppet/zulip/templates/postgresql/9.3/postgresql.conf.template.erb

@@ -492,11 +492,11 @@ datestyle = 'iso, mdy'
 					# encoding
 
 # These settings are initialized by initdb, but they can be changed.
-lc_messages = 'en_US.UTF-8'			# locale for system error message
+lc_messages = 'C.UTF-8'			# locale for system error message
 					# strings
-lc_monetary = 'en_US.UTF-8'			# locale for monetary formatting
-lc_numeric = 'en_US.UTF-8'			# locale for number formatting
-lc_time = 'en_US.UTF-8'				# locale for time formatting
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
 
 # default configuration for text search
 #default_text_search_config = 'pg_catalog.english'

puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb

@@ -549,11 +549,11 @@ timezone = 'localtime'
 					# encoding
 
 # These settings are initialized by initdb, but they can be changed.
-lc_messages = 'en_US.UTF-8'			# locale for system error message
+lc_messages = 'C.UTF-8'			# locale for system error message
 					# strings
-lc_monetary = 'en_US.UTF-8'			# locale for monetary formatting
-lc_numeric = 'en_US.UTF-8'			# locale for number formatting
-lc_time = 'en_US.UTF-8'				# locale for time formatting
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
 
 # default configuration for text search
 default_text_search_config = 'pg_catalog.english'

puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb

@@ -565,11 +565,11 @@ timezone = 'UTC'
 					# encoding
 
 # These settings are initialized by initdb, but they can be changed.
-lc_messages = 'en_US.UTF-8'			# locale for system error message
+lc_messages = 'C.UTF-8'			# locale for system error message
 					# strings
-lc_monetary = 'en_US.UTF-8'			# locale for monetary formatting
-lc_numeric = 'en_US.UTF-8'			# locale for number formatting
-lc_time = 'en_US.UTF-8'				# locale for time formatting
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
 
 # default configuration for text search
 default_text_search_config = 'pg_catalog.english'

puppet/zulip/templates/uwsgi.ini.template.erb

@@ -10,7 +10,7 @@ harakiri=20
 buffer-size=<%= @uwsgi_buffer_size %>
 listen=<%= @uwsgi_listen_backlog_limit %>
 post-buffering=4096
-env= LANG=en_US.UTF-8
+env= LANG=C.UTF-8
 uid=zulip
 gid=zulip
 

scripts/lib/install

@@ -168,10 +168,9 @@ set -x
 ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
 
 # Force a known locale.  Some packages on PyPI fail to install in some locales.
-localedef -i en_US -f UTF-8 en_US.UTF-8
-export LC_ALL="en_US.UTF-8"
-export LANG="en_US.UTF-8"
-export LANGUAGE="en_US.UTF-8"
+export LC_ALL="C.UTF-8"
+export LANG="C.UTF-8"
+export LANGUAGE="C.UTF-8"
 
 # Force a known path; this fixes problems on Debian where `su` from
 # non-root may not adjust `$PATH` to root's.

scripts/lib/upgrade-zulip-stage-2

@@ -21,9 +21,9 @@ if TYPE_CHECKING:
 os.environ["PYTHONUNBUFFERED"] = "y"
 
 # Force a known locale.  Some packages on PyPI fail to install in some locales.
-os.environ["LC_ALL"] = "en_US.UTF-8"
-os.environ["LANG"] = "en_US.UTF-8"
-os.environ["LANGUAGE"] = "en_US.UTF-8"
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+os.environ["LANGUAGE"] = "C.UTF-8"
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
 from scripts.lib.zulip_tools import (

scripts/setup/create-db.sql

@@ -9,8 +9,8 @@ ALTER ROLE zulip SET search_path TO zulip,public;
 CREATE DATABASE zulip
     OWNER=zulip
     ENCODING=UTF8
-    LC_COLLATE='en_US.UTF-8'
-    LC_CTYPE='en_US.UTF-8'
+    LC_COLLATE='C.UTF-8'
+    LC_CTYPE='C.UTF-8'
     TEMPLATE=template0;
 \connect zulip
 CREATE SCHEMA zulip AUTHORIZATION zulip;

tools/ci/Dockerfile.template

@@ -40,19 +40,10 @@ RUN apt-get update \
     unzip zip jq \
     python3-pip \
   && ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime \
-  && {{ locale-gen en_US.UTF-8 || true; }} \
-  && echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
+  && echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale
 
 # Set the locale, together with the locale-related steps above.
-# It's not entirely clear why, but alternatives that don't work include
-#  * using `C.UTF-8` instead of `en_US.UTF-8`, here and above
-#    (causes mysterious failures in zerver.tests.test_message_fetch)
-#  * skipping the /etc/default/locale step above (ditto)
-#  * skipping this ENV instruction (causes provision to fail,
-#    because Python tries to use the `ascii` codec)
-# Details in https://github.com/zulip/zulip/pull/7762#issuecomment-353197289
-# and particularly https://circleci.com/gh/hackerkid/zulip/80 .
-ENV LC_ALL en_US.UTF-8
+ENV LC_ALL C.UTF-8
 
 # Install Docker.  This logic comes from Circle's Dockerfile; it's probably
 # faster than the upstream-recommended approach of using their apt repo,

tools/setup/dev-vagrant-docker/Dockerfile

@@ -3,8 +3,8 @@ FROM ubuntu:18.04
 ARG UBUNTU_MIRROR
 
 # Basic packages and dependencies of docker-systemctl-replacement
-RUN echo locales locales/default_environment_locale select en_US.UTF-8 | debconf-set-selections \
-    && echo locales locales/locales_to_be_generated select "en_US.UTF-8 UTF-8" | debconf-set-selections \
+RUN echo locales locales/default_environment_locale select C.UTF-8 | debconf-set-selections \
+    && echo locales locales/locales_to_be_generated select "C.UTF-8 UTF-8" | debconf-set-selections \
     && { [ ! "$UBUNTU_MIRROR" ] || sed -i "s|http://\(\w*\.\)*archive\.ubuntu\.com/ubuntu/\? |$UBUNTU_MIRROR |" /etc/apt/sources.list; } \
     # This restores man pages and other documentation that have been
     # stripped from the default Ubuntu cloud image and installs

tools/test-install/prepare-base

@@ -60,8 +60,8 @@ run apt-get install -y --no-install-recommends \
     "${extra_packages[@]}"
 
 run ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
-run locale-gen en_US.UTF-8 || true
-echo "LC_ALL=en_US.UTF-8" | run tee /etc/default/locale
+run locale-gen C.UTF-8 || true
+echo "LC_ALL=C.UTF-8" | run tee /etc/default/locale
 
 # TODO: on failure, either stop or print message
 lxc-stop -n "$CONTAINER_NAME"

tools/test-js-with-puppeteer

@@ -20,7 +20,7 @@ os.environ["PUPPETEER_TESTS"] = "1"
 
 # The locale can have impact how Firefox does locale-aware sorting,
 # which we do verify in some tests.
-os.environ["LC_ALL"] = "en_US.UTF-8"
+os.environ["LC_ALL"] = "C.UTF-8"
 
 os.environ["CHROMIUM_EXECUTABLE"] = os.path.join(ZULIP_PATH, "node_modules/.bin/chromium")
 os.environ.pop("http_proxy", "")