i18n: Move static/locale back to locale.

As of commit cff40c557b (#9300), these
files are no longer served directly to the browser.  Disentangle them
from the static asset pipeline so we can refactor it without worrying
about them.

This has the side effect of eliminating the accidental duplication of
translation data via hash-naming in our release tarballs.

This reverts commit b546391f0b (#1148).

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

.gitignore

@@ -34,6 +34,12 @@ package-lock.json
 # Dockerfiles generated for CircleCI
 /tools/circleci/images
 
+# Generated i18n data
+/locale/en
+/locale/language_options.json
+/locale/language_name_map.json
+/locale/*/mobile.json
+
 # Static build
 *.mo
 npm-debug.log

.tx/config

@@ -3,32 +3,32 @@ host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
 [zulip.djangopo]
-file_filter = static/locale/<lang>/LC_MESSAGES/django.po
+file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = static/locale/en/LC_MESSAGES/django.po
+source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
 [zulip.translationsjson]
-file_filter = static/locale/<lang>/translations.json
+file_filter = locale/<lang>/translations.json
-source_file = static/locale/en/translations.json
+source_file = locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 
 [zulip.mobile]
-file_filter = static/locale/<lang>/mobile.json
+file_filter = locale/<lang>/mobile.json
-source_file = static/locale/en/mobile.json
+source_file = locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 
 [zulip-test.djangopo]
-file_filter = static/locale/<lang>/LC_MESSAGES/django.po
+file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = static/locale/en/LC_MESSAGES/django.po
+source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
 [zulip-test.translationsjson]
-file_filter = static/locale/<lang>/translations.json
+file_filter = locale/<lang>/translations.json
-source_file = static/locale/en/translations.json
+source_file = locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 

docs/overview/directory-structure.md

@@ -169,9 +169,7 @@ This is used to deploy essentially all configuration in production.
 
 ### Translation files
 
-* `locale/` Backend (Django) translations data files.
+* `locale/` Backend (Django) and frontend translation data files.
-
-* `static/locale/` Frontend translations data files.
 
 -----------------------------------------------------------------------
 

docs/subsystems/dependencies.md

@@ -318,7 +318,7 @@ several files from the source data, which we manage similar to our
 emoji, but without the caching (and thus without the
 garbage-collection).  New translations data is downloaded from
 Transifex and then compiled to generate both the production locale
-files and also language data in `static/locale/language*.json` using
+files and also language data in `locale/language*.json` using
 `manage.py compilemessages`, which extends the default Django
 implementation of that tool.
 

docs/subsystems/email.md

@@ -133,7 +133,7 @@ in our emails.
 
 One can test whether you did the translating part right by running
 `tools/inline-email-css && manage.py makemessages` and then searching
-for the strings in `static/locale/en/LC_MESSAGES/django.po`; if there
+for the strings in `locale/en/LC_MESSAGES/django.po`; if there
 are multiple copies or they contain CSS colors, you did it wrong.
 
 A final note for translating emails is that strings that are sent to

docs/translating/internationalization.md

@@ -110,9 +110,9 @@ sense of how everything fits together.
 
 All the translation magic happens through resource files which hold
 the translated text. Backend resource files are located at
-`static/locale/<lang_code>/LC_MESSAGES/django.po`, while frontend
+`locale/<lang_code>/LC_MESSAGES/django.po`, while frontend
 resource files are located at
-`static/locale/<lang_code>/translations.json` (and mobile at
+`locale/<lang_code>/translations.json` (and mobile at
 `mobile.json`).
 
 These files are uploaded to [Transifex][], where they can be translated.

scripts/lib/process-mobile-i18n

@@ -6,12 +6,12 @@ from subprocess import check_output
 from typing import Dict, List
 
 def get_json_filename(locale: str) -> str:
-    return "static/locale/{}/mobile.json".format(locale)
+    return "locale/{}/mobile.json".format(locale)
 
 def get_locales() -> List[str]:
-    tracked_files = check_output(['git', 'ls-files', 'static/locale'])
+    tracked_files = check_output(['git', 'ls-files', 'locale'])
     tracked_files = tracked_files.decode().split()
-    regex = re.compile(r'static/locale/(\w+)/LC_MESSAGES/django.po')
+    regex = re.compile(r'locale/(\w+)/LC_MESSAGES/django.po')
     locales = ['en']
     for tracked_file in tracked_files:
         matched = regex.search(tracked_file)
@@ -37,7 +37,7 @@ for locale in get_locales():
         translation_stats.update({locale: stats})
         locale_paths.append(path)
 
-stats_path = os.path.join('static', 'locale', 'mobile_info.json')
+stats_path = os.path.join('locale', 'mobile_info.json')
 with open(stats_path, 'w') as f:
     json.dump(translation_stats, f, indent=2, sort_keys=True)
     f.write('\n')

static/.gitignore

@@ -16,11 +16,5 @@
 # From tools/setup/generate-custom-icon-webfont
 /generated/icons/
 
-# Generated i18n data
-/locale/en
-/locale/language_options.json
-/locale/language_name_map.json
-/locale/*/mobile.json
-
 # Legacy emoji data directory
 /third/emoji-data

tools/build-release-tarball

@@ -114,7 +114,7 @@ echo "$version" > version
 
 cd "$TMPDIR"
 
-tar --append -f "$TARBALL" "$prefix/prod-static" "$prefix/build_id" "$prefix/version" "$prefix/zulip-git-version" "$prefix/staticfiles.json" "$prefix/templates/zerver/emails/compiled" "$prefix/webpack-stats-production.json"
+tar --append -f "$TARBALL" "$prefix/prod-static" "$prefix/build_id" "$prefix/version" "$prefix/zulip-git-version" "$prefix/locale" "$prefix/staticfiles.json" "$prefix/templates/zerver/emails/compiled" "$prefix/webpack-stats-production.json"
 
 rm -rf "$prefix"
 

tools/check-capitalization

@@ -33,12 +33,12 @@ if __name__ == '__main__':
         subprocess.call(['./manage.py', 'makemessages', '--locale', 'en'],
                         stderr=subprocess.STDOUT)
 
-    with open('static/locale/en/translations.json') as f:
+    with open('locale/en/translations.json') as f:
         data = json.load(f)
         frontend = check_capitalization(list(data.keys()))
         frontend_errors, frontend_ignored, banned_errors_front = frontend
 
-    with open('static/locale/en/LC_MESSAGES/django.po') as f:
+    with open('locale/en/LC_MESSAGES/django.po') as f:
         rows = [r for r in DJANGO_PO_REGEX.findall(f.read()) if r]
         backend = check_capitalization(rows)
         backend_errors, backend_ignored, banned_errors_back = backend

tools/check-frontend-i18n

@@ -33,7 +33,7 @@ if __name__ == '__main__':
         subprocess.call(['./manage.py', 'makemessages', '--locale', 'en'],
                         stderr=subprocess.STDOUT)
 
-    with open('static/locale/en/translations.json') as f:
+    with open('locale/en/translations.json') as f:
         data = json.load(f)
 
     found = find_handlebars(list(data.keys()))

tools/i18n/tagmessages

@@ -19,7 +19,7 @@ project_slug = 'zulip-test'  # HACK hardcode
 tools_dir = os.path.dirname(os.path.abspath(__file__))
 root_dir = os.path.dirname(tools_dir)
 # Choose any translation file for processing all strings.
-po = polib.pofile(os.path.join(root_dir, 'static', 'locale', 'de', 'LC_MESSAGES', 'django.po'))
+po = polib.pofile(os.path.join(root_dir, 'locale', 'de', 'LC_MESSAGES', 'django.po'))
 for entry in po:
     tag = entry.comment
     if tag:

tools/lib/provision.py

@@ -561,8 +561,8 @@ def main(options):
         # Consider updating generated translations data: both `.mo`
         # files and `language-options.json`.
         paths = ['zerver/management/commands/compilemessages.py']
-        paths += glob.glob('static/locale/*/LC_MESSAGES/*.po')
+        paths += glob.glob('locale/*/LC_MESSAGES/*.po')
-        paths += glob.glob('static/locale/*/translations.json')
+        paths += glob.glob('locale/*/translations.json')
 
         if file_or_package_hash_updated(paths, "last_compilemessages_hash", options.is_force):
             run(["./manage.py", "compilemessages"])

tools/linter_lib/exclude.py

@@ -9,7 +9,7 @@ EXCLUDED_FILES = [
     # Transifex syncs translation.json files without trailing
     # newlines; there's nothing other than trailing newlines we'd be
     # checking for in these anyway.
-    "static/locale",
+    "locale",
 ]
 
 PUPPET_CHECK_RULES_TO_EXCLUDE = [

tools/update-prod-static

@@ -100,10 +100,6 @@ os.makedirs('prod-static', exist_ok=True)
 
 shutil.move(os.path.join(settings.STATIC_ROOT, 'source-map'), 'prod-static/source-map')
 
-# Move language_options.json to the production release
-run(['cp', '-aT', 'static/locale', os.path.join(settings.STATIC_ROOT, 'locale')],
-    stdout=fp, stderr=fp)
-
 # Generate /team page markdown for authors
 authors_cmd = ['./tools/update-authors-json']
 if os.environ.get("TRAVIS"):

zerver/lib/i18n.py

@@ -26,7 +26,7 @@ def with_language(string: str, language: str) -> str:
 
 @lru_cache()
 def get_language_list() -> List[Dict[str, Any]]:
-    path = os.path.join(settings.STATIC_ROOT, 'locale', 'language_name_map.json')
+    path = os.path.join(settings.DEPLOY_ROOT, 'locale', 'language_name_map.json')
     with open(path, 'r') as reader:
         languages = ujson.load(reader)
         return languages['name_map']
@@ -88,7 +88,7 @@ def get_language_translation_data(language: str) -> Dict[str, str]:
         language = 'zh_Hant'
     elif language == 'id-id':
         language = 'id_ID'
-    path = os.path.join(settings.STATIC_ROOT, 'locale', language, 'translations.json')
+    path = os.path.join(settings.DEPLOY_ROOT, 'locale', language, 'translations.json')
     try:
         with open(path, 'r') as reader:
             return ujson.load(reader)

zerver/management/commands/compilemessages.py

@@ -27,13 +27,6 @@ class Command(compilemessages.Command):
             help='Stop execution in case of errors.')
 
     def handle(self, *args: Any, **options: Any) -> None:
-        if settings.PRODUCTION:
-            # HACK: When using upgrade-zulip-from-git, we're in a
-            # production environment where STATIC_ROOT will include
-            # past versions; this ensures we only process the current
-            # version
-            settings.STATIC_ROOT = os.path.join(settings.DEPLOY_ROOT, "static")
-            settings.LOCALE_PATHS = (os.path.join(settings.DEPLOY_ROOT, 'static/locale'),)
         super().handle(*args, **options)
         self.strict = options['strict']
         self.extract_language_options()
@@ -41,9 +34,9 @@ class Command(compilemessages.Command):
 
     def create_language_name_map(self) -> None:
         join = os.path.join
-        static_root = settings.STATIC_ROOT
+        deploy_root = settings.DEPLOY_ROOT
-        path = join(static_root, 'locale', 'language_options.json')
+        path = join(deploy_root, 'locale', 'language_options.json')
-        output_path = join(static_root, 'locale', 'language_name_map.json')
+        output_path = join(deploy_root, 'locale', 'language_name_map.json')
 
         with open(path, 'r') as reader:
             languages = ujson.load(reader)
@@ -80,9 +73,9 @@ class Command(compilemessages.Command):
                 raise Exception("Unknown language %s" % (locale,))
 
     def get_locales(self) -> List[str]:
-        tracked_files = check_output(['git', 'ls-files', 'static/locale'])
+        tracked_files = check_output(['git', 'ls-files', 'locale'])
         tracked_files = tracked_files.decode().split()
-        regex = re.compile(r'static/locale/(\w+)/LC_MESSAGES/django.po')
+        regex = re.compile(r'locale/(\w+)/LC_MESSAGES/django.po')
         locales = ['en']
         for tracked_file in tracked_files:
             matched = regex.search(tracked_file)
@@ -92,7 +85,7 @@ class Command(compilemessages.Command):
         return locales
 
     def extract_language_options(self) -> None:
-        locale_path = "{}/locale".format(settings.STATIC_ROOT)
+        locale_path = "{}/locale".format(settings.DEPLOY_ROOT)
         output_path = "{}/language_options.json".format(locale_path)
 
         data = {'languages': []}  # type: Dict[str, List[Dict[str, Any]]]

zerver/management/commands/makemessages.py

@@ -78,7 +78,7 @@ class Command(makemessages.Command):
                             default='static/templates',
                             help='Name of the Handlebars template directory')
         parser.add_argument('--frontend-output', type=str,
-                            default='static/locale',
+                            default='locale',
                             help='Name of the frontend messages output directory')
         parser.add_argument('--frontend-namespace', type=str,
                             default='translations.json',

zproject/settings.py

@@ -889,7 +889,7 @@ else:
 
 # If changing this, you need to also the hack modifications to this in
 # our compilemessages management command.
-LOCALE_PATHS = (os.path.join(STATIC_ROOT, 'locale'),)
+LOCALE_PATHS = (os.path.join(DEPLOY_ROOT, 'locale'),)
 
 # We want all temporary uploaded files to be stored on disk.
 FILE_UPLOAD_MAX_MEMORY_SIZE = 0