zulip/zerver/management/commands/compilemessages.py

import json
import os
import re
import unicodedata
from subprocess import CalledProcessError, check_output
from typing import Any

import orjson
import polib
from django.conf import settings
from django.conf.locale import LANG_INFO
from django.core.management.base import CommandParser
from django.core.management.commands import compilemessages
from django.utils.translation import gettext as _
from django.utils.translation import override as override_language
from django.utils.translation import to_language
from pyuca import Collator
from typing_extensions import override


class Command(compilemessages.Command):
    @override
    def add_arguments(self, parser: CommandParser) -> None:
        super().add_arguments(parser)

        parser.add_argument(
            "--strict", "-s", action="store_true", help="Stop execution in case of errors."
        )

    @override
    def handle(self, *args: Any, **options: Any) -> None:
        super().handle(*args, **options)
        self.strict = options["strict"]
        self.extract_language_options()
        self.create_language_name_map()

    def create_language_name_map(self) -> None:
        join = os.path.join
        deploy_root = settings.DEPLOY_ROOT
        path = join(deploy_root, "locale", "language_options.json")
        output_path = join(deploy_root, "locale", "language_name_map.json")

        with open(path, "rb") as reader:
            languages = orjson.loads(reader.read())
            lang_list = []
            for lang_info in languages["languages"]:
                lang_info["name"] = lang_info["name_local"]
                del lang_info["name_local"]
                lang_list.append(lang_info)

            collator = Collator()
            lang_list.sort(key=lambda lang: collator.sort_key(lang["name"]))

        with open(output_path, "wb") as output_file:
            output_file.write(
                orjson.dumps(
                    {"name_map": lang_list},
                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
                )
            )

    def get_po_filename(self, locale_path: str, locale: str) -> str:
        po_template = "{}/{}/LC_MESSAGES/django.po"
        return po_template.format(locale_path, locale)

    def get_json_filename(self, locale_path: str, locale: str) -> str:
        return f"{locale_path}/{locale}/translations.json"

    def get_name_from_po_file(self, po_filename: str, locale: str) -> str:
        try:
            team = polib.pofile(po_filename).metadata["Language-Team"]
            return team[: team.rindex(" (")]
        except (KeyError, ValueError):
            raise Exception(f"Unknown language {locale}")

    def get_locales(self) -> list[str]:
        output = check_output(["git", "ls-files", "locale"], text=True)
        tracked_files = output.split()
        regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po")
        locales = ["en"]
        for tracked_file in tracked_files:
            matched = regex.search(tracked_file)
            if matched:
                locales.append(matched.group(1))

        return locales

    def extract_language_options(self) -> None:
        locale_path = f"{settings.DEPLOY_ROOT}/locale"
        output_path = f"{locale_path}/language_options.json"

        data: dict[str, list[dict[str, Any]]] = {"languages": []}

        try:
            locales = self.get_locales()
        except CalledProcessError:
            # In case we are not under a Git repo, fallback to getting the
            # locales using listdir().
            locales = os.listdir(locale_path)
            locales.append("en")
            locales = list(set(locales))

        for locale in sorted(locales):
            if locale == "en":
                data["languages"].append(
                    {
                        "name": "English",
                        "name_local": "English",
                        "code": "en",
                        "locale": "en",
                    }
                )
                continue

            lc_messages_path = os.path.join(locale_path, locale, "LC_MESSAGES")
            if not os.path.exists(lc_messages_path):
                # Not a locale.
                continue

            info: dict[str, Any] = {}
            code = to_language(locale)
            percentage = self.get_translation_percentage(locale_path, locale)
            try:
                name = LANG_INFO[code]["name"]
                name_local = LANG_INFO[code]["name_local"]
            except KeyError:
                # Fallback to getting the name from PO file.
                filename = self.get_po_filename(locale_path, locale)
                name = self.get_name_from_po_file(filename, locale)
                with override_language(code):
                    name_local = _(name)

            info["name"] = unicodedata.normalize("NFC", name)
            info["name_local"] = unicodedata.normalize("NFC", name_local)
            info["code"] = code
            info["locale"] = locale
            info["percent_translated"] = percentage
            data["languages"].append(info)

        with open(output_path, "w") as writer:
            json.dump(data, writer, indent=2, sort_keys=True)
            writer.write("\n")

    def get_translation_percentage(self, locale_path: str, locale: str) -> int:
        # backend stats
        po = polib.pofile(self.get_po_filename(locale_path, locale))
        not_translated = len(po.untranslated_entries())
        total = len(po.translated_entries()) + not_translated

        # frontend stats
        with open(self.get_json_filename(locale_path, locale), "rb") as reader:
            for value in orjson.loads(reader.read()).values():
                total += 1
                if value == "":
                    not_translated += 1

        # mobile stats
        with open(os.path.join(locale_path, "mobile_info.json"), "rb") as mob:
            mobile_info = orjson.loads(mob.read())
        try:
            info = mobile_info[locale]
        except KeyError:
            if self.strict:
                raise
            info = {"total": 0, "not_translated": 0}

        total += info["total"]
        not_translated += info["not_translated"]

        return (total - not_translated) * 100 // total