zulip/zerver/management/commands/compilemessages.py

import json
import os
import re
from subprocess import CalledProcessError, check_output
from typing import Any, Dict, List

import orjson
import polib
from django.conf import settings
from django.conf.locale import LANG_INFO
from django.core.management.base import CommandParser
from django.core.management.commands import compilemessages
from django.utils.translation.trans_real import to_language

from zerver.lib.i18n import with_language


class Command(compilemessages.Command):

    def add_arguments(self, parser: CommandParser) -> None:
        super().add_arguments(parser)

        parser.add_argument(
            '--strict', '-s',
            action='store_true',
            default=False,
            help='Stop execution in case of errors.')

    def handle(self, *args: Any, **options: Any) -> None:
        super().handle(*args, **options)
        self.strict = options['strict']
        self.extract_language_options()
        self.create_language_name_map()

    def create_language_name_map(self) -> None:
        join = os.path.join
        deploy_root = settings.DEPLOY_ROOT
        path = join(deploy_root, 'locale', 'language_options.json')
        output_path = join(deploy_root, 'locale', 'language_name_map.json')

        with open(path, "rb") as reader:
            languages = orjson.loads(reader.read())
            lang_list = []
            for lang_info in languages['languages']:
                lang_info['name'] = lang_info['name_local']
                del lang_info['name_local']
                lang_list.append(lang_info)

            lang_list.sort(key=lambda lang: lang['name'])

        with open(output_path, 'wb') as output_file:
            output_file.write(
                orjson.dumps(
                    {'name_map': lang_list},
                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
                )
            )

    def get_po_filename(self, locale_path: str, locale: str) -> str:
        po_template = '{}/{}/LC_MESSAGES/django.po'
        return po_template.format(locale_path, locale)

    def get_json_filename(self, locale_path: str, locale: str) -> str:
        return f"{locale_path}/{locale}/translations.json"

    def get_name_from_po_file(self, po_filename: str, locale: str) -> str:
        lang_name_re = re.compile(r'"Language-Team: (.*?) \(')
        with open(po_filename) as reader:
            result = lang_name_re.search(reader.read())
            if result:
                try:
                    return result.group(1)
                except Exception:
                    print(f"Problem in parsing {po_filename}")
                    raise
            else:
                raise Exception(f"Unknown language {locale}")

    def get_locales(self) -> List[str]:
        output = check_output(['git', 'ls-files', 'locale'])
        tracked_files = output.decode().split()
        regex = re.compile(r'locale/(\w+)/LC_MESSAGES/django.po')
        locales = ['en']
        for tracked_file in tracked_files:
            matched = regex.search(tracked_file)
            if matched:
                locales.append(matched.group(1))

        return locales

    def extract_language_options(self) -> None:
        locale_path = f"{settings.DEPLOY_ROOT}/locale"
        output_path = f"{locale_path}/language_options.json"

        data: Dict[str, List[Dict[str, Any]]] = {'languages': []}

        try:
            locales = self.get_locales()
        except CalledProcessError:
            # In case we are not under a Git repo, fallback to getting the
            # locales using listdir().
            locales = os.listdir(locale_path)
            locales.append('en')
            locales = list(set(locales))

        for locale in locales:
            if locale == 'en':
                data['languages'].append({
                    'name': 'English',
                    'name_local': 'English',
                    'code': 'en',
                    'locale': 'en',
                })
                continue

            lc_messages_path = os.path.join(locale_path, locale, 'LC_MESSAGES')
            if not os.path.exists(lc_messages_path):
                # Not a locale.
                continue

            info: Dict[str, Any] = {}
            code = to_language(locale)
            percentage = self.get_translation_percentage(locale_path, locale)
            try:
                name = LANG_INFO[code]['name']
                name_local = LANG_INFO[code]['name_local']
            except KeyError:
                # Fallback to getting the name from PO file.
                filename = self.get_po_filename(locale_path, locale)
                name = self.get_name_from_po_file(filename, locale)
                name_local = with_language(name, code)

            info['name'] = name
            info['name_local'] = name_local
            info['code'] = code
            info['locale'] = locale
            info['percent_translated'] = percentage
            data['languages'].append(info)

        with open(output_path, 'w') as writer:
            json.dump(data, writer, indent=2, sort_keys=True)
            writer.write('\n')

    def get_translation_percentage(self, locale_path: str, locale: str) -> int:

        # backend stats
        po = polib.pofile(self.get_po_filename(locale_path, locale))
        not_translated = len(po.untranslated_entries())
        total = len(po.translated_entries()) + not_translated

        # frontend stats
        with open(self.get_json_filename(locale_path, locale), "rb") as reader:
            for key, value in orjson.loads(reader.read()).items():
                total += 1
                if value == '':
                    not_translated += 1

        # mobile stats
        with open(os.path.join(locale_path, 'mobile_info.json'), "rb") as mob:
            mobile_info = orjson.loads(mob.read())
        try:
            info = mobile_info[locale]
        except KeyError:
            if self.strict:
                raise
            info = {'total': 0, 'not_translated': 0}

        total += info['total']
        not_translated += info['not_translated']

        return (total - not_translated) * 100 // total