[third] Integrate i18next with Handlebars

.tx/config

@@ -8,3 +8,10 @@ type = PO
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 lang_map = zh-Hans: zh_CN
 
+[zulip.translationsjson]
+source_file = static/locale/en/translations.json
+source_lang = en
+type = KEYVALUEJSON
+file_filter = static/locale/<lang>/translations.json
+lang_map = zh-Hans: zh_CN
+

docs/translating.md

@@ -58,6 +58,21 @@ You can instead use:
 {% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %}
 ```
 
+## Frontend Translations
+The first step in translating the frontend is to create the translation
+files using `python manage makemessages`. This command will create
+translation files under `static/locale`, the location can be changed by
+passing an argument to the command, however make sure that the location is
+publically accessible since these files are loaded through XHR in the
+frontend which will only work with publically accessible resources.
+
+The second step is to upload the translatable strings to Transifex using
+`tx push -s -a`.
+
+The final step is to get the translated files from Transifex using
+`tx pull -a`.
+
+
 [Django]: https://docs.djangoproject.com/en/1.9/topics/templates/#the-django-template-language
 [Jinja2]: http://jinja.pocoo.org/
 [Handlebars]: http://handlebarsjs.com/

frontend_tests/node_tests/i18n.js

@@ -0,0 +1,87 @@
+add_dependencies({
+    Handlebars: 'handlebars',
+    templates: 'js/templates',
+    i18n: 'i18next'
+});
+
+var i18n = global.i18n;
+i18n.init({
+    nsSeparator: false,
+    keySeparator: false,
+    interpolation: {
+        prefix: "__",
+        suffix: "__"
+    },
+    lng: 'fr',
+    resources: {
+        fr: {
+            translation: {
+                'Reply': "French",
+                "You'll receive notifications when a message arrives and __page_params.product_name__ isn't in focus or the message is offscreen.": "Some French text with __page_params.product_name__"
+            }
+        }
+    }
+});
+
+global.$ = require('jQuery');
+var _ = global._;
+
+// When writing these tests, the following command might be helpful:
+// ./tools/get-handlebar-vars static/templates/*.handlebars
+
+function render(template_name, args) {
+    global.use_template(template_name);
+    return global.templates.render(template_name, args);
+}
+
+(function test_t_tag() {
+    var args = {
+        "message": {
+            is_stream: true,
+            id: "99",
+            stream: "devel",
+            subject: "testing",
+            sender_full_name: "King Lear"
+        },
+        "can_edit_message": true,
+        "can_mute_topic": true,
+        "narrowed": true
+    };
+
+    var html = '<div style="height: 250px">';
+    html += render('actions_popover_content', args);
+    html += "</div>";
+    var link = $(html).find("a.respond_button");
+    assert.equal(link.text().trim(), 'French');
+    global.write_test_output("actions_popover_content.handlebars", html);
+}());
+
+(function test_tr_tag() {
+    var args = {
+        "page_params": {
+            "fullname": "John Doe",
+            "product_name": "Zulip",
+            "password_auth_enabled": false,
+            "avatar_url": "http://example.com",
+            "left_side_userlist": false,
+            "twenty_four_hour_time": false,
+            "stream_desktop_notifications_enabled": false,
+            "stream_sounds_enabled": false,
+            "desktop_notifications_enabled": false,
+            "sounds_enabled": false,
+            "enable_offline_email_notifications": false,
+            "enable_offline_push_notifications": false,
+            "enable_digest_emails": false,
+            "domain": "zulip.com",
+            "autoscroll_forever": false,
+            "default_desktop_notifications": false
+        }
+    };
+
+    var html = '<div style="height: 250px">';
+    html += render('settings_tab', args);
+    html += "</div>";
+    var div = $(html).find("div.notification-reminder");
+    assert.equal(div.text().trim(), 'Some French text with Zulip');
+    global.write_test_output("actions_popover_content.handlebars", html);
+}());

frontend_tests/node_tests/templates.js

@@ -1,6 +1,18 @@
 add_dependencies({
     Handlebars: 'handlebars',
-    templates: 'js/templates'
+    templates: 'js/templates',
+    i18n: 'i18next'
+});
+
+var i18n = global.i18n;
+i18n.init({
+    nsSeparator: false,
+    keySeparator: false,
+    interpolation: {
+        prefix: "__",
+        suffix: "__"
+    },
+    lng: 'en'
 });
 
 global.$ = require('jQuery');

package.json

@@ -15,7 +15,11 @@
     "htmlparser2": "3.8.3",
     "cssstyle": "0.2.29",
     "webpack": "1.12.2",
-    "webpack-dev-server": "1.12.1"
+    "webpack-dev-server": "1.12.1",
+    "i18next": "3.0.0",
+    "i18next-parser": "0.7.0",
+    "i18next-xhr-backend": "0.5.4",
+    "i18next-browser-languagedetector": "0.3.0"
   },
   "scripts": {},
   "repository": {

static/js/src/main.js

@@ -1 +1,26 @@
 // commonjs code goes here
+
+var i18n = window.i18n = require('i18next');
+var XHR = require('i18next-xhr-backend');
+var lngDetector = require('i18next-browser-languagedetector');
+var backendOptions = {
+    loadPath: '/static/locale/__lng__/__ns__.json'
+};
+
+var detectionOptions = {
+    order: ['htmlTag'],
+    htmlTag: document.documentElement
+};
+
+i18n.use(XHR)
+    .use(lngDetector)
+    .init({
+        nsSeparator: false,
+        keySeparator: false,
+        interpolation: {
+            prefix: "__",
+            suffix: "__"
+        },
+        backend: backendOptions,
+        detection: detectionOptions
+});

static/js/templates.js

@@ -73,6 +73,34 @@ Handlebars.registerHelper('if_or', function () {
     return options.inverse(this);
 });
 
+Handlebars.registerHelper('t', function (i18n_key) {
+    // Marks a string for translation.
+    // Example usage:
+    //     {{t "some English text"}}
+    var result = i18n.t(i18n_key);
+    return new Handlebars.SafeString(result);
+});
+
+Handlebars.registerHelper('tr', function (context, options) {
+    // Marks a block for translation.
+    // Example usage 1:
+    //     {{#tr context}}
+    //         <p>some English text</p>
+    //     {{/tr}}
+    //
+    // Example usage 2:
+    //     {{#tr context}}
+    //         <p>This __variable__ will get value from context</p>
+    //     {{/tr}}
+    //
+    // Notes:
+    //     1. `context` is very important. It can be `this` or an
+    //        object or key of the current context.
+    //     2. Use `__` instead of `{{` and `}}` to declare expressions
+    var result = i18n.t(options.fn(context), context);
+    return new Handlebars.SafeString(result);
+});
+
 return exports;
 }());
 if (typeof module !== 'undefined') {

templates/zerver/base.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang='{{LANGUAGE_CODE}}'>
 
 {# Base template for the whole site. #}
 

tools/jslint/check-all.js

@@ -4,10 +4,10 @@
 var globals =
     // Third-party libraries
       ' $ _ jQuery Spinner Handlebars XDate zxcvbn Intl Notification'
-    + ' LazyLoad Dropbox SockJS marked'
+    + ' LazyLoad Dropbox SockJS marked i18n'
 
     // Node-based unit tests
-    + ' module'
+    + ' module require'
 
     // Cocoa<-> Javascript bridge
     + ' bridge'

zerver/management/commands/makemessages.py

@@ -28,8 +28,16 @@ Credit for the approach goes to:
 http://stackoverflow.com/questions/2090717/getting-translation-strings-for-jinja2-templates-integrated-with-django-1-x
 
 """
+from __future__ import absolute_import
 
+import os
 import re
+import glob
+import json
+from six.moves import filter
+from six.moves import map
+from six.moves import zip
+
 from django.core.management.commands import makemessages
 from django.utils.translation import trans_real
 from django.template.base import BLOCK_TAG_START, BLOCK_TAG_END
@@ -37,6 +45,13 @@ from django.template.base import BLOCK_TAG_START, BLOCK_TAG_END
 strip_whitespace_right = re.compile(r"(%s-?\s*(trans|pluralize).*?-%s)\s+" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)
 strip_whitespace_left = re.compile(r"\s+(%s-\s*(endtrans|pluralize).*?-?%s)" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)
 
+regexes = ['{{#tr .*?}}(.*?){{/tr}}',
+           '{{t "(.*?)"\W*}}',
+           "{{t '(.*?)'\W*}}",
+           ]
+
+frontend_compiled_regexes = [re.compile(regex) for regex in regexes]
+
 def strip_whitespaces(src):
     src = strip_whitespace_left.sub(r'\1', src)
     src = strip_whitespace_right.sub(r'\1', src)
@@ -44,7 +59,34 @@ def strip_whitespaces(src):
 
 class Command(makemessages.Command):
 
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+        parser.add_argument('--frontend-source', type=str,
+                            default='static/templates',
+                            help='Name of the Handlebars template directory')
+        parser.add_argument('--frontend-output', type=str,
+                            default='static/locale',
+                            help='Name of the frontend messages output directory')
+        parser.add_argument('--frontend-namespace', type=str,
+                            default='translations.json',
+                            help='Namespace of the frontend locale file')
+
     def handle(self, *args, **options):
+        self.handle_django_locales(*args, **options)
+        self.handle_frontend_locales(*args, **options)
+
+    def handle_frontend_locales(self, *args, **options):
+        self.frontend_source = options.get('frontend_source')
+        self.frontend_output = options.get('frontend_output')
+        self.frontend_namespace = options.get('frontend_namespace')
+        self.frontend_locale = options.get('locale')
+        self.frontend_exclude = options.get('exclude')
+        self.frontend_all = options.get('all')
+
+        translation_strings = self.get_translation_strings()
+        self.write_translation_strings(translation_strings)
+
+    def handle_django_locales(self, *args, **options):
         old_endblock_re = trans_real.endblock_re
         old_block_re = trans_real.block_re
         old_constant_re = trans_real.constant_re
@@ -73,3 +115,84 @@ class Command(makemessages.Command):
             trans_real.block_re = old_block_re
             trans_real.templatize = old_templatize
             trans_real.constant_re = old_constant_re
+
+    def extract_strings(self, data):
+        translation_strings = {}
+        for regex in frontend_compiled_regexes:
+            for match in regex.findall(data):
+                translation_strings[match] = ""
+
+        return translation_strings
+
+    def get_translation_strings(self):
+        translation_strings = {}
+        dirname = self.get_template_dir()
+
+        for filename in os.listdir(dirname):
+            if filename.endswith('handlebars'):
+                with open(os.path.join(dirname, filename)) as reader:
+                    data = reader.read()
+                    translation_strings.update(self.extract_strings(data))
+
+        return translation_strings
+
+    def get_template_dir(self):
+        return self.frontend_source
+
+
+    def get_namespace(self):
+        return self.frontend_namespace
+
+    def get_locales(self):
+        locale = self.frontend_locale
+        exclude = self.frontend_exclude
+        process_all = self.frontend_all
+
+        paths = glob.glob('%s/*' % self.default_locale_path,)
+        locale_dirs = list(filter(os.path.isdir, paths))
+        all_locales = list(map(os.path.basename, locale_dirs))
+
+        # Account for excluded locales
+        if process_all:
+            locales = all_locales
+        else:
+            locales = locale or all_locales
+            locales = set(locales) - set(exclude)
+
+        return locales
+
+    def get_base_path(self):
+        return self.frontend_output
+
+    def get_output_paths(self):
+        base_path = self.get_base_path()
+        locales = self.get_locales()
+        for path in [os.path.join(base_path, locale) for locale in locales]:
+            if not os.path.exists(path):
+                os.makedirs(path)
+
+            yield os.path.join(path, self.get_namespace())
+
+    def get_new_strings(self, old_strings, translation_strings):
+        """
+        Missing strings are removed, new strings are added and already
+        translated strings are not touched.
+        """
+        new_strings = {}
+        for k in translation_strings:
+            new_strings[k] = old_strings.get(k, k)
+
+        return new_strings
+
+    def write_translation_strings(self, translation_strings):
+        for locale, output_path in zip(self.get_locales(), self.get_output_paths()):
+            self.stdout.write("[frontend] processing locale {}".format(locale))
+            try:
+                with open(output_path, 'r') as reader:
+                    old_strings = json.load(reader)
+            except (IOError, ValueError):
+                old_strings = {}
+
+            new_strings = self.get_new_strings(old_strings, translation_strings)
+            with open(output_path, 'w') as writer:
+                json.dump(new_strings, writer, indent=2)

zproject/settings.py

@@ -259,6 +259,7 @@ TEMPLATES = [
             'context_processors': [
                 'zerver.context_processors.add_settings',
                 'zerver.context_processors.add_metrics',
+                'django.core.context_processors.i18n',
             ],
         },
     },