Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

emoji: Add support for translating emoticons. 

Add `translate_emoticons` to `prop_types` and `expected_keys`.
Furthermore, create a emoji-translating Markdown inline pattern.

Also use a JavaScript version of `translate_emoticons` and then use
this function during Markdown previews and as a preprocessor. This
is only needed for previews, because usually emoticon translation
happens on the backend after sending.

Add tests for emoticon translation, a settings UI, and a /help/ page
as well.

Tweaked by tabbott to fix various test failurse as well as how this
handles whitespace, requiring emoticons to not have adjacent
characters.

Fixes #1768.
This commit is contained in:
Marco Burstein 2018-01-15 10:36:32 -08:00 committed by Tim Abbott
parent 038579b840
commit bdb86f1b5e
19 changed files with 315 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
frontend_tests/node_tests/dispatch.js
View File

@ -413,6 +413,12 @@ var event_fixtures = {
setting: true,
},
update_display_settings__translate_emoticons: {
type: 'update_display_settings',
setting_name: 'translate_emoticons',
setting: true,
},
update_global_notifications: {
type: 'update_global_notifications',
notification_name: 'enable_stream_sounds',
@ -813,6 +819,10 @@ with_overrides(function (override) {
dispatch(event);
assert_same(page_params.twenty_four_hour_time, true);
event = event_fixtures.update_display_settings__translate_emoticons;
page_params.translate_emoticons = false;
dispatch(event);
assert_same(page_params.translate_emoticons, true);
});
with_overrides(function (override) {

16
frontend_tests/node_tests/emoji.js
View File

@ -6,6 +6,8 @@ set_global('upload_widget', {});
zrequire('emoji_codes', 'generated/emoji/emoji_codes');
zrequire('emoji');
zrequire('markdown');
zrequire('util');
(function test_build_emoji_upload_widget() {
var build_widget_stub = false;
@ -74,3 +76,17 @@ zrequire('emoji');
emoji.get_canonical_name('non_existent');
assert(errored);
}());
(function test_translate_emoticons_to_names() {
global.emoji_codes = {
codepoint_to_name: {
'1f603': 'smiley',
},
};
var test_text = 'Testing :)';
var expected = 'Testing :smiley:';
var result = emoji.translate_emoticons_to_names(test_text);
assert.equal(expected, result);
}());

12
frontend_tests/node_tests/markdown.js
View File

@ -41,6 +41,7 @@ set_global('page_params', {
"https://zone_%(zone)s.zulip.net/ticket/%(id)s",
],
],
translate_emoticons: false,
});
set_global('blueslip', {error: function () {}});
@ -304,6 +305,17 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
assert.equal(expected, output);
});
// Here to arrange 100% test coverage for the new emoticons code
// path. TODO: Have a better way to test this setting in both
// states, once we implement the local echo feature properly.
// Probably a good technique would be to support toggling the
// page_params setting inside the `test_cases.forEach` loop above.
page_params.translate_emoticons = true;
var message = {raw_content: ":)"};
markdown.apply_markdown(message);
assert.equal('<p><span class="emoji emoji-1f603" title="smiley">:smiley:</span></p>', message.content);
page_params.translate_emoticons = false;
}());
(function test_subject_links() {

1
static/js/compose.js
View File

@ -885,6 +885,7 @@ exports.initialize = function () {
} else {
preview_html = rendered_content;
}
$("#preview_content").html(preview_html);
if (page_params.emojiset === "text") {
$("#preview_content").find(".emoji").replaceWith(function () {

28
static/js/emoji.js
View File

@ -17,6 +17,18 @@ var zulip_emoji = {
deactivated: false,
};
// Emoticons, and which emoji they should become (without colons). Duplicate
// emoji are allowed. Changes here should be mimicked in `zerver/lib/emoji.py`
// and `templates/zerver/help/enable-emoticon-translations.md`.
var EMOTICON_CONVERSIONS = {
':)': 'smiley',
'(:': 'smiley',
':(': 'slightly_frowning_face',
'<3': 'heart',
':|': 'expressionless',
':/': 'confused',
};
exports.update_emojis = function update_emojis(realm_emojis) {
// exports.all_realm_emojis is emptied before adding the realm-specific emoji to it.
// This makes sure that in case of deletion, the deleted realm_emojis don't
@ -105,6 +117,22 @@ exports.get_canonical_name = function (emoji_name) {
return emoji_codes.codepoint_to_name[codepoint];
};
// Translates emoticons in a string to their colon syntax.
exports.translate_emoticons_to_names = function translate_emoticons_to_names(text) {
var translated = text;
for (var emoticon in EMOTICON_CONVERSIONS) {
if (EMOTICON_CONVERSIONS.hasOwnProperty(emoticon)) {
var emoticon_reg_ex = new RegExp(util.escape_regexp(emoticon), "g");
translated = translated.replace(
emoticon_reg_ex,
':' + EMOTICON_CONVERSIONS[emoticon] + ':');
}
}
return translated;
};
return exports;
}());
if (typeof module !== 'undefined') {

16
static/js/markdown.js
View File

@ -358,6 +358,16 @@ exports.initialize = function () {
return fenced_code.process_fenced_code(src);
}
function preprocess_translate_emoticons(src) {
if (!page_params.translate_emoticons) {
return src;
}
// In this scenario, the message has to be from the user, so the only
// requirement should be that they have the setting on.
return emoji.translate_emoticons_to_names(src);
}
// Disable ordered lists
// We used GFM + tables, so replace the list start regex for that ruleset
// We remove the |[\d+]\. that matches the numbering in a numbered list
@ -406,7 +416,11 @@ exports.initialize = function () {
realmFilterHandler: handleRealmFilter,
texHandler: handleTex,
renderer: r,
preprocessors: [preprocess_code_blocks, preprocess_auto_olists],
preprocessors: [
preprocess_code_blocks,
preprocess_auto_olists,
preprocess_translate_emoticons,
],
});
};

1
static/js/server_events_dispatch.js
View File

@ -288,6 +288,7 @@ exports.dispatch_normal_event = function dispatch_normal_event(event) {
'left_side_userlist',
'timezone',
'twenty_four_hour_time',
'translate_emoticons',
];
if (_.contains(user_display_settings, event.setting_name)) {
page_params[event.setting_name] = event.setting;

26
static/js/settings_display.js
View File

@ -184,6 +184,31 @@ exports.set_up = function () {
},
});
});
$("#translate_emoticons").change(function () {
var data = {};
var setting_value = $("#translate_emoticons").is(":checked");
data.translate_emoticons = JSON.stringify(setting_value);
var context = {};
if (data.translate_emoticons === "true") {
context.new_mode = i18n.t("be");
} else {
context.new_mode = i18n.t("not be");
}
channel.patch({
url: '/json/settings/display',
data: data,
success: function () {
ui_report.success(i18n.t("Emoticons will now __new_mode__ translated!", context),
$('#display-settings-status').expectOne());
},
error: function (xhr) {
ui_report.error(i18n.t("Error updating emoticon translation setting"), xhr,
$('#display-settings-status').expectOne());
},
});
});
};
exports.report_emojiset_change = function () {
@ -214,6 +239,7 @@ function _update_page() {
$("#twenty_four_hour_time").prop('checked', page_params.twenty_four_hour_time);
$("#left_side_userlist").prop('checked', page_params.left_side_userlist);
$("#default_language_name").text(page_params.default_language_name);
$("#translate_emoticons").prop('checked', page_params.translate_emoticons);
}
exports.update_page = function () {

16
static/templates/settings/display-settings.handlebars
View File

@ -78,7 +78,21 @@
</select>
</div>
<h3>Emoji style</h3>
<h3 class="light">Emoji style</h3>
<div class="input-group side-padded-container">
<label class="checkbox">
<input type="checkbox" name="translate_emoticons" id="translate_emoticons"
{{#if page_params.translate_emoticons}}
checked="checked"
{{/if}} />
<span></span>
</label>
<label for="translate_emoticons" class="inline-block">
{{t "Translate emoticons (convert <code>:)</code> to 😃 in messages)" }}
</label>
</div>
<div class="input-group side-padded-container">
<div class="emojiset_choices grey-box">
{{#each page_params.emojiset_choices }}

92
templates/zerver/help/enable-emoticon-translations.md Normal file
View File

@ -0,0 +1,92 @@
# Enable emoticon translation
If you use emoticons like `:)` or `:/`, you can have them translated into
emoji equivalents like
<img
src="/static/generated/emoji/images/emoji/smile.png"
alt="smiley"
style="width: 3%;"
/>
or
<img
src="/static/generated/emoji/images/emoji/slightly_frowning_face.png"
alt="slightly_frowning_face"
style="width: 3%;"
/>
automatically by Zulip.
{!go-to-the.md!} [Display settings](/#settings/display-settings)
{!settings.md!}
2. Select the option labeled
**Translate emoticons (convert `:)` to 😃 in messages)**.
Then whenever you send a message with a supported emoticon, it will be
translated into an emoji.
## Current emoticon conversions
<table>
<thead>
<tr>
<th align="center">Emoticon</th>
<th align="center">Emoji</th>
</tr>
</thead>
<tbody>
<tr>
<td align="center"><code>:)</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/smiley.png"
alt="smiley"
style="width: 30%;">
</td>
</tr>
<tr>
<td align="center"><code>(:</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/smiley.png"
alt="smiley"
style="width: 30%;">
</td>
</tr>
<tr>
<td align="center"><code>:(</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/slightly_frowning_face.png"
alt="slightly_frowning_face"
style="width: 30%;">
</td>
</tr>
<tr>
<td align="center"><code>&lt;3</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/heart.png"
alt="heart"
style="width: 30%;">
</td>
</tr>
<tr>
<td align="center"><code>:|</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/expressionless.png"
alt="expressionless"
style="width: 30%;">
</td>
</tr>
<tr>
<td align="center"><code>:/</code></td>
<td align="center">
<img
src="/static/generated/emoji/images/emoji/confused.png"
alt="confused"
style="width: 30%;">
</td>
</tr>
</tbody>
</table>

1
templates/zerver/help/include/sidebar.md
View File

@ -99,6 +99,7 @@
* [Bots and integrations](/help/add-a-bot-or-integration)
* [Enable high contrast mode](/help/enable-high-contrast-mode)
* [Enable night mode](/help/enable-night-mode)
* [Enable emoticon translations](/help/enable-emoticon-translations)
* [Display the buddy list on narrow screens](/help/move-the-users-list-to-the-left-sidebar)
* [View organization statistics](/help/analytics)

25
zerver/fixtures/markdown_test_cases.json
View File

@ -336,9 +336,28 @@
},
{
"name": "many_emoji",
"input": "test :smile: again :poop:\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
"expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
"text_content": "test \ud83d\ude04 again \ud83d\udca9\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
"input": "test :smile: again :poop:\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
"expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
"text_content": "test \ud83d\ude04 again \ud83d\udca9\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
},
{
"name": "translate_emoticons",
"input": ":) foo :( bar <3 with space : ) real emoji :smiley:",
"expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slightly frowning face\">:slightly_frowning_face:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
"marked_expected_output": "<p>:) foo :( bar &lt;3 with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
"text_content": "\ud83d\ude03 foo \ud83d\ude41 bar \u2764 with space : ) real emoji \ud83d\ude03"
},
{
"name": "translate_emoticons_whitepsace",
"input": "a:) ;)b",
"expected_output": "<p>a:) ;)b</p>",
"text_content": "a:) ;)b"
},
{
"name": "translate_emoticons_in_code",
"input": "`:)`",
"expected_output": "<p><code>:)</code></p>",
"text_content": ":)"
},
{
"name": "random_emoji_1",

16
zerver/lib/bugdown/__init__.py
View File

@ -32,6 +32,7 @@ from markdown.extensions import codehilite
from zerver.lib.bugdown import fenced_code
from zerver.lib.bugdown.fenced_code import FENCE_RE
from zerver.lib.camo import get_camo_url
from zerver.lib.emoji import translate_emoticons, emoticon_regex
from zerver.lib.mention import possible_mentions, \
possible_user_group_mentions, extract_user_group
from zerver.lib.notifications import encode_stream
@ -998,6 +999,19 @@ def unicode_emoji_to_codepoint(unicode_emoji: Text) -> Text:
codepoint = '0' + codepoint
return codepoint
class EmoticonTranslation(markdown.inlinepatterns.Pattern):
""" Translates emoticons like `:)` into emoji like `:smile:`. """
def handleMatch(self, match: Match[Text]) -> Optional[Element]:
# If there is `db_data` and it is false, then don't do translating.
# If there is no `db_data`, such as during tests, translate.
if db_data is not None and not db_data['translate_emoticons']:
return None
emoticon = match.group('emoticon')
translated = translate_emoticons(emoticon)
name = translated[1:-1]
return make_emoji(name_to_codepoint[name], translated)
class UnicodeEmoji(markdown.inlinepatterns.Pattern):
def handleMatch(self, match: Match[Text]) -> Optional[Element]:
orig_syntax = match.group('syntax')
@ -1578,6 +1592,7 @@ class Bugdown(markdown.Extension):
Tex(r'\B(?<!\$)\$\$(?P<body>[^\n_$](\\\$|[^$\n])*)\$\$(?!\$)\B'),
'>backtick')
md.inlinePatterns.add('emoji', Emoji(EMOJI_REGEX), '_end')
md.inlinePatterns.add('translate_emoticons', EmoticonTranslation(emoticon_regex), '>emoji')
md.inlinePatterns.add('unicodeemoji', UnicodeEmoji(unicode_emoji_regex), '_end')
md.inlinePatterns.add('link', AtomicLinkPattern(markdown.inlinepatterns.LINK_RE, md), '>avatar')
@ -1956,6 +1971,7 @@ def do_convert(content: Text,
'realm_uri': message_realm.uri,
'sent_by_bot': sent_by_bot,
'stream_names': stream_name_info,
'translate_emoticons': message.sender.translate_emoticons,
}
try:

25
zerver/lib/emoji.py
View File

@ -14,6 +14,31 @@ from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
NAME_TO_CODEPOINT_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "name_to_codepoint.json")
CODEPOINT_TO_NAME_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "codepoint_to_name.json")
# Emoticons and which emoji they should become. Duplicate emoji are allowed.
# Changes here should be mimicked in `static/js/emoji.js`
# and `templates/zerver/help/enable-emoticon-translations.md`.
EMOTICON_CONVERSIONS = {
':)': ':smiley:',
'(:': ':smiley:',
':(': ':slightly_frowning_face:',
'<3': ':heart:',
':|': ':expressionless:',
':/': ':confused:',
}
possible_emoticons = EMOTICON_CONVERSIONS.keys()
possible_emoticon_regexes = map(re.escape, possible_emoticons) # type: ignore # AnyStr/str issues
emoticon_regex = '(?<![^\s])(?P<emoticon>(' + ')|('.join(possible_emoticon_regexes) + '))(?![\S])' # type: ignore # annoying
# Translates emoticons to their colon syntax, e.g. `:smiley:`.
def translate_emoticons(text: Text) -> Text:
translated = text
for emoticon in EMOTICON_CONVERSIONS:
translated = re.sub(re.escape(emoticon), EMOTICON_CONVERSIONS[emoticon], translated)
return translated
with open(NAME_TO_CODEPOINT_PATH) as fp:
name_to_codepoint = ujson.load(fp)

20
zerver/migrations/0142_userprofile_translate_emoticons.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-02-19 22:27
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('zerver', '0141_change_usergroup_description_to_textfield'),
]
operations = [
migrations.AddField(
model_name='userprofile',
name='translate_emoticons',
field=models.BooleanField(default=False),
),
]

2
zerver/models.py
View File

@ -588,6 +588,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin):
default_language = models.CharField(default=u'en', max_length=MAX_LANGUAGE_ID_LENGTH) # type: Text
high_contrast_mode = models.BooleanField(default=False) # type: bool
night_mode = models.BooleanField(default=False) # type: bool
translate_emoticons = models.BooleanField(default=False) # type: bool
# Hours to wait before sending another email to a user
EMAIL_REMINDER_WAITPERIOD = 24
@ -650,6 +651,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin):
twenty_four_hour_time=bool,
high_contrast_mode=bool,
night_mode=bool,
translate_emoticons=bool,
)
notification_setting_types = dict(

11
zerver/tests/test_bugdown.py
View File

@ -4,6 +4,7 @@ from django.test import TestCase, override_settings
from zerver.lib import bugdown
from zerver.lib.actions import (
do_set_user_display_setting,
do_remove_realm_emoji,
do_set_alert_words,
get_realm,
@ -617,6 +618,16 @@ class BugdownTest(ZulipTestCase):
converted = bugdown_convert(msg)
self.assertEqual(converted, u'<p><span class="emoji emoji-2615" title="coffee">:coffee:</span><span class="emoji emoji-2615" title="coffee">:coffee:</span></p>')
def test_no_translate_emoticons_if_off(self) -> None:
user_profile = self.example_user('othello')
do_set_user_display_setting(user_profile, 'translate_emoticons', False)
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = u':)'
expected = u'<p>:)</p>'
converted = render_markdown(msg, content)
self.assertEqual(converted, expected)
def test_same_markup(self) -> None:
msg = u'\u2615' # ☕
unicode_converted = bugdown_convert(msg)

1
zerver/tests/test_home.py
View File

@ -156,6 +156,7 @@ class HomeTest(ZulipTestCase):
"subscriptions",
"test_suite",
"timezone",
"translate_emoticons",
"twenty_four_hour_time",
"unread_msgs",
"unsubscribed",

1
zerver/views/user_settings.py
View File

@ -129,6 +129,7 @@ def update_display_settings_backend(
twenty_four_hour_time: Optional[bool]=REQ(validator=check_bool, default=None),
high_contrast_mode: Optional[bool]=REQ(validator=check_bool, default=None),
night_mode: Optional[bool]=REQ(validator=check_bool, default=None),
translate_emoticons: Optional[bool]=REQ(validator=check_bool, default=None),
default_language: Optional[bool]=REQ(validator=check_string, default=None),
left_side_userlist: Optional[bool]=REQ(validator=check_bool, default=None),
emojiset: Optional[str]=REQ(validator=check_string, default=None),