"use strict"; const {strict: assert} = require("assert"); const markdown_test_cases = require("../../zerver/tests/fixtures/markdown_test_cases.json"); const markdown_assert = require("../zjsunit/markdown_assert"); const {set_global, zrequire} = require("../zjsunit/namespace"); const {run_test} = require("../zjsunit/test"); const {make_zjquery} = require("../zjsunit/zjquery"); zrequire("hash_util"); const emoji = zrequire("emoji", "shared/js/emoji"); const emoji_codes = zrequire("emoji_codes", "generated/emoji/emoji_codes.json"); const pygments_data = zrequire("pygments_data", "generated/pygments_data.json"); const fenced_code = zrequire("fenced_code", "shared/js/fenced_code"); const markdown_config = zrequire("markdown_config"); const marked = zrequire("marked", "third/marked/lib/marked"); const markdown = zrequire("markdown"); zrequire("message_store"); const people = zrequire("people"); const stream_data = zrequire("stream_data"); const user_groups = zrequire("user_groups"); set_global("location", { origin: "http://zulip.zulipdev.com", }); const emoji_params = { realm_emoji: { 1: { id: 1, name: "burrito", source_url: "/static/generated/emoji/images/emoji/burrito.png", deactivated: false, }, }, emoji_codes, }; set_global("page_params", { realm_users: [], realm_filters: [ ["#(?P[0-9]{2,8})", "https://trac.example.com/ticket/%(id)s"], ["ZBUG_(?P[0-9]{2,8})", "https://trac2.zulip.net/ticket/%(id)s"], [ "ZGROUP_(?P[0-9]{2,8}):(?P[0-9]{1,8})", "https://zone_%(zone)s.zulip.net/ticket/%(id)s", ], ], translate_emoticons: false, }); function Image() { return {}; } set_global("Image", Image); emoji.initialize(emoji_params); fenced_code.initialize(pygments_data); const doc = ""; set_global("document", doc); set_global("$", make_zjquery()); const cordelia = { full_name: "Cordelia Lear", user_id: 101, email: "cordelia@zulip.com", }; people.add_active_user(cordelia); people.add_active_user({ full_name: "Leo", user_id: 102, email: "leo@zulip.com", }); people.add_active_user({ full_name: "Bobby

Tables

", user_id: 103, email: "bobby@zulip.com", }); people.add_active_user({ full_name: "Mark Twin", user_id: 104, email: "twin1@zulip.com", }); people.add_active_user({ full_name: "Mark Twin", user_id: 105, email: "twin2@zulip.com", }); people.add_active_user({ full_name: "Brother of Bobby|123", user_id: 106, email: "bobby2@zulip.com", }); people.add_active_user({ full_name: "& & &", user_id: 107, email: "ampampamp@zulip.com", }); people.initialize_current_user(cordelia.user_id); const hamletcharacters = { name: "hamletcharacters", id: 1, description: "Characters of Hamlet", members: [cordelia.user_id], }; const backend = { name: "Backend", id: 2, description: "Backend team", members: [], }; const edgecase_group = { name: "Bobby

Tables

", id: 3, description: "HTML Syntax to check for Markdown edge cases.", members: [], }; const amp_group = { name: "& & &", id: 4, description: "Check ampersand escaping", members: [], }; user_groups.add(hamletcharacters); user_groups.add(backend); user_groups.add(edgecase_group); user_groups.add(amp_group); const denmark = { subscribed: false, color: "blue", name: "Denmark", stream_id: 1, is_muted: true, }; const social = { subscribed: true, color: "red", name: "social", stream_id: 2, is_muted: false, invite_only: true, }; const edgecase_stream = { subscribed: true, color: "green", name: "Bobby

Tables

", stream_id: 3, is_muted: false, }; const edgecase_stream_2 = { subscribed: true, color: "yellow", name: "Bobby { const input = "\n```\nfenced code\n```\n\nand then after\n"; const expected = '\n\n
fenced code\n
\n\n\nand then after\n\n'; const output = fenced_code.process_fenced_code(input); assert.equal(output, expected); }); markdown.initialize(page_params.realm_filters, markdown_config.get_helpers()); run_test("markdown_detection", () => { const no_markup = [ "This is a plaintext message", "This is a plaintext: message", "This is a :plaintext message", "This is a :plaintext message: message", "Contains a not an image.jpeg/ok file", "Contains a not an http://www.google.com/ok/image.png/stop file", "No png to be found here, a png", "No user mention **leo**", "No user mention @what there", "No group mention *hamletcharacters*", 'We like to code\n~~~\ndef code():\n we = "like to do"\n~~~', "This is a\nmultiline :emoji: here\n message", "This is an :emoji: message", "User Mention @**leo**", "User Mention @**leo f**", "User Mention @**leo with some name**", "Group Mention @*hamletcharacters*", "Stream #**Verona**", ]; const markup = [ "Contains a https://zulip.com/image.png file", "Contains a https://zulip.com/image.jpg file", "https://zulip.com/image.jpg", "also https://zulip.com/image.jpg", "https://zulip.com/image.jpg too", "Contains a zulip.com/foo.jpeg file", "Contains a https://zulip.com/image.png file", "Twitter URL https://twitter.com/jacobian/status/407886996565016579", "https://twitter.com/jacobian/status/407886996565016579", "then https://twitter.com/jacobian/status/407886996565016579", "Twitter URL http://twitter.com/jacobian/status/407886996565016579", "YouTube URL https://www.youtube.com/watch?v=HHZ8iqswiCw&feature=youtu.be&a", ]; for (const content of no_markup) { assert.equal(markdown.contains_backend_only_syntax(content), false); } for (const content of markup) { assert.equal(markdown.contains_backend_only_syntax(content), true); } }); run_test("marked_shared", () => { const tests = markdown_test_cases.regular_tests; for (const test of tests) { // Ignore tests if specified if (test.ignore === true) { continue; } const message = {raw_content: test.input}; page_params.translate_emoticons = test.translate_emoticons || false; markdown.apply_markdown(message); const output = message.content; const error_message = `Failure in test: ${test.name}`; if (test.marked_expected_output) { markdown_assert.notEqual(test.expected_output, output, error_message); markdown_assert.equal(test.marked_expected_output, output, error_message); } else if (test.backend_only_rendering) { assert.equal(markdown.contains_backend_only_syntax(test.input), true); } else { markdown_assert.equal(test.expected_output, output, error_message); } } }); run_test("message_flags", () => { let message = {raw_content: "@**Leo**"}; markdown.apply_markdown(message); assert(!message.mentioned); assert(!message.mentioned_me_directly); message = {raw_content: "@**Cordelia Lear**"}; markdown.apply_markdown(message); assert(message.mentioned); assert(message.mentioned_me_directly); message = {raw_content: "@**all**"}; markdown.apply_markdown(message); assert(message.mentioned); assert(!message.mentioned_me_directly); }); run_test("marked", () => { const test_cases = [ {input: "hello", expected: "

hello

"}, {input: "hello there", expected: "

hello there

"}, {input: "hello **bold** for you", expected: "

hello bold for you

"}, { input: "hello ***foo*** for you", expected: "

hello foo for you

", }, {input: "__hello__", expected: "

__hello__

"}, { input: "\n```\nfenced code\n```\n\nand then after\n", expected: '
fenced code\n
\n

and then after

', }, { input: "\n```\n fenced code trailing whitespace \n```\n\nand then after\n", expected: '
    fenced code trailing whitespace\n
\n

and then after

', }, { input: "* a\n* list \n* here", expected: "
    \n
  • a
  • \n
  • list
  • \n
  • here
  • \n
", }, { input: "\n```c#\nfenced code special\n```\n\nand then after\n", expected: '
fenced code special\n
\n

and then after

', }, { input: "\n```vb.net\nfenced code dot\n```\n\nand then after\n", expected: '
fenced code dot\n
\n

and then after

', }, { input: "Some text first\n* a\n* list \n* here\n\nand then after", expected: "

Some text first

\n
    \n
  • a
  • \n
  • list
  • \n
  • here
  • \n
\n

and then after

", }, { input: "1. an\n2. ordered \n3. list", expected: "
    \n
  1. an
  2. \n
  3. ordered
  4. \n
  5. list
  6. \n
", }, { input: "\n~~~quote\nquote this for me\n~~~\nthanks\n", expected: "
\n

quote this for me

\n
\n

thanks

", }, { input: "This is a @**CordeLIA Lear** mention", expected: '

This is a @Cordelia Lear mention

', }, { input: "These @ @**** are not mentions", expected: "

These @ @** are not mentions

", }, { input: "These # #**** are not mentions", expected: "

These # #** are not mentions

", }, {input: "These @* are not mentions", expected: "

These @* are not mentions

"}, { input: "These #* #*** are also not mentions", expected: "

These #* #*** are also not mentions

", }, { input: "This is a #**Denmark** stream link", expected: '

This is a #Denmark stream link

', }, { input: "This is #**Denmark** and #**social** stream links", expected: '

This is #Denmark and #social stream links

', }, { input: "And this is a #**wrong** stream link", expected: "

And this is a #**wrong** stream link

", }, { input: "This is a #**Denmark>some topic** stream_topic link", expected: '

This is a #Denmark > some topic stream_topic link

', }, { input: "This has two links: #**Denmark>some topic** and #**social>other topic**.", expected: '

This has two links: #Denmark > some topic and #social > other topic.

', }, { input: "This is not a #**Denmark>** stream_topic link", expected: "

This is not a #**Denmark>** stream_topic link

", }, { input: "mmm...:burrito:s", expected: '

mmm...:burrito:s

', }, { input: "This is an :poop: message", expected: '

This is an :poop: message

', }, { input: "\uD83D\uDCA9", expected: '

:poop:

', }, { input: "Silent mention: @_**Cordelia Lear**", expected: '

Silent mention: Cordelia Lear

', }, { input: "> Mention in quote: @**Cordelia Lear**\n\nMention outside quote: @**Cordelia Lear**", expected: '
\n

Mention in quote: Cordelia Lear

\n
\n

Mention outside quote: @Cordelia Lear

', }, // Test only those realm filters which don't return True for // `contains_backend_only_syntax()`. Those which return True // are tested separately. { input: "This is a realm filter #1234 with text after it", expected: '

This is a realm filter #1234 with text after it

', }, {input: "#1234is not a realm filter.", expected: "

#1234is not a realm filter.

"}, { input: "A pattern written as #1234is not a realm filter.", expected: "

A pattern written as #1234is not a realm filter.

", }, { input: "This is a realm filter with ZGROUP_123:45 groups", expected: '

This is a realm filter with ZGROUP_123:45 groups

', }, {input: "Test *italic*", expected: "

Test italic

"}, { input: "T\n#**Denmark**", expected: '

T
\n#Denmark

', }, { input: "T\n@**Cordelia Lear**", expected: '

T
\n@Cordelia Lear

', }, { input: "@**Mark Twin|104** and @**Mark Twin|105** are out to confuse you.", expected: '

@Mark Twin and @Mark Twin are out to confuse you.

', }, {input: "@**Invalid User|1234**", expected: "

@**Invalid User|1234**

"}, { input: "@**Cordelia LeAR|103** has a wrong user_id.", expected: "

@**Cordelia LeAR|103** has a wrong user_id.

", }, { input: "@**Brother of Bobby|123** is really the full name.", expected: '

@Brother of Bobby|123 is really the full name.

', }, { input: "@**Brother of Bobby|123|106**", expected: '

@Brother of Bobby|123

', }, {input: "T\n@hamletcharacters", expected: "

T
\n@hamletcharacters

"}, { input: "T\n@*hamletcharacters*", expected: '

T
\n@hamletcharacters

', }, {input: "T\n@*notagroup*", expected: "

T
\n@*notagroup*

"}, { input: "T\n@*backend*", expected: '

T
\n@Backend

', }, {input: "@*notagroup*", expected: "

@*notagroup*

"}, { input: "This is a realm filter `hello` with text after it", expected: "

This is a realm filter hello with text after it

", }, // Test the emoticon conversion {input: ":)", expected: "

:)

"}, { input: ":)", expected: '

:smile:

', translate_emoticons: true, }, // Test HTML escaping in custom Zulip rules { input: "@**

The Rogue One

**", expected: "

@**<h1>The Rogue One</h1>**

", }, { input: "#**

The Rogue One

**", expected: "

#**<h1>The Rogue One</h1>**

", }, { input: ":

The Rogue One

:", expected: "

:<h1>The Rogue One</h1>:

", }, {input: "@**O'Connell**", expected: "

@**O'Connell**

"}, { input: "@*Bobby

Tables

*", expected: '

@Bobby <h1>Tables</h1>

', }, { input: "@*& & &amp;*", expected: '

@& & &amp;

', }, { input: "@**Bobby

Tables

**", expected: '

@Bobby <h1>Tables</h1>

', }, { input: "@**& & &amp;**", expected: '

@& & &amp;

', }, { input: "#**Bobby

Tables

**", expected: '

#Bobby <h1 > Tables</h1>

', }, { input: "#**& & &amp;**", expected: '

#& & &amp;

', }, { input: "#**& & &amp;>& & &amp;**", expected: '

#& & &amp; > & & &amp;

', }, ]; for (const test_case of test_cases) { // Disable emoji conversion by default. page_params.translate_emoticons = test_case.translate_emoticons || false; const input = test_case.input; const expected = test_case.expected; const message = {raw_content: input}; markdown.apply_markdown(message); const output = message.content; assert.equal(output, expected); } }); run_test("topic_links", () => { let message = {type: "stream", topic: "No links here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 0); message = {type: "stream", topic: "One #123 link here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 1); assert.equal(message.topic_links[0], "https://trac.example.com/ticket/123"); message = {type: "stream", topic: "Two #123 #456 link here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 2); assert.equal(message.topic_links[0], "https://trac.example.com/ticket/123"); assert.equal(message.topic_links[1], "https://trac.example.com/ticket/456"); message = {type: "stream", topic: "New ZBUG_123 link here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 1); assert.equal(message.topic_links[0], "https://trac2.zulip.net/ticket/123"); message = {type: "stream", topic: "New ZBUG_123 with #456 link here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 2); assert(message.topic_links.includes("https://trac2.zulip.net/ticket/123")); assert(message.topic_links.includes("https://trac.example.com/ticket/456")); message = {type: "stream", topic: "One ZGROUP_123:45 link here"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 1); assert.equal(message.topic_links[0], "https://zone_45.zulip.net/ticket/123"); message = {type: "stream", topic: "Hello https://google.com"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 1); assert.equal(message.topic_links[0], "https://google.com"); message = {type: "stream", topic: "#456 https://google.com https://github.com"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 3); assert(message.topic_links.includes("https://google.com")); assert(message.topic_links.includes("https://github.com")); assert(message.topic_links.includes("https://trac.example.com/ticket/456")); message = {type: "not-stream"}; markdown.add_topic_links(message); assert.equal(message.topic_links.length, 0); }); run_test("message_flags", () => { let input = "/me is testing this"; let message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.is_me_message, true); assert(!message.unread); input = "/me is testing\nthis"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.is_me_message, true); input = "testing this @**all** @**Cordelia Lear**"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.is_me_message, false); assert.equal(message.mentioned, true); assert.equal(message.mentioned_me_directly, true); input = "test @**everyone**"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.is_me_message, false); assert.equal(message.mentioned, true); assert.equal(message.mentioned_me_directly, false); input = "test @**stream**"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.is_me_message, false); assert.equal(message.mentioned, true); assert.equal(message.mentioned_me_directly, false); input = "test @all"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); input = "test @everyone"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); input = "test @any"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); input = "test @alleycat.com"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); input = "test @*hamletcharacters*"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, true); input = "test @*backend*"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); input = "test @**invalid_user**"; message = {topic: "No links here", raw_content: input}; markdown.apply_markdown(message); assert.equal(message.mentioned, false); }); run_test("backend_only_realm_filters", () => { const backend_only_realm_filters = [ "Here is the PR-#123.", "Function abc() was introduced in (PR)#123.", ]; for (const content of backend_only_realm_filters) { assert.equal(markdown.contains_backend_only_syntax(content), true); } }); run_test("python_to_js_filter", () => { // The only way to reach python_to_js_filter is indirectly, hence the call // to update_realm_filter_rules. markdown.update_realm_filter_rules([["/a(?im)a/g"], ["/a(?L)a/g"]]); let actual_value = marked.InlineLexer.rules.zulip.realm_filters; let expected_value = [/\/aa\/g(?!\w)/gim, /\/aa\/g(?!\w)/g]; assert.deepEqual(actual_value, expected_value); // Test case with multiple replacements. markdown.update_realm_filter_rules([ ["#cf(?P\\d+)(?P[A-Z][\\dA-Z]*)", "http://google.com"], ]); actual_value = marked.InlineLexer.rules.zulip.realm_filters; expected_value = [/#cf(\d+)([A-Z][\dA-Z]*)(?!\w)/g]; assert.deepEqual(actual_value, expected_value); // Test incorrect syntax. blueslip.expect( "error", "python_to_js_filter: Invalid regular expression: /!@#@(!#&((!&(@#((?!\\w)/: Unterminated group", ); markdown.update_realm_filter_rules([["!@#@(!#&((!&(@#(", "http://google.com"]]); actual_value = marked.InlineLexer.rules.zulip.realm_filters; expected_value = []; assert.deepEqual(actual_value, expected_value); }); run_test("translate_emoticons_to_names", () => { // Simple test const test_text = "Testing :)"; const expected = "Testing :smile:"; const result = markdown.translate_emoticons_to_names(test_text); assert.equal(result, expected); // Extensive tests. // The following code loops over the test cases and each emoticon conversion // to generate multiple test cases. const testcases = [ {name: "only emoticon", original: "", expected: ""}, {name: "space at start", original: " ", expected: " "}, {name: "space at end", original: " ", expected: " "}, {name: "symbol at end", original: "!", expected: "!"}, {name: "symbol at start", original: "Hello,", expected: "Hello,"}, {name: "after a word", original: "Hello", expected: "Hello"}, {name: "between words", original: "HelloWorld", expected: "HelloWorld"}, { name: "end of sentence", original: "End of sentence. ", expected: "End of sentence. ", }, { name: "between symbols", original: "Hello.! World.", expected: "Hello.! World.", }, { name: "before end of sentence", original: "Hello !", expected: "Hello !", }, ]; for (const [shortcut, full_name] of Object.entries(emoji_codes.emoticon_conversions)) { for (const t of testcases) { const converted_value = full_name; let original = t.original; let expected = t.expected; original = original.replace(/()/g, shortcut); expected = expected .replace(/()/g, shortcut) .replace(/()/g, converted_value); const result = markdown.translate_emoticons_to_names(original); assert.equal(result, expected); } } }); run_test("missing unicode emojis", (override) => { const message = {raw_content: "\u{1F6B2}"}; markdown.apply_markdown(message); assert.equal( message.content, '

:bike:

', ); override(emoji, "get_emoji_name", (codepoint) => { // Now simulate that we don't know any emoji names. assert.equal(codepoint, "1f6b2"); // return undefined }); markdown.apply_markdown(message); assert.equal(message.content, "

\u{1F6B2}

"); });