2021-05-11 09:36:58 +02:00
|
|
|
import {strict as assert} from "assert";
|
|
|
|
|
|
|
|
import type {Page} from "puppeteer";
|
|
|
|
|
2023-02-22 23:04:11 +01:00
|
|
|
import * as common from "./lib/common";
|
2021-05-11 09:36:58 +02:00
|
|
|
|
|
|
|
async function test_add_linkifier(page: Page): Promise<void> {
|
|
|
|
await page.waitForSelector(".admin-linkifier-form", {visible: true});
|
|
|
|
await common.fill_form(page, "form.admin-linkifier-form", {
|
|
|
|
pattern: "#(?P<id>[0-9]+)",
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template: "https://trac.example.com/ticket/{id}",
|
2021-05-11 09:36:58 +02:00
|
|
|
});
|
|
|
|
await page.click("form.admin-linkifier-form button.button");
|
|
|
|
|
|
|
|
const admin_linkifier_status_selector = "div#admin-linkifier-status";
|
|
|
|
await page.waitForSelector(admin_linkifier_status_selector, {visible: true});
|
|
|
|
const admin_linkifier_status = await common.get_text_from_selector(
|
|
|
|
page,
|
|
|
|
admin_linkifier_status_selector,
|
|
|
|
);
|
|
|
|
assert.strictEqual(admin_linkifier_status, "Custom linkifier added!");
|
|
|
|
|
|
|
|
await page.waitForSelector(".linkifier_row", {visible: true});
|
|
|
|
assert.strictEqual(
|
|
|
|
await common.get_text_from_selector(page, ".linkifier_row span.linkifier_pattern"),
|
|
|
|
"#(?P<id>[0-9]+)",
|
|
|
|
);
|
|
|
|
assert.strictEqual(
|
2022-10-05 22:52:26 +02:00
|
|
|
await common.get_text_from_selector(page, ".linkifier_row span.linkifier_url_template"),
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
"https://trac.example.com/ticket/{id}",
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function test_delete_linkifier(page: Page): Promise<void> {
|
|
|
|
await page.click(".linkifier_row .delete");
|
2022-09-07 20:29:18 +02:00
|
|
|
await common.wait_for_micromodal_to_open(page);
|
|
|
|
await page.click("#confirm_delete_linkifiers_modal .dialog_submit_button");
|
|
|
|
await common.wait_for_micromodal_to_close(page);
|
2021-05-11 09:36:58 +02:00
|
|
|
await page.waitForSelector(".linkifier_row", {hidden: true});
|
|
|
|
}
|
|
|
|
|
|
|
|
async function test_add_invalid_linkifier_pattern(page: Page): Promise<void> {
|
|
|
|
await page.waitForSelector(".admin-linkifier-form", {visible: true});
|
|
|
|
await common.fill_form(page, "form.admin-linkifier-form", {
|
CVE-2021-41115: Use re2 for user-supplied linkifier patterns.
Zulip attempts to validate that the regular expressions that admins
enter for linkifiers are well-formatted, and only contain a specific
subset of regex grammar. The process of checking these
properties (via a regex!) can cause denial-of-service via
backtracking.
Furthermore, this validation itself does not prevent the creation of
linkifiers which themselves cause denial-of-service when they are
executed. As the validator accepts literally anything inside of a
`(?P<word>...)` block, any quadratic backtracking expression can be
hidden therein.
Switch user-provided linkifier patterns to be matched in the Markdown
processor by the `re2` library, which is guaranteed constant-time.
This somewhat limits the possible features of the regular
expression (notably, look-head and -behind, and back-references);
however, these features had never been advertised as working in the
context of linkifiers.
A migration removes any existing linkifiers which would not function
under re2, after printing them for posterity during the upgrade; they
are unlikely to be common, and are impossible to fix automatically.
The denial-of-service in the linkifier validator was discovered by
@erik-krogh and @yoff, as GHSL-2021-118.
2021-09-29 01:27:54 +02:00
|
|
|
pattern: "(foo",
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template: "https://trac.example.com/ticket/{id}",
|
2021-05-11 09:36:58 +02:00
|
|
|
});
|
|
|
|
await page.click("form.admin-linkifier-form button.button");
|
|
|
|
|
2021-06-07 11:38:57 +02:00
|
|
|
await page.waitForSelector("div#admin-linkifier-status", {visible: true});
|
2021-05-11 09:36:58 +02:00
|
|
|
assert.strictEqual(
|
2021-06-07 11:38:57 +02:00
|
|
|
await common.get_text_from_selector(page, "div#admin-linkifier-status"),
|
CVE-2021-41115: Use re2 for user-supplied linkifier patterns.
Zulip attempts to validate that the regular expressions that admins
enter for linkifiers are well-formatted, and only contain a specific
subset of regex grammar. The process of checking these
properties (via a regex!) can cause denial-of-service via
backtracking.
Furthermore, this validation itself does not prevent the creation of
linkifiers which themselves cause denial-of-service when they are
executed. As the validator accepts literally anything inside of a
`(?P<word>...)` block, any quadratic backtracking expression can be
hidden therein.
Switch user-provided linkifier patterns to be matched in the Markdown
processor by the `re2` library, which is guaranteed constant-time.
This somewhat limits the possible features of the regular
expression (notably, look-head and -behind, and back-references);
however, these features had never been advertised as working in the
context of linkifiers.
A migration removes any existing linkifiers which would not function
under re2, after printing them for posterity during the upgrade; they
are unlikely to be common, and are impossible to fix automatically.
The denial-of-service in the linkifier validator was discovered by
@erik-krogh and @yoff, as GHSL-2021-118.
2021-09-29 01:27:54 +02:00
|
|
|
"Failed: Bad regular expression: missing ): (foo",
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function test_edit_linkifier(page: Page): Promise<void> {
|
|
|
|
await page.click(".linkifier_row .edit");
|
2021-07-04 08:47:08 +02:00
|
|
|
await common.wait_for_micromodal_to_open(page);
|
2021-05-11 09:36:58 +02:00
|
|
|
await common.fill_form(page, "form.linkifier-edit-form", {
|
|
|
|
pattern: "(?P<num>[0-9a-f]{40})",
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template: "https://trac.example.com/commit/{num}",
|
2021-05-11 09:36:58 +02:00
|
|
|
});
|
2021-07-05 20:30:47 +02:00
|
|
|
await page.click(".dialog_submit_button");
|
2021-05-11 09:36:58 +02:00
|
|
|
|
2021-07-05 20:30:47 +02:00
|
|
|
await page.waitForSelector("#dialog_widget_modal", {hidden: true});
|
2021-07-04 08:47:08 +02:00
|
|
|
await common.wait_for_micromodal_to_close(page);
|
2021-05-11 09:36:58 +02:00
|
|
|
|
|
|
|
await page.waitForSelector(".linkifier_row", {visible: true});
|
|
|
|
await page.waitForFunction(
|
|
|
|
() => document.querySelector(".linkifier_pattern")?.textContent === "(?P<num>[0-9a-f]{40})",
|
|
|
|
);
|
|
|
|
assert.strictEqual(
|
2022-10-05 22:52:26 +02:00
|
|
|
await common.get_text_from_selector(page, ".linkifier_row span.linkifier_url_template"),
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
"https://trac.example.com/commit/{num}",
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function test_edit_invalid_linkifier(page: Page): Promise<void> {
|
|
|
|
await page.click(".linkifier_row .edit");
|
2021-07-04 08:47:08 +02:00
|
|
|
await common.wait_for_micromodal_to_open(page);
|
2021-05-11 09:36:58 +02:00
|
|
|
await common.fill_form(page, "form.linkifier-edit-form", {
|
CVE-2021-41115: Use re2 for user-supplied linkifier patterns.
Zulip attempts to validate that the regular expressions that admins
enter for linkifiers are well-formatted, and only contain a specific
subset of regex grammar. The process of checking these
properties (via a regex!) can cause denial-of-service via
backtracking.
Furthermore, this validation itself does not prevent the creation of
linkifiers which themselves cause denial-of-service when they are
executed. As the validator accepts literally anything inside of a
`(?P<word>...)` block, any quadratic backtracking expression can be
hidden therein.
Switch user-provided linkifier patterns to be matched in the Markdown
processor by the `re2` library, which is guaranteed constant-time.
This somewhat limits the possible features of the regular
expression (notably, look-head and -behind, and back-references);
however, these features had never been advertised as working in the
context of linkifiers.
A migration removes any existing linkifiers which would not function
under re2, after printing them for posterity during the upgrade; they
are unlikely to be common, and are impossible to fix automatically.
The denial-of-service in the linkifier validator was discovered by
@erik-krogh and @yoff, as GHSL-2021-118.
2021-09-29 01:27:54 +02:00
|
|
|
pattern: "#(?P<id>d????)",
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template: "{id",
|
2021-05-11 09:36:58 +02:00
|
|
|
});
|
2021-07-05 20:30:47 +02:00
|
|
|
await page.click(".dialog_submit_button");
|
2021-05-11 09:36:58 +02:00
|
|
|
|
2021-07-05 20:30:47 +02:00
|
|
|
const edit_linkifier_pattern_status_selector = "div#dialog_error";
|
2021-05-11 09:36:58 +02:00
|
|
|
await page.waitForSelector(edit_linkifier_pattern_status_selector, {visible: true});
|
|
|
|
const edit_linkifier_pattern_status = await common.get_text_from_selector(
|
|
|
|
page,
|
|
|
|
edit_linkifier_pattern_status_selector,
|
|
|
|
);
|
|
|
|
assert.strictEqual(
|
|
|
|
edit_linkifier_pattern_status,
|
CVE-2021-41115: Use re2 for user-supplied linkifier patterns.
Zulip attempts to validate that the regular expressions that admins
enter for linkifiers are well-formatted, and only contain a specific
subset of regex grammar. The process of checking these
properties (via a regex!) can cause denial-of-service via
backtracking.
Furthermore, this validation itself does not prevent the creation of
linkifiers which themselves cause denial-of-service when they are
executed. As the validator accepts literally anything inside of a
`(?P<word>...)` block, any quadratic backtracking expression can be
hidden therein.
Switch user-provided linkifier patterns to be matched in the Markdown
processor by the `re2` library, which is guaranteed constant-time.
This somewhat limits the possible features of the regular
expression (notably, look-head and -behind, and back-references);
however, these features had never been advertised as working in the
context of linkifiers.
A migration removes any existing linkifiers which would not function
under re2, after printing them for posterity during the upgrade; they
are unlikely to be common, and are impossible to fix automatically.
The denial-of-service in the linkifier validator was discovered by
@erik-krogh and @yoff, as GHSL-2021-118.
2021-09-29 01:27:54 +02:00
|
|
|
"Failed: Bad regular expression: bad repetition operator: ????",
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
|
|
|
|
2022-10-05 22:52:26 +02:00
|
|
|
const edit_linkifier_template_status_selector = "div#edit-linkifier-template-status";
|
|
|
|
await page.waitForSelector(edit_linkifier_template_status_selector, {visible: true});
|
|
|
|
const edit_linkifier_template_status = await common.get_text_from_selector(
|
2021-05-11 09:36:58 +02:00
|
|
|
page,
|
2022-10-05 22:52:26 +02:00
|
|
|
edit_linkifier_template_status_selector,
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
2022-10-05 22:52:26 +02:00
|
|
|
assert.strictEqual(edit_linkifier_template_status, "Failed: Invalid URL template.");
|
2021-05-11 09:36:58 +02:00
|
|
|
|
2021-07-04 08:47:08 +02:00
|
|
|
await page.click(".dialog_cancel_button");
|
2021-07-05 20:30:47 +02:00
|
|
|
await page.waitForSelector("#dialog_widget_modal", {hidden: true});
|
2021-05-11 09:36:58 +02:00
|
|
|
|
|
|
|
await page.waitForSelector(".linkifier_row", {visible: true});
|
|
|
|
assert.strictEqual(
|
|
|
|
await common.get_text_from_selector(page, ".linkifier_row span.linkifier_pattern"),
|
|
|
|
"(?P<num>[0-9a-f]{40})",
|
|
|
|
);
|
|
|
|
assert.strictEqual(
|
2022-10-05 22:52:26 +02:00
|
|
|
await common.get_text_from_selector(page, ".linkifier_row span.linkifier_url_template"),
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
"https://trac.example.com/commit/{num}",
|
2021-05-11 09:36:58 +02:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function linkifier_test(page: Page): Promise<void> {
|
|
|
|
await common.log_in(page);
|
|
|
|
await common.manage_organization(page);
|
|
|
|
await page.click("li[data-section='linkifier-settings']");
|
|
|
|
|
|
|
|
await test_add_linkifier(page);
|
|
|
|
await test_edit_linkifier(page);
|
|
|
|
await test_edit_invalid_linkifier(page);
|
|
|
|
await test_add_invalid_linkifier_pattern(page);
|
|
|
|
await test_delete_linkifier(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
common.run_test(linkifier_test);
|