realm_playgrounds: Replace url_prefix with url_template.

Dropping support for url_prefix for RealmPlayground, the server now uses
url_template instead only for playground creation, retrieval and audit
logging upon removal.

This does the necessary handling so that url_template is expanded with
the extracted code.

Fixes #25723.

Signed-off-by: Zixuan James Li <p359101898@gmail.com>
This commit is contained in:
Zixuan James Li 2023-05-26 23:04:50 -04:00 committed by Tim Abbott
parent c4bc0ad589
commit 000761ac0c
25 changed files with 142 additions and 92 deletions

View File

@ -20,6 +20,14 @@ format used by the Zulip server that they are interacting with.
## Changes in Zulip 8.0
**Feature level 196**
* [`POST /realm/playgrounds`](/api/add-code-playground): `url_prefix` is
replaced by `url_template`, which only accepts [RFC 6570][rfc6570] compliant
URL templates. The old prefix format is no longer supported.
* [`GET /events`](/api/get-events), [`POST /register`](/api/register-queue):
`url_prefix` is replaced by `url_template` in `realm_playgrounds` events.
**Feature level 195**
* [`GET /events`](/api/get-events), [`POST /register`](/api/register-queue):

View File

@ -67,15 +67,24 @@ prefix**.
{end_tabs}
For example, to configure code playgrounds for languages like Python or
JavaScript, you could specify the language and URL prefix fields as:
JavaScript, you could specify the language and URL templates as:
* `Python` and `https://replit.com/languages/python3/?code=`
* `JavaScript` and `https://replit.com/languages/javascript/?code=`
* `Python` and `https://replit.com/languages/python3/code={code}`
* `JavaScript` and `https://replit.com/languages/javascript/code={code}`
When a code block is labeled as Python or JavaScript (either explicitly or by
organization default), users would get a on-hover option to open the code block
in the specified code playground.
!!! tip ""
Code playgrounds use [RFC 6570](https://www.rfc-editor.org/rfc/rfc6570.html)
compliant URL templates to describe how links should be generated. Zulip's
rendering engine will pass the URL-encoded code from the code block as the
`code` parameter, denoted as `{code}` in this URL template, in order to
generate the URL. You can refer to parts of the documentation on URL
templates from [adding a custom linkifier](/help/add-a-custom-linkifier).
### Technical details
* You can configure multiple playgrounds for a given language; if you do that,
@ -87,8 +96,9 @@ to these human-readable Pygments names; e.g., `py3` and `py` are mapped to
`Python`. One can use the typeahead (which appears when you type something
or just click on the language field) to look up the Pygments name.
* The links for opening code playgrounds are always constructed by concatenating
the provided URL prefix with the URL-encoded contents of the code block.
* The links for opening code playgrounds are always constructed by substituting
the URL-encoded contents of the code block into `code` variable in the URL template.
The URL template is required to contain exactly one variable named `code`.
* Code playground sites do not always clearly document their URL format; often
you can just get the prefix from your browser's URL bar.

View File

@ -539,7 +539,7 @@ html_rules: List["Rule"] = [
},
"exclude": {
"templates/analytics/support.html",
# We have URL prefix and Pygments language name as placeholders
# We have URL template and Pygments language name as placeholders
# in the below template which we don't want to be translatable.
"web/templates/settings/playground_settings_admin.hbs",
},
@ -553,6 +553,8 @@ html_rules: List["Rule"] = [
"description": "Likely missing quoting in HTML attribute",
"good_lines": ['<a href="{{variable}}">'],
"bad_lines": ["<a href={{variable}}>"],
# Exclude the use of URL templates from this check.
"exclude_pattern": "={code}",
},
{
"pattern": " '}}",

View File

@ -33,7 +33,7 @@ DESKTOP_WARNING_VERSION = "5.9.3"
# Changes should be accompanied by documentation explaining what the
# new level means in api_docs/changelog.md, as well as "**Changes**"
# entries in the endpoint's documentation in `zulip.yaml`.
API_FEATURE_LEVEL = 195
API_FEATURE_LEVEL = 196
# Bump the minor PROVISION_VERSION to indicate that folks should provision
# only when going from an old version of the code to a newer version. Bump

View File

@ -7,7 +7,7 @@ import * as common from "./lib/common";
type Playground = {
playground_name: string;
pygments_language: string;
url_prefix: string;
url_template: string;
};
async function _add_playground_and_return_status(page: Page, payload: Playground): Promise<string> {
@ -35,7 +35,7 @@ async function test_successful_playground_creation(page: Page): Promise<void> {
const payload = {
pygments_language: "Python",
playground_name: "Python3 playground",
url_prefix: "https://python.example.com",
url_template: "https://python.example.com?code={code}",
};
const status = await _add_playground_and_return_status(page, payload);
assert.strictEqual(status, "Custom playground added!");
@ -52,8 +52,8 @@ async function test_successful_playground_creation(page: Page): Promise<void> {
"Python3 playground",
);
assert.strictEqual(
await common.get_text_from_selector(page, ".playground_row span.playground_url_prefix"),
"https://python.example.com",
await common.get_text_from_selector(page, ".playground_row span.playground_url_template"),
"https://python.example.com?code={code}",
);
}
@ -61,12 +61,12 @@ async function test_invalid_playground_parameters(page: Page): Promise<void> {
const payload = {
pygments_language: "Python",
playground_name: "Python3 playground",
url_prefix: "not_a_url",
url_template: "not_a_url_template{",
};
let status = await _add_playground_and_return_status(page, payload);
assert.strictEqual(status, "Failed: url_prefix is not a URL");
assert.strictEqual(status, "Failed: Invalid URL template.");
payload.url_prefix = "https://python.example.com";
payload.url_template = "https://python.example.com?code={code}";
payload.pygments_language = "py!@%&";
status = await _add_playground_and_return_status(page, payload);
assert.strictEqual(status, "Failed: Invalid characters in pygments language");

View File

@ -2,6 +2,7 @@ import ClipboardJS from "clipboard";
import {parseISO} from "date-fns";
import $ from "jquery";
import tippy, {hideAll} from "tippy.js";
import url_template_lib from "url-template";
import render_no_arrow_popover from "../templates/no_arrow_popover.hbs";
import render_playground_links_popover_content from "../templates/playground_links_popover_content.hbs";
@ -783,20 +784,21 @@ export function register_click_handlers() {
const playground_info = realm_playground.get_playground_info_for_languages(
$codehilite_div.data("code-language"),
);
// We do the code extraction here and set the target href combining the url_prefix
// and the extracted code. Depending on whether the language has multiple playground
// links configured, a popover is show.
// We do the code extraction here and set the target href expanding
// the url_template with the extracted code. Depending on whether
// the language has multiple playground links configured, a popover
// is shown.
const extracted_code = $codehilite_div.find("code").text();
if (playground_info.length === 1) {
const url_prefix = playground_info[0].url_prefix;
const url_template = url_template_lib.parse(playground_info[0].url_template);
$view_in_playground_button.attr(
"href",
url_prefix + encodeURIComponent(extracted_code),
url_template.expand({code: extracted_code}),
);
} else {
for (const $playground of playground_info) {
$playground.playground_url =
$playground.url_prefix + encodeURIComponent(extracted_code);
const url_template = url_template_lib.parse($playground.url_template);
$playground.playground_url = url_template.expand({code: extracted_code});
}
toggle_playground_link_popover(this, playground_info);
}

View File

@ -23,7 +23,7 @@ export function update_playgrounds(playgrounds_data: RealmPlayground[]): void {
const element_to_push: Omit<RealmPlayground, "pygments_language"> = {
id: data.id,
name: data.name,
url_prefix: data.url_prefix,
url_template: data.url_template,
};
if (map_language_to_playground_info.has(data.pygments_language)) {
map_language_to_playground_info.get(data.pygments_language)!.push(element_to_push);

View File

@ -41,7 +41,7 @@ export function populate_playgrounds(playgrounds_data) {
playground: {
playground_name: playground.name,
pygments_language: playground.pygments_language,
url_prefix: playground.url_prefix,
url_template: playground.url_template,
id: playground.id,
},
can_modify: page_params.is_admin,
@ -65,7 +65,7 @@ export function populate_playgrounds(playgrounds_data) {
...ListWidget.generic_sort_functions("alphabetic", [
"pygments_language",
"name",
"url_prefix",
"url_template",
]),
},
$simplebar_container: $("#playground-settings .progressive-table-wrapper"),
@ -110,7 +110,7 @@ function build_page() {
const data = {
name: $("#playground_name").val(),
pygments_language: $("#playground_pygments_language").val(),
url_prefix: $("#playground_url_prefix").val(),
url_template: $("#playground_url_template").val(),
};
channel.post({
url: "/json/realm/playgrounds",
@ -118,7 +118,7 @@ function build_page() {
success() {
$("#playground_pygments_language").val("");
$("#playground_name").val("");
$("#playground_url_prefix").val("");
$("#playground_url_template").val("");
$add_playground_button.prop("disabled", false);
ui_report.success(
$t_html({defaultMessage: "Custom playground added!"}),

View File

@ -250,7 +250,7 @@ h3,
#playground-settings {
#playground_pygments_language,
#playground_name,
#playground_url_prefix {
#playground_url_template {
width: calc(100% - 10em - 6em);
}
}

View File

@ -7,7 +7,7 @@
<span class="playground_name">{{playground_name}}</span>
</td>
<td>
<span class="playground_url_prefix">{{url_prefix}}</span>
<span class="playground_url_template">{{url_template}}</span>
</td>
{{#if ../can_modify}}
<td class="no-select actions">

View File

@ -23,7 +23,7 @@
{{t "Name" }}: <span class="rendered_markdown"><code>Python3 playground</code></span>
</li>
<li>
{{t "URL prefix" }}: <span class="rendered_markdown"><code>https://replit.com/languages/python3/?code=</code></span>
{{t "URL template" }}: <span class="rendered_markdown"><code>https://replit.com/languages/python3/?code={code}</code></span>
</li>
</ul>
<p>
@ -52,8 +52,8 @@
<input type="text" id="playground_name" class="settings_text_input" name="playground_name" autocomplete="off" placeholder="Python3 playground" />
</div>
<div class="input-group">
<label for="playground_url_prefix"> {{t "URL prefix" }}</label>
<input type="text" id="playground_url_prefix" class="settings_text_input" name="url_prefix" placeholder="https://replit.com/languages/python3/?code=" />
<label for="playground_url_template"> {{t "URL template" }}</label>
<input type="text" id="playground_url_template" class="settings_text_input" name="url_template" placeholder="https://replit.com/languages/python3/?code={code}" />
</div>
<button type="submit" id="submit_playground_button" class="button rounded sea-green">
{{t 'Add code playground' }}
@ -73,7 +73,7 @@
<thead class="table-sticky-headers">
<th class="active" data-sort="alphabetic" data-sort-prop="pygments_language">{{t "Language" }}</th>
<th data-sort="alphabetic" data-sort-prop="name">{{t "Name" }}</th>
<th data-sort="alphabetic" data-sort-prop="url_prefix">{{t "URL prefix" }}</th>
<th data-sort="alphabetic" data-sort-prop="url_template">{{t "URL template" }}</th>
{{#if is_admin}}
<th class="actions">{{t "Actions" }}</th>
{{/if}}

View File

@ -518,7 +518,7 @@ exports.fixtures = {
id: 1,
name: "Lean playground",
pygments_language: "Lean",
url_prefix: "https://leanprover.github.io/live/latest/#code=",
url_template: "https://leanprover.github.io/live/latest/{#code}",
},
],
},

View File

@ -25,7 +25,7 @@ run_test("get_pygments_typeahead_list_for_composebox", () => {
id: 2,
name: "Custom Lang",
pygments_language: custom_pygment_language,
url_prefix: "https://example.com/?q=",
url_template: "https://example.com/?q={code}",
},
];
realm_playground.initialize({
@ -52,19 +52,19 @@ run_test("get_pygments_typeahead_list_for_settings", () => {
id: 1,
name: "Custom Lang #1",
pygments_language: custom_pygment_language,
url_prefix: "https://example.com/?q=",
url_template: "https://example.com/?q={code}",
},
{
id: 2,
name: "Custom Lang #2",
pygments_language: custom_pygment_language,
url_prefix: "https://example.com/?q=",
url_template: "https://example.com/?q={code}",
},
{
id: 3,
name: "Invent a Language",
pygments_language: "invent_a_lang",
url_prefix: "https://example.com/?q=",
url_template: "https://example.com/?q={code}",
},
];
realm_playground.initialize({

View File

@ -28,18 +28,14 @@ def do_add_realm_playground(
acting_user: Optional[UserProfile],
name: str,
pygments_language: str,
url_prefix: str,
url_template: str,
) -> int:
realm_playground = RealmPlayground(
realm=realm,
name=name,
pygments_language=pygments_language,
url_prefix=url_prefix,
url_template=url_prefix + "{code}",
url_template=url_template,
)
# We expect full_clean to always pass since a thorough input validation
# is performed in the view (using check_url, check_pygments_language, etc)
# before calling this function.
realm_playground.full_clean()
realm_playground.save()
realm_playgrounds = get_realm_playgrounds(realm)
@ -55,7 +51,7 @@ def do_add_realm_playground(
id=realm_playground.id,
name=realm_playground.name,
pygments_language=realm_playground.pygments_language,
url_prefix=realm_playground.url_prefix,
url_template=realm_playground.url_template,
),
}
).decode(),
@ -71,7 +67,6 @@ def do_remove_realm_playground(
removed_playground = {
"name": realm_playground.name,
"pygments_language": realm_playground.pygments_language,
"url_prefix": realm_playground.url_prefix,
"url_template": realm_playground.url_template,
}

View File

@ -702,7 +702,7 @@ realm_domains_remove_event = event_dict_type(
check_realm_domains_remove = make_checker(realm_domains_remove_event)
realm_playground_type = DictType(
required_keys=[("id", int), ("name", str), ("pygments_language", str), ("url_prefix", str)]
required_keys=[("id", int), ("name", str), ("pygments_language", str), ("url_template", str)]
)
realm_playgrounds_event = event_dict_type(

View File

@ -271,7 +271,7 @@ class RealmPlaygroundDict(TypedDict):
id: int
name: str
pygments_language: str
url_prefix: str
url_template: str
@dataclass

View File

@ -0,0 +1,23 @@
# Generated by Django 4.2.1 on 2023-05-27 03:13
from django.db import migrations, models
from zerver.models import url_template_validator
class Migration(migrations.Migration):
dependencies = [
("zerver", "0463_backfill_realmplayground_url_template"),
]
operations = [
migrations.RemoveField(
model_name="realmplayground",
name="url_prefix",
),
migrations.AlterField(
model_name="realmplayground",
name="url_template",
field=models.TextField(validators=[url_template_validator], null=False),
),
]

View File

@ -42,7 +42,7 @@ from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import SearchVectorField
from django.core.exceptions import ValidationError
from django.core.serializers.json import DjangoJSONEncoder
from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email
from django.core.validators import MinLengthValidator, RegexValidator, validate_email
from django.db import models, transaction
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.models import CASCADE, Exists, F, OuterRef, Q, QuerySet, Sum
@ -1365,8 +1365,7 @@ class RealmPlayground(models.Model):
MAX_PYGMENTS_LANGUAGE_LENGTH = 40
realm = models.ForeignKey(Realm, on_delete=CASCADE)
url_prefix = models.TextField(validators=[URLValidator()])
url_template = models.TextField(validators=[url_template_validator], null=True)
url_template = models.TextField(validators=[url_template_validator])
# User-visible display name used when configuring playgrounds in the settings page and
# when displaying them in the playground links popover.
@ -1399,22 +1398,17 @@ class RealmPlayground(models.Model):
and stores all ValidationErrors from all stages to return as JSON.
"""
# Prior to the completion of this migration, we make url_template nullable,
# while ensuring that no code path will create a RealmPlayground without populating it.
assert self.url_template is not None
# Do not continue the check if the url template is invalid to begin with.
# The ValidationError for invalid template will only be raised by the validator
# set on the url_template field instead of here to avoid duplicates.
# Do not continue the check if the url template is invalid to begin
# with. The ValidationError for invalid template will only be raised by
# the validator set on the url_template field instead of here to avoid
# duplicates.
if not uri_template.validate(self.url_template):
return
# Extract variables used in the URL template.
template_variables = set(uri_template.URITemplate(self.url_template).variable_names)
if (
"code" not in template_variables
): # nocoverage: prior to the completion of the migration, it is impossible to generate a URL template without the "code" variable
if "code" not in template_variables:
raise ValidationError(_('Missing the required variable "code" in the URL template'))
# The URL template should only contain a single variable, which is "code".
@ -1432,7 +1426,7 @@ def get_realm_playgrounds(realm: Realm) -> List[RealmPlaygroundDict]:
id=playground.id,
name=playground.name,
pygments_language=playground.pygments_language,
url_prefix=playground.url_prefix,
url_template=playground.url_template,
)
)
return playgrounds

View File

@ -296,7 +296,7 @@ def add_realm_playground() -> Dict[str, object]:
return {
"name": "Python2 playground",
"pygments_language": "Python2",
"url_prefix": "https://python2.example.com",
"url_template": "https://python2.example.com?code={code}",
}
@ -307,7 +307,7 @@ def remove_realm_playground() -> Dict[str, object]:
acting_user=None,
name="Python playground",
pygments_language="Python",
url_prefix="https://python.example.com",
url_template="https://python.example.com?code={code}",
)
return {
"playground_id": playground_id,

View File

@ -481,7 +481,7 @@ def add_realm_playground(client: Client) -> None:
request = {
"name": "Python playground",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com?code={code}",
}
result = client.call_endpoint(url="/realm/playgrounds", method="POST", request=request)
# {code_example|end}

View File

@ -3305,7 +3305,7 @@ paths:
"id": 1,
"name": "Python playground",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com",
},
],
"id": 0,
@ -10716,13 +10716,20 @@ paths:
type: string
example: Python
required: true
- name: url_prefix
- name: url_template
in: query
description: |
The url prefix for the playground.
The [RFC 6570](https://www.rfc-editor.org/rfc/rfc6570.html)
compliant URL template for the playground. The template should
contain exactly one variable named `code`, which determines how the
extracted code should be substituted in the playground URL.
**Changes**: New in Zulip 8.0 (feature level 196). This replaced the
`url_prefix` parameter, which was used to construct URLs by just
concatenating `url_prefix` and `code`.
schema:
type: string
example: https://python.example.com
example: https://python.example.com?code={code}
required: true
responses:
"200":
@ -17408,10 +17415,17 @@ components:
description: |
The name of the Pygments language lexer for that
programming language.
url_prefix:
url_template:
type: string
description: |
The url prefix for the playground.
The [RFC 6570](https://www.rfc-editor.org/rfc/rfc6570.html)
compliant URL template for the playground. The template contains
exactly one variable named `code`, which determines how the
extracted code should be substituted in the playground URL.
**Changes**: New in Zulip 8.0 (feature level 196). This replaced the
`url_prefix` parameter, which was used to construct URLs by just
concatenating url_prefix and code.
RealmExport:
type: object
additionalProperties: false

View File

@ -866,13 +866,13 @@ class TestRealmAuditLog(ZulipTestCase):
acting_user=user,
name="Python playground",
pygments_language="Python",
url_prefix="https://python.example.com",
url_template="https://python.example.com{code}",
)
added_playground = RealmPlaygroundDict(
id=playground_id,
name="Python playground",
pygments_language="Python",
url_prefix="https://python.example.com",
url_template="https://python.example.com{code}",
)
expected_extra_data = {
"realm_playgrounds": [*initial_playgrounds, added_playground],
@ -899,7 +899,6 @@ class TestRealmAuditLog(ZulipTestCase):
removed_playground = {
"name": "Python playground",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com{code}",
}
expected_extra_data = {

View File

@ -2228,7 +2228,7 @@ class NormalActionsTest(BaseAction):
acting_user=None,
name="Python playground",
pygments_language="Python",
url_prefix="https://python.example.com",
url_template="https://python.example.com{code}",
)
)
check_realm_playgrounds("events[0]", events[0])

View File

@ -10,7 +10,7 @@ class RealmPlaygroundTests(ZulipTestCase):
payload = {
"name": "Python playground",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com{code}",
}
# Now send a POST request to the API endpoint.
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
@ -29,12 +29,12 @@ class RealmPlaygroundTests(ZulipTestCase):
{
"name": "Python playground 1",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com{code}",
},
{
"name": "Python playground 2",
"pygments_language": "Python",
"url_prefix": "https://python2.example.com",
"url_template": "https://python2.example.com{code}",
},
]
for payload in data:
@ -53,22 +53,17 @@ class RealmPlaygroundTests(ZulipTestCase):
iago = self.example_user("iago")
payload = {
"name": "Invalid URL",
"pygments_language": "Python",
"url_prefix": "https://invalid-url",
"name": "Invalid characters in pygments language",
"pygments_language": "a$b$c",
"url_template": "https://template.com{code}",
}
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_error(resp, "url_prefix is not a URL")
payload["url_prefix"] = "https://python.example.com"
payload["pygments_language"] = "a$b$c"
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_error(resp, "Invalid characters in pygments language")
payload = {
"name": "Template with an unexpected variable",
"pygments_language": "Python",
"url_prefix": "https://template.com?test={test}",
"url_template": "https://template.com{?test,code}",
}
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_error(
@ -78,18 +73,26 @@ class RealmPlaygroundTests(ZulipTestCase):
payload = {
"name": "Invalid URL template",
"pygments_language": "Python",
"url_prefix": "https://template.com?test={test",
"url_template": "https://template.com?test={test",
}
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_error(resp, "Invalid URL template.")
payload = {
"name": "Template without the required variable",
"pygments_language": "Python",
"url_template": "https://template.com{?test}",
}
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_error(resp, 'Missing the required variable "code" in the URL template')
def test_create_already_existing_playground(self) -> None:
iago = self.example_user("iago")
payload = {
"name": "Python playground",
"pygments_language": "Python",
"url_prefix": "https://python.example.com",
"url_template": "https://python.example.com{code}",
}
resp = self.api_post(iago, "/api/v1/realm/playgrounds", payload)
self.assert_json_success(resp)
@ -117,7 +120,7 @@ class RealmPlaygroundTests(ZulipTestCase):
acting_user=iago,
name="Python playground",
pygments_language="Python",
url_prefix="https://python.example.com",
url_template="https://python.example.com{code}",
)
self.assertTrue(RealmPlayground.objects.filter(name="Python playground").exists())

View File

@ -9,7 +9,7 @@ from zerver.decorator import require_realm_admin
from zerver.lib.exceptions import JsonableError, ValidationFailureError
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_success
from zerver.lib.validator import check_capped_string, check_url
from zerver.lib.validator import check_capped_string
from zerver.models import Realm, RealmPlayground, UserProfile
@ -40,7 +40,7 @@ def add_realm_playground(
request: HttpRequest,
user_profile: UserProfile,
name: str = REQ(),
url_prefix: str = REQ(str_validator=check_url),
url_template: str = REQ(),
pygments_language: str = REQ(str_validator=check_pygments_language),
) -> HttpResponse:
try:
@ -49,7 +49,7 @@ def add_realm_playground(
acting_user=user_profile,
name=name.strip(),
pygments_language=pygments_language.strip(),
url_prefix=url_prefix.strip(),
url_template=url_template.strip(),
)
except ValidationError as e:
raise ValidationFailureError(e)