2022-05-16 11:55:36 +02:00
|
|
|
from typing import Dict, List, Optional
|
2022-04-14 23:32:56 +02:00
|
|
|
|
2022-03-14 11:50:24 +01:00
|
|
|
import orjson
|
2022-03-14 14:58:19 +01:00
|
|
|
from django.db import transaction
|
2022-03-14 11:50:24 +01:00
|
|
|
from django.utils.timezone import now as timezone_now
|
|
|
|
|
2022-05-16 11:55:36 +02:00
|
|
|
from zerver.lib.types import LinkifierDict
|
2022-04-14 23:32:56 +02:00
|
|
|
from zerver.models import (
|
|
|
|
Realm,
|
2022-03-14 11:50:24 +01:00
|
|
|
RealmAuditLog,
|
2022-04-14 23:32:56 +02:00
|
|
|
RealmFilter,
|
2022-03-14 11:50:24 +01:00
|
|
|
UserProfile,
|
2022-04-14 23:32:56 +02:00
|
|
|
active_user_ids,
|
|
|
|
linkifiers_for_realm,
|
|
|
|
)
|
|
|
|
from zerver.tornado.django_api import send_event
|
|
|
|
|
|
|
|
|
2022-05-16 11:55:36 +02:00
|
|
|
def notify_linkifiers(realm: Realm, realm_linkifiers: List[LinkifierDict]) -> None:
|
2022-04-14 23:32:56 +02:00
|
|
|
event: Dict[str, object] = dict(type="realm_linkifiers", realm_linkifiers=realm_linkifiers)
|
2022-03-14 14:58:19 +01:00
|
|
|
transaction.on_commit(lambda: send_event(realm, event, active_user_ids(realm.id)))
|
2022-04-14 23:32:56 +02:00
|
|
|
|
|
|
|
|
|
|
|
# NOTE: Regexes must be simple enough that they can be easily translated to JavaScript
|
|
|
|
# RegExp syntax. In addition to JS-compatible syntax, the following features are available:
|
|
|
|
# * Named groups will be converted to numbered groups automatically
|
|
|
|
# * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags
|
2022-03-14 14:58:19 +01:00
|
|
|
@transaction.atomic(durable=True)
|
2022-03-14 11:50:24 +01:00
|
|
|
def do_add_linkifier(
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
realm: Realm,
|
|
|
|
pattern: str,
|
|
|
|
url_template: str,
|
|
|
|
*,
|
|
|
|
acting_user: Optional[UserProfile],
|
2022-03-14 11:50:24 +01:00
|
|
|
) -> int:
|
2022-04-14 23:32:56 +02:00
|
|
|
pattern = pattern.strip()
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template = url_template.strip()
|
|
|
|
linkifier = RealmFilter(realm=realm, pattern=pattern, url_template=url_template)
|
2022-04-14 23:32:56 +02:00
|
|
|
linkifier.full_clean()
|
|
|
|
linkifier.save()
|
2022-05-16 11:55:36 +02:00
|
|
|
|
|
|
|
realm_linkifiers = linkifiers_for_realm(realm.id)
|
2022-03-14 11:50:24 +01:00
|
|
|
RealmAuditLog.objects.create(
|
|
|
|
realm=realm,
|
|
|
|
acting_user=acting_user,
|
|
|
|
event_type=RealmAuditLog.REALM_LINKIFIER_ADDED,
|
|
|
|
event_time=timezone_now(),
|
|
|
|
extra_data=orjson.dumps(
|
|
|
|
{
|
|
|
|
"realm_linkifiers": realm_linkifiers,
|
|
|
|
"added_linkifier": LinkifierDict(
|
|
|
|
pattern=pattern,
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template=url_template,
|
2022-03-14 11:50:24 +01:00
|
|
|
id=linkifier.id,
|
|
|
|
),
|
|
|
|
}
|
|
|
|
).decode(),
|
|
|
|
)
|
2022-05-16 11:55:36 +02:00
|
|
|
notify_linkifiers(realm, realm_linkifiers)
|
2022-04-14 23:32:56 +02:00
|
|
|
|
|
|
|
return linkifier.id
|
|
|
|
|
|
|
|
|
2022-03-14 14:59:29 +01:00
|
|
|
@transaction.atomic(durable=True)
|
2022-04-14 23:32:56 +02:00
|
|
|
def do_remove_linkifier(
|
2022-03-14 14:50:55 +01:00
|
|
|
realm: Realm,
|
|
|
|
pattern: Optional[str] = None,
|
|
|
|
id: Optional[int] = None,
|
|
|
|
*,
|
|
|
|
acting_user: Optional[UserProfile] = None,
|
2022-04-14 23:32:56 +02:00
|
|
|
) -> None:
|
|
|
|
if pattern is not None:
|
2022-03-14 14:50:55 +01:00
|
|
|
realm_linkifier = RealmFilter.objects.get(realm=realm, pattern=pattern)
|
2022-04-14 23:32:56 +02:00
|
|
|
else:
|
2022-05-29 21:52:25 +02:00
|
|
|
assert id is not None
|
2022-03-14 14:50:55 +01:00
|
|
|
realm_linkifier = RealmFilter.objects.get(realm=realm, id=id)
|
|
|
|
|
|
|
|
pattern = realm_linkifier.pattern
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template = realm_linkifier.url_template
|
2022-03-14 14:50:55 +01:00
|
|
|
realm_linkifier.delete()
|
2022-05-16 11:55:36 +02:00
|
|
|
|
|
|
|
realm_linkifiers = linkifiers_for_realm(realm.id)
|
2022-03-14 14:50:55 +01:00
|
|
|
RealmAuditLog.objects.create(
|
|
|
|
realm=realm,
|
|
|
|
acting_user=acting_user,
|
|
|
|
event_type=RealmAuditLog.REALM_LINKIFIER_REMOVED,
|
|
|
|
event_time=timezone_now(),
|
|
|
|
extra_data=orjson.dumps(
|
|
|
|
{
|
|
|
|
"realm_linkifiers": realm_linkifiers,
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
"removed_linkifier": {"pattern": pattern, "url_template": url_template},
|
2022-03-14 14:50:55 +01:00
|
|
|
}
|
|
|
|
).decode(),
|
|
|
|
)
|
2022-05-16 11:55:36 +02:00
|
|
|
notify_linkifiers(realm, realm_linkifiers)
|
2022-04-14 23:32:56 +02:00
|
|
|
|
|
|
|
|
2022-03-14 14:59:01 +01:00
|
|
|
@transaction.atomic(durable=True)
|
2022-03-14 12:10:25 +01:00
|
|
|
def do_update_linkifier(
|
|
|
|
realm: Realm,
|
|
|
|
id: int,
|
|
|
|
pattern: str,
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template: str,
|
2022-03-14 12:10:25 +01:00
|
|
|
*,
|
|
|
|
acting_user: Optional[UserProfile],
|
|
|
|
) -> None:
|
2022-04-14 23:32:56 +02:00
|
|
|
pattern = pattern.strip()
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template = url_template.strip()
|
2022-04-14 23:32:56 +02:00
|
|
|
linkifier = RealmFilter.objects.get(realm=realm, id=id)
|
|
|
|
linkifier.pattern = pattern
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
linkifier.url_template = url_template
|
2022-04-14 23:32:56 +02:00
|
|
|
linkifier.full_clean()
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
linkifier.save(update_fields=["pattern", "url_template"])
|
2022-05-16 11:55:36 +02:00
|
|
|
|
|
|
|
realm_linkifiers = linkifiers_for_realm(realm.id)
|
2022-03-14 12:10:25 +01:00
|
|
|
RealmAuditLog.objects.create(
|
|
|
|
realm=realm,
|
|
|
|
acting_user=acting_user,
|
|
|
|
event_type=RealmAuditLog.REALM_LINKIFIER_CHANGED,
|
|
|
|
event_time=timezone_now(),
|
|
|
|
extra_data=orjson.dumps(
|
|
|
|
{
|
|
|
|
"realm_linkifiers": realm_linkifiers,
|
|
|
|
"changed_linkifier": LinkifierDict(
|
|
|
|
pattern=pattern,
|
linkifier: Support URL templates for linkifiers.
This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits will be squashed with this commit.
We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.
The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.
This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.
Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.
With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.
We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.
Fixes #23124.
Signed-off-by: Zixuan James Li <p359101898@gmail.com>
2022-10-05 20:55:31 +02:00
|
|
|
url_template=url_template,
|
2022-03-14 12:10:25 +01:00
|
|
|
id=linkifier.id,
|
|
|
|
),
|
|
|
|
}
|
|
|
|
).decode(),
|
|
|
|
)
|
|
|
|
|
2022-05-16 11:55:36 +02:00
|
|
|
notify_linkifiers(realm, realm_linkifiers)
|