linkifiers: Add url_template field.

This will later be used to expand matching linkifier patterns.
Making it nullable for now, but we will make it required in
the APIs.

As a part of this transition, we temporarily make url_format_string
nullable as well, which will be later removed. This allows us to
switch to populating url_template without caring about passing
url_format_string.

Note that the validators are imported in the migration because Django
otherwise diffs it and considers the schema to be different, generating
a migration, failing the "tools/test-migrations" test.

Signed-off-by: Zixuan James Li <p359101898@gmail.com>

tools/semgrep.yml

@@ -22,6 +22,7 @@ rules:
       - pattern-not: from zerver.lib.redis_utils import get_redis_client
       - pattern-not: from zerver.models import filter_pattern_validator
       - pattern-not: from zerver.models import filter_format_validator
+      - pattern-not: from zerver.models import url_template_validator
       - pattern-not: from zerver.models import generate_email_token_for_stream
       - pattern-either:
           - pattern: from zerver import $X

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 175
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (233, 1)
+PROVISION_VERSION = (233, 2)

zerver/migrations/0440_realmfilter_url_template.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.0.7 on 2022-10-02 20:50
+
+from django.db import migrations, models
+
+from zerver.models import filter_format_validator, url_template_validator
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0439_fix_deleteduser_email"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="realmfilter",
+            name="url_template",
+            field=models.TextField(validators=[url_template_validator], null=True),
+        ),
+        migrations.AlterField(
+            model_name="realmfilter",
+            name="url_format_string",
+            field=models.TextField(validators=[filter_format_validator], null=True, blank=True),
+        ),
+    ]

zerver/models.py

@@ -26,6 +26,7 @@ from uuid import uuid4
 import django.contrib.auth
 import orjson
 import re2
+import uri_template
 from bitfield import BitField
 from bitfield.types import Bit, BitHandler
 from django.conf import settings
@@ -1298,6 +1299,15 @@ def filter_format_validator(value: str) -> None:
         raise ValidationError(_("Invalid format string in URL."))
 
 
+def url_template_validator(value: str) -> None:
+    """Verifies URL-ness, and then validates as a URL template"""
+    # URLValidator is assumed to catch anything which is malformed as a URL
+    URLValidator()(value)
+
+    if not uri_template.validate(value):
+        raise ValidationError(_("Invalid URL template."))
+
+
 class RealmFilter(models.Model):
     """Realm-specific regular expressions to automatically linkify certain
     strings inside the Markdown processor.  See "Custom filters" in the settings UI.
@@ -1305,7 +1315,10 @@ class RealmFilter(models.Model):
 
     realm = models.ForeignKey(Realm, on_delete=CASCADE)
     pattern = models.TextField()
-    url_format_string = models.TextField(validators=[filter_format_validator])
+    url_format_string = models.TextField(
+        validators=[filter_format_validator], null=True, blank=True
+    )
+    url_template = models.TextField(validators=[url_template_validator], null=True)
 
     class Meta:
         unique_together = ("realm", "pattern")