Add initial implementation of custom realm filters.

This PR was abandoned by Vladislav and then substantially modified by
Igor Tokarev and Tim Abbott to complete it and fix a number of bugs.

Fixes #544.

docs/prod-maintain-secure-upgrade.md

@@ -613,11 +613,5 @@ server, and suggested procedure.
 ### Other useful manage.py commands
 
 There are a large number of useful management commands under
-`zerver/manangement/commands/`; you can also see them listed using
+`zerver/management/commands/`; you can also see them listed using
 `./manage.py` with no arguments.
-
-One such command worth highlighting because it's a valuable feature
-with no UI in the Administration page is `./manage.py realm_filters`,
-which allows you to configure certain patterns in messages to be
-automatically linkified, e.g., whenever someone mentions "T1234", it
-could be auto-linkified to ticket 1234 in your team's Trac instance.

frontend_tests/casper_tests/10-admin.js

@@ -177,6 +177,41 @@ casper.then(function () {
     });
 });
 
+// Test custom realm filters
+casper.waitForSelector('.admin-filter-form', function () {
+    casper.fill('form.admin-filter-form', {
+        'pattern': '#(?P<id>[0-9]+)',
+        'url_format_string': 'https://trac.example.com/ticket/%(id)s'
+    });
+    casper.click('form.admin-filter-form input.btn');
+});
+
+casper.waitUntilVisible('div#admin-filter-status', function () {
+    casper.test.assertSelectorHasText('div#admin-filter-status', 'Custom filter added!');
+});
+
+casper.waitForSelector('.filter_row', function () {
+    casper.test.assertSelectorHasText('.filter_row span.filter_pattern', '#(?P<id>[0-9]+)');
+    casper.test.assertSelectorHasText('.filter_row span.filter_url_format_string', 'https://trac.example.com/ticket/%(id)s');
+    casper.click('.filter_row button');
+});
+
+casper.waitWhileSelector('.filter_row', function () {
+    casper.test.assertDoesntExist('.filter_row');
+});
+
+casper.waitForSelector('.admin-filter-form', function () {
+    casper.fill('form.admin-filter-form', {
+        'pattern': 'a$',
+        'url_format_string': 'https://trac.example.com/ticket/%(id)s'
+    });
+    casper.click('form.admin-filter-form input.btn');
+});
+
+casper.waitUntilVisible('div#admin-filter-pattern-status', function () {
+    casper.test.assertSelectorHasText('div#admin-filter-pattern-status', 'Failed: Invalid filter pattern, you must use the following format PREFIX-(?P<id>.+)');
+});
+
 function get_suggestions(str) {
     casper.then(function () {
         casper.evaluate(function (str) {

frontend_tests/node_tests/dispatch.js

@@ -569,6 +569,7 @@ run(function (override, capture, args) {
     // realm_filters
     var event = event_fixtures.realm_filters;
     page_params.realm_filters = [];
+    override('admin', 'populate_filters', noop);
     dispatch(event);
     assert_same(page_params.realm_filters, event.realm_filters);
 

frontend_tests/node_tests/templates.js

@@ -129,6 +129,28 @@ function render(template_name, args) {
     assert.equal(emoji_url.attr('src'), 'http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png');
 }());
 
+(function admin_filter_list() {
+    var args = {
+        filter: {
+            "pattern": "#(?P<id>[0-9]+)",
+            "url_format_string": "https://trac.example.com/ticket/%(id)s"
+        }
+    };
+
+    var html = '';
+    html += '<tbody id="admin_filters_table">';
+    html += render('admin_filter_list', args);
+    html += '</tbody>';
+
+    global.write_test_output('admin_filter_list', html);
+
+    var filter_pattern = $(html).find('tr.filter_row:first span.filter_pattern');
+    var filter_format = $(html).find('tr.filter_row:first span.filter_url_format_string');
+
+    assert.equal(filter_pattern.text(), '#(?P<id>[0-9]+)');
+    assert.equal(filter_format.text(), 'https://trac.example.com/ticket/%(id)s');
+}());
+
 (function admin_streams_list() {
     var html = '<table>';
     var streams = ['devel', 'trac', 'zulip'];

static/js/admin.js

@@ -186,6 +186,25 @@ exports.populate_emoji = function (emoji_data) {
     loading.destroy_indicator($('#admin_page_emoji_loading_indicator'));
 };
 
+exports.populate_filters = function (filters_data) {
+    var filters_table = $("#admin_filters_table").expectOne();
+    filters_table.find("tr.filter_row").remove();
+    _.each(filters_data, function (filter) {
+        filters_table.append(
+            templates.render(
+                "admin_filter_list", {
+                    filter: {
+                        pattern: filter[0],
+                        url_format_string: filter[1],
+                        id: filter[2]
+                    }
+                }
+            )
+        );
+    });
+    loading.destroy_indicator($('#admin_page_filters_loading_indicator'));
+};
+
 exports.reset_realm_default_language = function () {
     $("#id_realm_default_language").val(page_params.realm_default_language);
 };
@@ -275,6 +294,9 @@ function _setup_page() {
     $("#admin-realm-message-editing-status").expectOne().hide();
     $("#admin-realm-default-language-status").expectOne().hide();
     $("#admin-emoji-status").expectOne().hide();
+    $('#admin-filter-status').expectOne().hide();
+    $('#admin-filter-pattern-status').expectOne().hide();
+    $('#admin-filter-format-status').expectOne().hide();
 
     $("#id_realm_default_language").val(page_params.realm_default_language);
 
@@ -285,6 +307,7 @@ function _setup_page() {
     loading.make_indicator($('#admin_page_deactivated_users_loading_indicator'));
     loading.make_indicator($('#admin_page_emoji_loading_indicator'));
     loading.make_indicator($('#admin_page_auth_methods_loading_indicator'));
+    loading.make_indicator($('#admin_page_filters_loading_indicator'));
 
     // Populate users and bots tables
     channel.get({
@@ -311,6 +334,9 @@ function _setup_page() {
     exports.populate_emoji(page_params.realm_emoji);
     exports.update_default_streams_table();
 
+    // Populate filters table
+    exports.populate_filters(page_params.realm_filters);
+
     // Setup click handlers
     $(".admin_user_table").on("click", ".deactivate", function (e) {
         e.preventDefault();
@@ -780,6 +806,66 @@ function _setup_page() {
         });
     });
 
+    $('.admin_filters_table').on('click', '.delete', function (e) {
+        e.preventDefault();
+        e.stopPropagation();
+        var btn = $(this);
+
+        channel.del({
+            url: '/json/realm/filters/' + encodeURIComponent(btn.attr('data-filter-id')),
+            error: function (xhr, error_type) {
+                if (xhr.status.toString().charAt(0) === "4") {
+                    btn.closest("td").html(
+                        $("<p>").addClass("text-error").text($.parseJSON(xhr.responseText).msg)
+                    );
+                } else {
+                     btn.text("Failed!");
+                }
+            },
+            success: function () {
+                var row = btn.parents('tr');
+                row.remove();
+            }
+        });
+    });
+
+    $(".administration").on("submit", "form.admin-filter-form", function (e) {
+        e.preventDefault();
+        e.stopPropagation();
+        var filter_status = $('#admin-filter-status');
+        var pattern_status = $('#admin-filter-pattern-status');
+        var format_status = $('#admin-filter-format-status');
+        filter_status.hide();
+        pattern_status.hide();
+        format_status.hide();
+        var filter = {};
+        $(this).serializeArray().map(function (x){filter[x.name] = x.value;});
+
+        channel.post({
+            url: "/json/realm/filters",
+            data: $(this).serialize(),
+            success: function (data) {
+                filter.id = data.id;
+                ui.report_success(i18n.t("Custom filter added!"), filter_status);
+            },
+            error: function (xhr, error) {
+                var errors = $.parseJSON(xhr.responseText).errors;
+                if (errors.pattern !== undefined) {
+                    xhr.responseText = JSON.stringify({msg: errors.pattern});
+                    ui.report_error(i18n.t("Failed"), xhr, pattern_status);
+                }
+                if (errors.url_format_string !== undefined) {
+                    xhr.responseText = JSON.stringify({msg: errors.url_format_string});
+                    ui.report_error(i18n.t("Failed"), xhr, format_status);
+                }
+                if (errors.__all__ !== undefined) {
+                    xhr.responseText = JSON.stringify({msg: errors.__all__});
+                    ui.report_error(i18n.t("Failed"), xhr, filter_status);
+                }
+            }
+        });
+    });
+
 }
 
 exports.setup_page = function () {

static/js/server_events.js

@@ -92,6 +92,7 @@ function dispatch_normal_event(event) {
     case 'realm_filters':
         page_params.realm_filters = event.realm_filters;
         echo.set_realm_filters(page_params.realm_filters);
+        admin.populate_filters(page_params.realm_filters);
         break;
 
     case 'realm_user':

static/styles/settings.css

@@ -193,6 +193,7 @@ input[type=checkbox].inline-block {
     #settings .settings-section .new-bot-form,
     #settings .settings-section .new-alert-word-form,
     #emoji-settings .new-emoji-form,
+    #filter-settings .new-filter-form,
     #settings .settings-section .notification-settings-form,
     #settings .settings-section .display-settings-form,
     #settings .settings-section .edit-bot-form-box {
@@ -204,6 +205,7 @@ input[type=checkbox].inline-block {
     #settings .settings-section .new-bot-form .control-label,
     #settings .settings-section .new-alert-word-form .control-label,
     #emoji-settings .new-emoji-form .control-label,
+    #filter-settings .new-filter-form .control-label,
     #settings .settings-section .edit-bot-form-box .control-label {
         display: block;
         width: 120px;
@@ -219,6 +221,7 @@ input[type=checkbox].inline-block {
     #settings .settings-section .new-bot-form .controls,
     #settings .settings-section .new-alert-word-form button,
     #emoji-settings .new-emoji-form .controls,
+    #filter-settings .new-filter-form .controls,
     #settings .settings-section .edit-bot-form-box .controls {
         margin: auto;
         text-align: center;
@@ -271,6 +274,14 @@ input[type=checkbox].inline-block {
     margin: 0 auto;
 }
 
+.admin_filters_table {
+    margin-top: 20px;
+}
+
+#admin-filter-pattern-status, #admin-filter-format-status {
+    margin: 20px 0 0 0;
+}
+
 #bots_list {
     display: none;
     list-style-type: none;
@@ -349,6 +360,7 @@ input[type=checkbox].inline-block {
 #create_bot_form .control-label,
 #create_alert_word_form .control-label,
 .admin-emoji-form .control-label,
+.admin-filter-form .control-label,
 .default-stream-form .control-label {
     width: 10em;
     text-align: right;

static/templates/admin_filter_list.handlebars

@@ -0,0 +1,15 @@
+{{#with filter}}
+<tr class="filter_row">
+  <td>
+      <span class="filter_pattern">{{pattern}}</span>
+  </td>
+  <td>
+      <span class="filter_url_format_string">{{url_format_string}}</span>
+  </td>
+  <td>
+      <button class="btn delete btn-danger" data-filter-id="{{id}}">
+        {{t "Delete" }}
+      </button>
+  </td>
+</tr>
+{{/with}}

static/templates/admin_tab.handlebars

@@ -27,6 +27,7 @@
         <div role="tabpanel" class="tab-pane active" id="organization">
           {{ partial "organization-settings-admin" }}
           {{ partial "emoji-settings-admin" }}
+          {{ partial "realm-filter-settings-admin" }}
           {{ partial "auth-methods-settings-admin" }}
         </div>
         <div role="tabpanel" class="tab-pane" id="users">

static/templates/settings/realm-filter-settings-admin.handlebars

@@ -0,0 +1,36 @@
+<div id="filter-settings" class="settings-section">
+  <div class="settings-section-title"><i class="icon-vector-filter settings-section-icon"></i>{{t "Custom linkification filters" }}</div>
+  <div class="admin-table-wrapper">
+    <p>{{#tr this}}Configure patterns that will be automatically linkified when used in Zulip message bodies or topics.  For example, you could make typing "#123" automatically become a link to issue #123 in an issue tracker.{{/tr}}</p>
+    <table class="table table-condensed table-striped admin_filters_table">
+      <tbody id="admin_filters_table">
+        <th>{{t "Pattern" }}</th>
+        <th>{{t "URL format string" }}</th>
+        <th>{{t "Actions" }}</th>
+      </tbody>
+    </table>
+  </div>
+  <form class="form-horizontal admin-filter-form">
+    <div class="add-new-filter-box grey-bg green-bg">
+      <div class="new-filter-form">
+        <div class="settings-section-title new-filter-section-title">{{t "Add a New Filter" }}</div>
+        <div class="alert" id="admin-filter-status"></div>
+        <div class="control-group">
+          <label for="filter_pattern" class="control-label">{{t "Regular expression" }}</label>
+          <input type="text" id="filter_pattern" name="pattern" placeholder="#(?P<id>[0-9]+)" />
+          <div class="alert" id="admin-filter-pattern-status"></div>
+        </div>
+        <div class="control-group">
+          <label for="filter_format_string" class="control-label">{{t "URL format string" }}</label>
+          <input type="text" id="filter_format_string" name="url_format_string" placeholder="https://github.com/zulip/zulip/issues/%(id)s" />
+          <div class="alert" id="admin-filter-format-status"></div>
+        </div>
+        <div class="control-group">
+          <div class="controls">
+            <input type="submit" class="btn btn-big btn-primary" value="{{t 'Add filter' }}" />
+          </div>
+        </div>
+      </div>
+    </div>
+  </form>
+</div>

tools/lint-all

@@ -296,6 +296,7 @@ def build_custom_checkers(by_lang):
              ('zerver/views/streams.py', 'return json_error(property_conversion)'),
              # We can't do anything about this.
              ('zerver/views/realm_emoji.py', 'return json_error(e.messages[0])'),
+             ('zerver/views/realm_filters.py', 'return json_error(e.messages[0], data={"errors": dict(e)})'),
          ]),
          'description': 'Argument to json_error should a literal string enclosed by _()'},
         # To avoid JsonableError(_variable) and JsonableError(_(variable))
@@ -352,7 +353,8 @@ def build_custom_checkers(by_lang):
     html_rules = whitespace_rules + prose_style_rules + [
         {'pattern': 'placeholder="[^{]',
          'description': "`placeholder` value should be translatable.",
-         'exclude': "static/templates/settings/emoji-settings-admin.handlebars"},
+         'exclude': set(["static/templates/settings/emoji-settings-admin.handlebars",
+                         "static/templates/settings/realm-filter-settings-admin.handlebars"])},
         {'pattern': "placeholder='[^{]",
          'description': "`placeholder` value should be translatable."},
         ] # type: RuleList

zerver/lib/actions.py

@@ -3448,14 +3448,24 @@ def notify_realm_filters(realm):
 #   * Named groups will be converted to numbered groups automatically
 #   * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags
 def do_add_realm_filter(realm, pattern, url_format_string):
-    # type: (Realm, text_type, text_type) -> None
+    # type: (Realm, text_type, text_type) -> int
-    RealmFilter(realm=realm, pattern=pattern,
+    pattern = pattern.strip()
-                url_format_string=url_format_string).save()
+    url_format_string = url_format_string.strip()
+    realm_filter = RealmFilter(
+        realm=realm, pattern=pattern,
+        url_format_string=url_format_string)
+    realm_filter.full_clean()
+    realm_filter.save()
     notify_realm_filters(realm)
 
-def do_remove_realm_filter(realm, pattern):
+    return realm_filter.id
-    # type: (Realm, text_type) -> None
+
+def do_remove_realm_filter(realm, pattern=None, id=None):
+    # type: (Realm, Optional[text_type], Optional[int]) -> None
+    if pattern is not None:
         RealmFilter.objects.get(realm=realm, pattern=pattern).delete()
+    else:
+        RealmFilter.objects.get(realm=realm, pk=id).delete()
     notify_realm_filters(realm)
 
 def get_emails_from_user_ids(user_ids):

zerver/lib/bugdown/__init__.py

@@ -1057,7 +1057,7 @@ class Bugdown(markdown.Extension):
 
         md.inlinePatterns.add('link', AtomicLinkPattern(markdown.inlinepatterns.LINK_RE, md), '>avatar')
 
-        for (pattern, format_string) in self.getConfig("realm_filters"):
+        for (pattern, format_string, id) in self.getConfig("realm_filters"):
             md.inlinePatterns.add('realm_filters/%s' % (pattern,),
                                   RealmFilterPattern(pattern, format_string), '>link')
 
@@ -1135,7 +1135,7 @@ class Bugdown(markdown.Extension):
                     del md.parser.blockprocessors[k]
 
 md_engines = {}
-realm_filter_data = {} # type: Dict[text_type, List[Tuple[text_type, text_type]]]
+realm_filter_data = {} # type: Dict[text_type, List[Tuple[text_type, text_type, int]]]
 
 class EscapeHtml(markdown.Extension):
     def extendMarkdown(self, md, md_globals):
@@ -1173,7 +1173,7 @@ def subject_links(domain, subject):
     return matches
 
 def make_realm_filters(domain, filters):
-    # type: (text_type, List[Tuple[text_type, text_type]]) -> None
+    # type: (text_type, List[Tuple[text_type, text_type, int]]) -> None
     global md_engines, realm_filter_data
     if domain in md_engines:
         del md_engines[domain]

zerver/management/commands/realm_filters.py

@@ -62,7 +62,7 @@ Example: python manage.py realm_filters --realm=zulip.com --op=show
             do_add_realm_filter(realm, pattern, url_format_string)
             sys.exit(0)
         elif options["op"] == "remove":
-            do_remove_realm_filter(realm, pattern)
+            do_remove_realm_filter(realm, pattern=pattern)
             sys.exit(0)
         else:
             self.print_help("python manage.py", "realm_filters")

zerver/migrations/0043_realm_filter_validators.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.core.validators
+import zerver.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0042_attachment_file_name_length'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='realmfilter',
+            name='pattern',
+            field=models.TextField(validators=[zerver.models.filter_pattern_validator]),
+        ),
+        migrations.AlterField(
+            model_name='realmfilter',
+            name='url_format_string',
+            field=models.TextField(validators=[django.core.validators.URLValidator, zerver.models.filter_format_validator]),
+        ),
+    ]

zerver/models.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from typing import Any, Dict, List, Set, Tuple, TypeVar, \
-    Union, Optional, Sequence, AbstractSet
+    Union, Optional, Sequence, AbstractSet, Pattern, AnyStr
 from typing.re import Match
 from zerver.lib.str_utils import NonBinaryStr
 
@@ -11,6 +11,8 @@ from django.conf import settings
 from django.contrib.auth.models import AbstractBaseUser, UserManager, \
     PermissionsMixin
 import django.contrib.auth
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.dispatch import receiver
 from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \
     user_profile_by_id_cache_key, user_profile_by_email_cache_key, \
@@ -384,10 +386,31 @@ def flush_realm_emoji(sender, **kwargs):
 post_save.connect(flush_realm_emoji, sender=RealmEmoji)
 post_delete.connect(flush_realm_emoji, sender=RealmEmoji)
 
-class RealmFilter(ModelReprMixin, models.Model):
+def filter_pattern_validator(value):
+    # type: (text_type) -> None
+    regex = re.compile(r'(?:[\w\-#]+)(\(\?P<\w+>.+\))')
+    error_msg = 'Invalid filter pattern, you must use the following format PREFIX-(?P<id>.+)'
+
+    if not regex.match(str(value)):
+        raise ValidationError(error_msg)
+
+    try:
+        re.compile(value)
+    except:
+        # Regex is invalid
+        raise ValidationError(error_msg)
+
+def filter_format_validator(value):
+    # type: (str) -> None
+    regex = re.compile(r'^[\.\/:a-zA-Z0-9_-]+%\(([a-zA-Z0-9_-]+)\)s[a-zA-Z0-9_-]*$')
+
+    if not regex.match(value):
+        raise ValidationError('URL format string must be in the following format: `https://example.com/%(\w+)s`')
+
+class RealmFilter(models.Model):
     realm = models.ForeignKey(Realm) # type: Realm
-    pattern = models.TextField() # type: text_type
+    pattern = models.TextField(validators=[filter_pattern_validator]) # type: text_type
-    url_format_string = models.TextField() # type: text_type
+    url_format_string = models.TextField(validators=[URLValidator, filter_format_validator]) # type: text_type
 
     class Meta(object):
         unique_together = ("realm", "pattern")
@@ -401,14 +424,14 @@ def get_realm_filters_cache_key(domain):
     return u'all_realm_filters:%s' % (domain,)
 
 # We have a per-process cache to avoid doing 1000 remote cache queries during page load
-per_request_realm_filters_cache = {} # type: Dict[text_type, List[Tuple[text_type, text_type]]]
+per_request_realm_filters_cache = {} # type: Dict[text_type, List[Tuple[text_type, text_type, int]]]
 
 def domain_in_local_realm_filters_cache(domain):
     # type: (text_type) -> bool
     return domain in per_request_realm_filters_cache
 
 def realm_filters_for_domain(domain):
-    # type: (text_type) -> List[Tuple[text_type, text_type]]
+    # type: (text_type) -> List[Tuple[text_type, text_type, int]]
     domain = domain.lower()
     if not domain_in_local_realm_filters_cache(domain):
         per_request_realm_filters_cache[domain] = realm_filters_for_domain_remote_cache(domain)
@@ -416,18 +439,18 @@ def realm_filters_for_domain(domain):
 
 @cache_with_key(get_realm_filters_cache_key, timeout=3600*24*7)
 def realm_filters_for_domain_remote_cache(domain):
-    # type: (text_type) -> List[Tuple[text_type, text_type]]
+    # type: (text_type) -> List[Tuple[text_type, text_type, int]]
     filters = []
     for realm_filter in RealmFilter.objects.filter(realm=get_realm(domain)):
-       filters.append((realm_filter.pattern, realm_filter.url_format_string))
+        filters.append((realm_filter.pattern, realm_filter.url_format_string, realm_filter.id))
 
     return filters
 
 def all_realm_filters():
-    # type: () -> Dict[text_type, List[Tuple[text_type, text_type]]]
+    # type: () -> Dict[text_type, List[Tuple[text_type, text_type, int]]]
-    filters = defaultdict(list) # type: Dict[text_type, List[Tuple[text_type, text_type]]]
+    filters = defaultdict(list) # type: Dict[text_type, List[Tuple[text_type, text_type, int]]]
     for realm_filter in RealmFilter.objects.all():
-       filters[realm_filter.realm.domain].append((realm_filter.pattern, realm_filter.url_format_string))
+        filters[realm_filter.realm.domain].append((realm_filter.pattern, realm_filter.url_format_string, realm_filter.id))
 
     return filters
 

zerver/tests/test_bugdown.py

@@ -444,7 +444,7 @@ class BugdownTest(TestCase):
                                    url_format_string=url_format_string)
         realm_filter.save()
         self.assertEqual(
-            str(realm_filter),
+            realm_filter.__unicode__(),
             '<RealmFilter(zulip.com): #(?P<id>[0-9]{2,8})'
             ' https://trac.zulip.net/ticket/%(id)s>')
 
@@ -461,6 +461,15 @@ class BugdownTest(TestCase):
         self.assertEqual(converted, '<p>We should fix <a href="https://trac.zulip.net/ticket/224" target="_blank" title="https://trac.zulip.net/ticket/224">#224</a> and <a href="https://trac.zulip.net/ticket/115" target="_blank" title="https://trac.zulip.net/ticket/115">#115</a>, but not issue#124 or #1124z or <a href="https://trac.zulip.net/ticket/16" target="_blank" title="https://trac.zulip.net/ticket/16">trac #15</a> today.</p>')
         self.assertEqual(converted_subject,  [u'https://trac.zulip.net/ticket/444'])
 
+        RealmFilter(realm=get_realm_by_string_id('zulip'), pattern=r'#(?P<id>[a-zA-Z]+-[0-9]+)',
+                    url_format_string=r'https://trac.zulip.net/ticket/%(id)s').save()
+        msg = Message(sender=get_user_profile_by_email('hamlet@zulip.com'))
+
+        content = '#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging'
+        converted = bugdown.convert(content, realm_domain='zulip.com', message=msg)
+
+        self.assertEqual(converted, '<p><a href="https://trac.zulip.net/ticket/ZUL-123" target="_blank" title="https://trac.zulip.net/ticket/ZUL-123">#ZUL-123</a> was fixed and code was deployed to production, also <a href="https://trac.zulip.net/ticket/zul-321" target="_blank" title="https://trac.zulip.net/ticket/zul-321">#zul-321</a> was deployed to staging</p>')
+
     def test_maybe_update_realm_filters(self):
         # type: () -> None
         realm = get_realm_by_string_id('zulip')
@@ -476,7 +485,7 @@ class BugdownTest(TestCase):
         zulip_filters = all_filters['zulip.com']
         self.assertEqual(len(zulip_filters), 1)
         self.assertEqual(zulip_filters[0],
-            (u'#(?P<id>[0-9]{2,8})', u'https://trac.zulip.net/ticket/%(id)s'))
+            (u'#(?P<id>[0-9]{2,8})', u'https://trac.zulip.net/ticket/%(id)s', realm_filter.id))
 
     def test_flush_realm_filter(self):
         # type: () -> None

zerver/tests/test_events.py

@@ -663,12 +663,12 @@ class EventsRegisterTest(ZulipTestCase):
             ('type', equals('realm_filters')),
             ('realm_filters', check_list(None)), # TODO: validate tuples in the list
         ])
-        events = self.do_test(lambda: do_add_realm_filter(get_realm_by_string_id("zulip"), "#[123]",
+        events = self.do_test(lambda: do_add_realm_filter(get_realm_by_string_id("zulip"), "#(?P<id>[123])",
                                                           "https://realm.com/my_realm_filter/%(id)s"))
         error = schema_checker('events[0]', events[0])
         self.assert_on_error(error)
 
-        self.do_test(lambda: do_remove_realm_filter(get_realm_by_string_id("zulip"), "#[123]"))
+        self.do_test(lambda: do_remove_realm_filter(get_realm_by_string_id("zulip"), "#(?P<id>[123])"))
         error = schema_checker('events[0]', events[0])
         self.assert_on_error(error)
 

zerver/tests/test_realm_filters.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from zerver.lib.actions import get_realm_by_string_id, do_add_realm_filter
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.models import RealmFilter
+import ujson
+
+
+class RealmFilterTest(ZulipTestCase):
+
+    def test_list(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        realm = get_realm_by_string_id('zulip')
+        do_add_realm_filter(
+            realm,
+            "#(?P<id>[123])",
+            "https://realm.com/my_realm_filter/%(id)s")
+        result = self.client_get("/json/realm/filters")
+        self.assert_json_success(result)
+        self.assertEqual(200, result.status_code)
+        content = ujson.loads(result.content)
+        self.assertEqual(len(content["filters"]), 1)
+
+    def test_create(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        data = {"pattern": "", "url_format_string": "https://realm.com/my_realm_filter/%(id)s"}
+        result = self.client_post("/json/realm/filters", info=data)
+        self.assert_json_error(result, 'This field cannot be blank.')
+
+        data['pattern'] = '$a'
+        result = self.client_post("/json/realm/filters", info=data)
+        self.assert_json_error(result, 'Invalid filter pattern, you must use the following format PREFIX-(?P<id>.+)')
+
+        data['pattern'] = 'ZUL-(?P<id>\d++)'
+        result = self.client_post("/json/realm/filters", info=data)
+        self.assert_json_error(result, 'Invalid filter pattern, you must use the following format PREFIX-(?P<id>.+)')
+
+        data['pattern'] = 'ZUL-(?P<id>\d+)'
+        data['url_format_string'] = '$fgfg'
+        result = self.client_post("/json/realm/filters", info=data)
+        self.assert_json_error(result, 'URL format string must be in the following format: `https://example.com/%(\\w+)s`')
+
+        data['url_format_string'] = 'https://realm.com/my_realm_filter/%(id)s'
+        result = self.client_post("/json/realm/filters", info=data)
+        self.assert_json_success(result)
+
+    def test_not_realm_admin(self):
+        self.login("hamlet@zulip.com")
+        result = self.client_post("/json/realm/filters")
+        self.assert_json_error(result, 'Must be a realm administrator')
+        result = self.client_delete("/json/realm/filters/15")
+        self.assert_json_error(result, 'Must be a realm administrator')
+
+    def test_delete(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        realm = get_realm_by_string_id('zulip')
+        filter_id = do_add_realm_filter(
+            realm,
+            "#(?P<id>[123])",
+            "https://realm.com/my_realm_filter/%(id)s")
+        filters_count = RealmFilter.objects.count()
+        result = self.client_delete("/json/realm/filters/{0}".format(filter_id + 1))
+        self.assert_json_error(result, 'Filter not found')
+
+        result = self.client_delete("/json/realm/filters/{0}".format(filter_id))
+        self.assert_json_success(result)
+        self.assertEqual(RealmFilter.objects.count(), filters_count - 1)

zerver/views/realm_filters.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import
+
+from six import text_type
+from django.core.exceptions import ValidationError
+from django.http import HttpRequest, HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+from django.utils.translation import ugettext as _
+
+from zerver.decorator import has_request_variables, REQ, require_realm_admin
+from zerver.lib.actions import do_add_realm_filter, do_remove_realm_filter
+from zerver.lib.response import json_success, json_error
+from zerver.lib.rest import rest_dispatch as _rest_dispatch
+from zerver.lib.validator import check_string
+from zerver.models import realm_filters_for_domain, UserProfile, RealmFilter
+
+
+# Custom realm filters
+def list_filters(request, user_profile):
+    # type: (HttpRequest, UserProfile) -> HttpResponse
+    filters = realm_filters_for_domain(user_profile.realm.domain)
+    return json_success({'filters': filters})
+
+
+@require_realm_admin
+@has_request_variables
+def create_filter(request, user_profile, pattern=REQ(),
+                  url_format_string=REQ()):
+    # type: (HttpRequest, UserProfile, text_type, text_type) -> HttpResponse
+    try:
+        filter_id = do_add_realm_filter(
+            realm=user_profile.realm,
+            pattern=pattern,
+            url_format_string=url_format_string
+        )
+        return json_success({'id': filter_id})
+    except ValidationError as e:
+        return json_error(e.messages[0], data={"errors": dict(e)})
+
+
+@require_realm_admin
+def delete_filter(request, user_profile, filter_id):
+    # type: (HttpRequest, UserProfile, int) -> HttpResponse
+    try:
+        do_remove_realm_filter(realm=user_profile.realm, id=filter_id)
+    except RealmFilter.DoesNotExist:
+        return json_error(_('Filter not found'))
+    return json_success()

zproject/urls.py

@@ -162,6 +162,13 @@ v1_api_and_json_patterns = [
     url(r'^realm/emoji/(?P<emoji_name>[0-9a-zA-Z.\-_]+(?<![.\-_]))$', rest_dispatch,
         {'DELETE': 'zerver.views.realm_emoji.delete_emoji'}),
 
+    # realm/filters -> zerver.views.realm_filters
+    url(r'^realm/filters$', rest_dispatch,
+        {'GET': 'zerver.views.realm_filters.list_filters',
+         'POST': 'zerver.views.realm_filters.create_filter'}),
+    url(r'^realm/filters/(?P<filter_id>\d+)$', rest_dispatch,
+        {'DELETE': 'zerver.views.realm_filters.delete_filter'}),
+
     # users -> zerver.views.users
     url(r'^users$', rest_dispatch,
         {'GET': 'zerver.views.users.get_members_backend',