From d7e1e4a2c0b11e6fc4a3fa396aa7be9f75da8943 Mon Sep 17 00:00:00 2001 From: Vladislav Manchev Date: Sat, 13 Feb 2016 20:17:15 +0200 Subject: [PATCH] Add initial implementation of custom realm filters. This PR was abandoned by Vladislav and then substantially modified by Igor Tokarev and Tim Abbott to complete it and fix a number of bugs. Fixes #544. --- docs/prod-maintain-secure-upgrade.md | 8 +- frontend_tests/casper_tests/10-admin.js | 35 ++++++++ frontend_tests/node_tests/dispatch.js | 1 + frontend_tests/node_tests/templates.js | 22 +++++ static/js/admin.js | 86 +++++++++++++++++++ static/js/server_events.js | 1 + static/styles/settings.css | 12 +++ static/templates/admin_filter_list.handlebars | 15 ++++ static/templates/admin_tab.handlebars | 1 + .../realm-filter-settings-admin.handlebars | 36 ++++++++ tools/lint-all | 4 +- zerver/lib/actions.py | 22 +++-- zerver/lib/bugdown/__init__.py | 6 +- zerver/management/commands/realm_filters.py | 2 +- .../0043_realm_filter_validators.py | 26 ++++++ zerver/models.py | 45 +++++++--- zerver/tests/test_bugdown.py | 13 ++- zerver/tests/test_events.py | 4 +- zerver/tests/test_realm_filters.py | 71 +++++++++++++++ zerver/views/realm_filters.py | 47 ++++++++++ zproject/urls.py | 7 ++ 21 files changed, 431 insertions(+), 33 deletions(-) create mode 100644 static/templates/admin_filter_list.handlebars create mode 100644 static/templates/settings/realm-filter-settings-admin.handlebars create mode 100644 zerver/migrations/0043_realm_filter_validators.py create mode 100644 zerver/tests/test_realm_filters.py create mode 100644 zerver/views/realm_filters.py diff --git a/docs/prod-maintain-secure-upgrade.md b/docs/prod-maintain-secure-upgrade.md index 68fbb7cb39..f91c292dec 100644 --- a/docs/prod-maintain-secure-upgrade.md +++ b/docs/prod-maintain-secure-upgrade.md @@ -613,11 +613,5 @@ server, and suggested procedure. ### Other useful manage.py commands There are a large number of useful management commands under -`zerver/manangement/commands/`; you can also see them listed using +`zerver/management/commands/`; you can also see them listed using `./manage.py` with no arguments. - -One such command worth highlighting because it's a valuable feature -with no UI in the Administration page is `./manage.py realm_filters`, -which allows you to configure certain patterns in messages to be -automatically linkified, e.g., whenever someone mentions "T1234", it -could be auto-linkified to ticket 1234 in your team's Trac instance. diff --git a/frontend_tests/casper_tests/10-admin.js b/frontend_tests/casper_tests/10-admin.js index 6014114353..e418ee7631 100644 --- a/frontend_tests/casper_tests/10-admin.js +++ b/frontend_tests/casper_tests/10-admin.js @@ -177,6 +177,41 @@ casper.then(function () { }); }); +// Test custom realm filters +casper.waitForSelector('.admin-filter-form', function () { + casper.fill('form.admin-filter-form', { + 'pattern': '#(?P[0-9]+)', + 'url_format_string': 'https://trac.example.com/ticket/%(id)s' + }); + casper.click('form.admin-filter-form input.btn'); +}); + +casper.waitUntilVisible('div#admin-filter-status', function () { + casper.test.assertSelectorHasText('div#admin-filter-status', 'Custom filter added!'); +}); + +casper.waitForSelector('.filter_row', function () { + casper.test.assertSelectorHasText('.filter_row span.filter_pattern', '#(?P[0-9]+)'); + casper.test.assertSelectorHasText('.filter_row span.filter_url_format_string', 'https://trac.example.com/ticket/%(id)s'); + casper.click('.filter_row button'); +}); + +casper.waitWhileSelector('.filter_row', function () { + casper.test.assertDoesntExist('.filter_row'); +}); + +casper.waitForSelector('.admin-filter-form', function () { + casper.fill('form.admin-filter-form', { + 'pattern': 'a$', + 'url_format_string': 'https://trac.example.com/ticket/%(id)s' + }); + casper.click('form.admin-filter-form input.btn'); +}); + +casper.waitUntilVisible('div#admin-filter-pattern-status', function () { + casper.test.assertSelectorHasText('div#admin-filter-pattern-status', 'Failed: Invalid filter pattern, you must use the following format PREFIX-(?P.+)'); +}); + function get_suggestions(str) { casper.then(function () { casper.evaluate(function (str) { diff --git a/frontend_tests/node_tests/dispatch.js b/frontend_tests/node_tests/dispatch.js index 84355380c2..eb24eead38 100644 --- a/frontend_tests/node_tests/dispatch.js +++ b/frontend_tests/node_tests/dispatch.js @@ -569,6 +569,7 @@ run(function (override, capture, args) { // realm_filters var event = event_fixtures.realm_filters; page_params.realm_filters = []; + override('admin', 'populate_filters', noop); dispatch(event); assert_same(page_params.realm_filters, event.realm_filters); diff --git a/frontend_tests/node_tests/templates.js b/frontend_tests/node_tests/templates.js index d146d7e253..82543a39c2 100644 --- a/frontend_tests/node_tests/templates.js +++ b/frontend_tests/node_tests/templates.js @@ -129,6 +129,28 @@ function render(template_name, args) { assert.equal(emoji_url.attr('src'), 'http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png'); }()); +(function admin_filter_list() { + var args = { + filter: { + "pattern": "#(?P[0-9]+)", + "url_format_string": "https://trac.example.com/ticket/%(id)s" + } + }; + + var html = ''; + html += ''; + html += render('admin_filter_list', args); + html += ''; + + global.write_test_output('admin_filter_list', html); + + var filter_pattern = $(html).find('tr.filter_row:first span.filter_pattern'); + var filter_format = $(html).find('tr.filter_row:first span.filter_url_format_string'); + + assert.equal(filter_pattern.text(), '#(?P[0-9]+)'); + assert.equal(filter_format.text(), 'https://trac.example.com/ticket/%(id)s'); +}()); + (function admin_streams_list() { var html = ''; var streams = ['devel', 'trac', 'zulip']; diff --git a/static/js/admin.js b/static/js/admin.js index 753979472c..b3b1177d03 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -186,6 +186,25 @@ exports.populate_emoji = function (emoji_data) { loading.destroy_indicator($('#admin_page_emoji_loading_indicator')); }; +exports.populate_filters = function (filters_data) { + var filters_table = $("#admin_filters_table").expectOne(); + filters_table.find("tr.filter_row").remove(); + _.each(filters_data, function (filter) { + filters_table.append( + templates.render( + "admin_filter_list", { + filter: { + pattern: filter[0], + url_format_string: filter[1], + id: filter[2] + } + } + ) + ); + }); + loading.destroy_indicator($('#admin_page_filters_loading_indicator')); +}; + exports.reset_realm_default_language = function () { $("#id_realm_default_language").val(page_params.realm_default_language); }; @@ -275,6 +294,9 @@ function _setup_page() { $("#admin-realm-message-editing-status").expectOne().hide(); $("#admin-realm-default-language-status").expectOne().hide(); $("#admin-emoji-status").expectOne().hide(); + $('#admin-filter-status').expectOne().hide(); + $('#admin-filter-pattern-status').expectOne().hide(); + $('#admin-filter-format-status').expectOne().hide(); $("#id_realm_default_language").val(page_params.realm_default_language); @@ -285,6 +307,7 @@ function _setup_page() { loading.make_indicator($('#admin_page_deactivated_users_loading_indicator')); loading.make_indicator($('#admin_page_emoji_loading_indicator')); loading.make_indicator($('#admin_page_auth_methods_loading_indicator')); + loading.make_indicator($('#admin_page_filters_loading_indicator')); // Populate users and bots tables channel.get({ @@ -311,6 +334,9 @@ function _setup_page() { exports.populate_emoji(page_params.realm_emoji); exports.update_default_streams_table(); + // Populate filters table + exports.populate_filters(page_params.realm_filters); + // Setup click handlers $(".admin_user_table").on("click", ".deactivate", function (e) { e.preventDefault(); @@ -780,6 +806,66 @@ function _setup_page() { }); }); + $('.admin_filters_table').on('click', '.delete', function (e) { + e.preventDefault(); + e.stopPropagation(); + var btn = $(this); + + channel.del({ + url: '/json/realm/filters/' + encodeURIComponent(btn.attr('data-filter-id')), + error: function (xhr, error_type) { + if (xhr.status.toString().charAt(0) === "4") { + btn.closest("td").html( + $("

").addClass("text-error").text($.parseJSON(xhr.responseText).msg) + ); + } else { + btn.text("Failed!"); + } + }, + success: function () { + var row = btn.parents('tr'); + row.remove(); + } + }); + }); + + $(".administration").on("submit", "form.admin-filter-form", function (e) { + e.preventDefault(); + e.stopPropagation(); + var filter_status = $('#admin-filter-status'); + var pattern_status = $('#admin-filter-pattern-status'); + var format_status = $('#admin-filter-format-status'); + filter_status.hide(); + pattern_status.hide(); + format_status.hide(); + var filter = {}; + $(this).serializeArray().map(function (x){filter[x.name] = x.value;}); + + channel.post({ + url: "/json/realm/filters", + data: $(this).serialize(), + success: function (data) { + filter.id = data.id; + ui.report_success(i18n.t("Custom filter added!"), filter_status); + }, + error: function (xhr, error) { + var errors = $.parseJSON(xhr.responseText).errors; + if (errors.pattern !== undefined) { + xhr.responseText = JSON.stringify({msg: errors.pattern}); + ui.report_error(i18n.t("Failed"), xhr, pattern_status); + } + if (errors.url_format_string !== undefined) { + xhr.responseText = JSON.stringify({msg: errors.url_format_string}); + ui.report_error(i18n.t("Failed"), xhr, format_status); + } + if (errors.__all__ !== undefined) { + xhr.responseText = JSON.stringify({msg: errors.__all__}); + ui.report_error(i18n.t("Failed"), xhr, filter_status); + } + } + }); + }); + } exports.setup_page = function () { diff --git a/static/js/server_events.js b/static/js/server_events.js index 1056b101a5..bf73c57628 100644 --- a/static/js/server_events.js +++ b/static/js/server_events.js @@ -92,6 +92,7 @@ function dispatch_normal_event(event) { case 'realm_filters': page_params.realm_filters = event.realm_filters; echo.set_realm_filters(page_params.realm_filters); + admin.populate_filters(page_params.realm_filters); break; case 'realm_user': diff --git a/static/styles/settings.css b/static/styles/settings.css index 6c8b43ed29..67a81137fe 100644 --- a/static/styles/settings.css +++ b/static/styles/settings.css @@ -193,6 +193,7 @@ input[type=checkbox].inline-block { #settings .settings-section .new-bot-form, #settings .settings-section .new-alert-word-form, #emoji-settings .new-emoji-form, + #filter-settings .new-filter-form, #settings .settings-section .notification-settings-form, #settings .settings-section .display-settings-form, #settings .settings-section .edit-bot-form-box { @@ -204,6 +205,7 @@ input[type=checkbox].inline-block { #settings .settings-section .new-bot-form .control-label, #settings .settings-section .new-alert-word-form .control-label, #emoji-settings .new-emoji-form .control-label, + #filter-settings .new-filter-form .control-label, #settings .settings-section .edit-bot-form-box .control-label { display: block; width: 120px; @@ -219,6 +221,7 @@ input[type=checkbox].inline-block { #settings .settings-section .new-bot-form .controls, #settings .settings-section .new-alert-word-form button, #emoji-settings .new-emoji-form .controls, + #filter-settings .new-filter-form .controls, #settings .settings-section .edit-bot-form-box .controls { margin: auto; text-align: center; @@ -271,6 +274,14 @@ input[type=checkbox].inline-block { margin: 0 auto; } +.admin_filters_table { + margin-top: 20px; +} + +#admin-filter-pattern-status, #admin-filter-format-status { + margin: 20px 0 0 0; +} + #bots_list { display: none; list-style-type: none; @@ -349,6 +360,7 @@ input[type=checkbox].inline-block { #create_bot_form .control-label, #create_alert_word_form .control-label, .admin-emoji-form .control-label, +.admin-filter-form .control-label, .default-stream-form .control-label { width: 10em; text-align: right; diff --git a/static/templates/admin_filter_list.handlebars b/static/templates/admin_filter_list.handlebars new file mode 100644 index 0000000000..a5f2fd08c0 --- /dev/null +++ b/static/templates/admin_filter_list.handlebars @@ -0,0 +1,15 @@ +{{#with filter}} +

+ + + + +{{/with}} diff --git a/static/templates/admin_tab.handlebars b/static/templates/admin_tab.handlebars index 5a6fac5f47..b7e0bd4ec4 100644 --- a/static/templates/admin_tab.handlebars +++ b/static/templates/admin_tab.handlebars @@ -27,6 +27,7 @@
{{ partial "organization-settings-admin" }} {{ partial "emoji-settings-admin" }} + {{ partial "realm-filter-settings-admin" }} {{ partial "auth-methods-settings-admin" }}
diff --git a/static/templates/settings/realm-filter-settings-admin.handlebars b/static/templates/settings/realm-filter-settings-admin.handlebars new file mode 100644 index 0000000000..b2e0f19b82 --- /dev/null +++ b/static/templates/settings/realm-filter-settings-admin.handlebars @@ -0,0 +1,36 @@ +
+
{{t "Custom linkification filters" }}
+
+

{{#tr this}}Configure patterns that will be automatically linkified when used in Zulip message bodies or topics. For example, you could make typing "#123" automatically become a link to issue #123 in an issue tracker.{{/tr}}

+
+ {{pattern}} + + {{url_format_string}} + + +
+ + + + + +
{{t "Pattern" }}{{t "URL format string" }}{{t "Actions" }}
+ +
+
+
+
{{t "Add a New Filter" }}
+
+
+ + +
+
+
+ + +
+
+
+
+ +
+
+
+
+
+ diff --git a/tools/lint-all b/tools/lint-all index e127fc75a3..39d9d0f12f 100755 --- a/tools/lint-all +++ b/tools/lint-all @@ -296,6 +296,7 @@ def build_custom_checkers(by_lang): ('zerver/views/streams.py', 'return json_error(property_conversion)'), # We can't do anything about this. ('zerver/views/realm_emoji.py', 'return json_error(e.messages[0])'), + ('zerver/views/realm_filters.py', 'return json_error(e.messages[0], data={"errors": dict(e)})'), ]), 'description': 'Argument to json_error should a literal string enclosed by _()'}, # To avoid JsonableError(_variable) and JsonableError(_(variable)) @@ -352,7 +353,8 @@ def build_custom_checkers(by_lang): html_rules = whitespace_rules + prose_style_rules + [ {'pattern': 'placeholder="[^{]', 'description': "`placeholder` value should be translatable.", - 'exclude': "static/templates/settings/emoji-settings-admin.handlebars"}, + 'exclude': set(["static/templates/settings/emoji-settings-admin.handlebars", + "static/templates/settings/realm-filter-settings-admin.handlebars"])}, {'pattern': "placeholder='[^{]", 'description': "`placeholder` value should be translatable."}, ] # type: RuleList diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 8804397526..ef22b993ce 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -3448,14 +3448,24 @@ def notify_realm_filters(realm): # * Named groups will be converted to numbered groups automatically # * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags def do_add_realm_filter(realm, pattern, url_format_string): - # type: (Realm, text_type, text_type) -> None - RealmFilter(realm=realm, pattern=pattern, - url_format_string=url_format_string).save() + # type: (Realm, text_type, text_type) -> int + pattern = pattern.strip() + url_format_string = url_format_string.strip() + realm_filter = RealmFilter( + realm=realm, pattern=pattern, + url_format_string=url_format_string) + realm_filter.full_clean() + realm_filter.save() notify_realm_filters(realm) -def do_remove_realm_filter(realm, pattern): - # type: (Realm, text_type) -> None - RealmFilter.objects.get(realm=realm, pattern=pattern).delete() + return realm_filter.id + +def do_remove_realm_filter(realm, pattern=None, id=None): + # type: (Realm, Optional[text_type], Optional[int]) -> None + if pattern is not None: + RealmFilter.objects.get(realm=realm, pattern=pattern).delete() + else: + RealmFilter.objects.get(realm=realm, pk=id).delete() notify_realm_filters(realm) def get_emails_from_user_ids(user_ids): diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index cdb754ce27..289ac0e261 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -1057,7 +1057,7 @@ class Bugdown(markdown.Extension): md.inlinePatterns.add('link', AtomicLinkPattern(markdown.inlinepatterns.LINK_RE, md), '>avatar') - for (pattern, format_string) in self.getConfig("realm_filters"): + for (pattern, format_string, id) in self.getConfig("realm_filters"): md.inlinePatterns.add('realm_filters/%s' % (pattern,), RealmFilterPattern(pattern, format_string), '>link') @@ -1135,7 +1135,7 @@ class Bugdown(markdown.Extension): del md.parser.blockprocessors[k] md_engines = {} -realm_filter_data = {} # type: Dict[text_type, List[Tuple[text_type, text_type]]] +realm_filter_data = {} # type: Dict[text_type, List[Tuple[text_type, text_type, int]]] class EscapeHtml(markdown.Extension): def extendMarkdown(self, md, md_globals): @@ -1173,7 +1173,7 @@ def subject_links(domain, subject): return matches def make_realm_filters(domain, filters): - # type: (text_type, List[Tuple[text_type, text_type]]) -> None + # type: (text_type, List[Tuple[text_type, text_type, int]]) -> None global md_engines, realm_filter_data if domain in md_engines: del md_engines[domain] diff --git a/zerver/management/commands/realm_filters.py b/zerver/management/commands/realm_filters.py index 8e3e73cfc3..2ed3b019b7 100644 --- a/zerver/management/commands/realm_filters.py +++ b/zerver/management/commands/realm_filters.py @@ -62,7 +62,7 @@ Example: python manage.py realm_filters --realm=zulip.com --op=show do_add_realm_filter(realm, pattern, url_format_string) sys.exit(0) elif options["op"] == "remove": - do_remove_realm_filter(realm, pattern) + do_remove_realm_filter(realm, pattern=pattern) sys.exit(0) else: self.print_help("python manage.py", "realm_filters") diff --git a/zerver/migrations/0043_realm_filter_validators.py b/zerver/migrations/0043_realm_filter_validators.py new file mode 100644 index 0000000000..5c1e492c79 --- /dev/null +++ b/zerver/migrations/0043_realm_filter_validators.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.core.validators +import zerver.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('zerver', '0042_attachment_file_name_length'), + ] + + operations = [ + migrations.AlterField( + model_name='realmfilter', + name='pattern', + field=models.TextField(validators=[zerver.models.filter_pattern_validator]), + ), + migrations.AlterField( + model_name='realmfilter', + name='url_format_string', + field=models.TextField(validators=[django.core.validators.URLValidator, zerver.models.filter_format_validator]), + ), + ] diff --git a/zerver/models.py b/zerver/models.py index 7e53d172b5..4117c35e7e 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from typing import Any, Dict, List, Set, Tuple, TypeVar, \ - Union, Optional, Sequence, AbstractSet + Union, Optional, Sequence, AbstractSet, Pattern, AnyStr from typing.re import Match from zerver.lib.str_utils import NonBinaryStr @@ -11,6 +11,8 @@ from django.conf import settings from django.contrib.auth.models import AbstractBaseUser, UserManager, \ PermissionsMixin import django.contrib.auth +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator from django.dispatch import receiver from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \ user_profile_by_id_cache_key, user_profile_by_email_cache_key, \ @@ -384,10 +386,31 @@ def flush_realm_emoji(sender, **kwargs): post_save.connect(flush_realm_emoji, sender=RealmEmoji) post_delete.connect(flush_realm_emoji, sender=RealmEmoji) -class RealmFilter(ModelReprMixin, models.Model): +def filter_pattern_validator(value): + # type: (text_type) -> None + regex = re.compile(r'(?:[\w\-#]+)(\(\?P<\w+>.+\))') + error_msg = 'Invalid filter pattern, you must use the following format PREFIX-(?P.+)' + + if not regex.match(str(value)): + raise ValidationError(error_msg) + + try: + re.compile(value) + except: + # Regex is invalid + raise ValidationError(error_msg) + +def filter_format_validator(value): + # type: (str) -> None + regex = re.compile(r'^[\.\/:a-zA-Z0-9_-]+%\(([a-zA-Z0-9_-]+)\)s[a-zA-Z0-9_-]*$') + + if not regex.match(value): + raise ValidationError('URL format string must be in the following format: `https://example.com/%(\w+)s`') + +class RealmFilter(models.Model): realm = models.ForeignKey(Realm) # type: Realm - pattern = models.TextField() # type: text_type - url_format_string = models.TextField() # type: text_type + pattern = models.TextField(validators=[filter_pattern_validator]) # type: text_type + url_format_string = models.TextField(validators=[URLValidator, filter_format_validator]) # type: text_type class Meta(object): unique_together = ("realm", "pattern") @@ -401,14 +424,14 @@ def get_realm_filters_cache_key(domain): return u'all_realm_filters:%s' % (domain,) # We have a per-process cache to avoid doing 1000 remote cache queries during page load -per_request_realm_filters_cache = {} # type: Dict[text_type, List[Tuple[text_type, text_type]]] +per_request_realm_filters_cache = {} # type: Dict[text_type, List[Tuple[text_type, text_type, int]]] def domain_in_local_realm_filters_cache(domain): # type: (text_type) -> bool return domain in per_request_realm_filters_cache def realm_filters_for_domain(domain): - # type: (text_type) -> List[Tuple[text_type, text_type]] + # type: (text_type) -> List[Tuple[text_type, text_type, int]] domain = domain.lower() if not domain_in_local_realm_filters_cache(domain): per_request_realm_filters_cache[domain] = realm_filters_for_domain_remote_cache(domain) @@ -416,18 +439,18 @@ def realm_filters_for_domain(domain): @cache_with_key(get_realm_filters_cache_key, timeout=3600*24*7) def realm_filters_for_domain_remote_cache(domain): - # type: (text_type) -> List[Tuple[text_type, text_type]] + # type: (text_type) -> List[Tuple[text_type, text_type, int]] filters = [] for realm_filter in RealmFilter.objects.filter(realm=get_realm(domain)): - filters.append((realm_filter.pattern, realm_filter.url_format_string)) + filters.append((realm_filter.pattern, realm_filter.url_format_string, realm_filter.id)) return filters def all_realm_filters(): - # type: () -> Dict[text_type, List[Tuple[text_type, text_type]]] - filters = defaultdict(list) # type: Dict[text_type, List[Tuple[text_type, text_type]]] + # type: () -> Dict[text_type, List[Tuple[text_type, text_type, int]]] + filters = defaultdict(list) # type: Dict[text_type, List[Tuple[text_type, text_type, int]]] for realm_filter in RealmFilter.objects.all(): - filters[realm_filter.realm.domain].append((realm_filter.pattern, realm_filter.url_format_string)) + filters[realm_filter.realm.domain].append((realm_filter.pattern, realm_filter.url_format_string, realm_filter.id)) return filters diff --git a/zerver/tests/test_bugdown.py b/zerver/tests/test_bugdown.py index df814f6957..5818fa8004 100644 --- a/zerver/tests/test_bugdown.py +++ b/zerver/tests/test_bugdown.py @@ -444,7 +444,7 @@ class BugdownTest(TestCase): url_format_string=url_format_string) realm_filter.save() self.assertEqual( - str(realm_filter), + realm_filter.__unicode__(), '[0-9]{2,8})' ' https://trac.zulip.net/ticket/%(id)s>') @@ -461,6 +461,15 @@ class BugdownTest(TestCase): self.assertEqual(converted, '

We should fix #224 and #115, but not issue#124 or #1124z or trac #15 today.

') self.assertEqual(converted_subject, [u'https://trac.zulip.net/ticket/444']) + RealmFilter(realm=get_realm_by_string_id('zulip'), pattern=r'#(?P[a-zA-Z]+-[0-9]+)', + url_format_string=r'https://trac.zulip.net/ticket/%(id)s').save() + msg = Message(sender=get_user_profile_by_email('hamlet@zulip.com')) + + content = '#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging' + converted = bugdown.convert(content, realm_domain='zulip.com', message=msg) + + self.assertEqual(converted, '

#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging

') + def test_maybe_update_realm_filters(self): # type: () -> None realm = get_realm_by_string_id('zulip') @@ -476,7 +485,7 @@ class BugdownTest(TestCase): zulip_filters = all_filters['zulip.com'] self.assertEqual(len(zulip_filters), 1) self.assertEqual(zulip_filters[0], - (u'#(?P[0-9]{2,8})', u'https://trac.zulip.net/ticket/%(id)s')) + (u'#(?P[0-9]{2,8})', u'https://trac.zulip.net/ticket/%(id)s', realm_filter.id)) def test_flush_realm_filter(self): # type: () -> None diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 68d0aab0ff..9199dc6f5d 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -663,12 +663,12 @@ class EventsRegisterTest(ZulipTestCase): ('type', equals('realm_filters')), ('realm_filters', check_list(None)), # TODO: validate tuples in the list ]) - events = self.do_test(lambda: do_add_realm_filter(get_realm_by_string_id("zulip"), "#[123]", + events = self.do_test(lambda: do_add_realm_filter(get_realm_by_string_id("zulip"), "#(?P[123])", "https://realm.com/my_realm_filter/%(id)s")) error = schema_checker('events[0]', events[0]) self.assert_on_error(error) - self.do_test(lambda: do_remove_realm_filter(get_realm_by_string_id("zulip"), "#[123]")) + self.do_test(lambda: do_remove_realm_filter(get_realm_by_string_id("zulip"), "#(?P[123])")) error = schema_checker('events[0]', events[0]) self.assert_on_error(error) diff --git a/zerver/tests/test_realm_filters.py b/zerver/tests/test_realm_filters.py new file mode 100644 index 0000000000..d7ac80a3b6 --- /dev/null +++ b/zerver/tests/test_realm_filters.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from zerver.lib.actions import get_realm_by_string_id, do_add_realm_filter +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import RealmFilter +import ujson + + +class RealmFilterTest(ZulipTestCase): + + def test_list(self): + # type: () -> None + self.login("iago@zulip.com") + realm = get_realm_by_string_id('zulip') + do_add_realm_filter( + realm, + "#(?P[123])", + "https://realm.com/my_realm_filter/%(id)s") + result = self.client_get("/json/realm/filters") + self.assert_json_success(result) + self.assertEqual(200, result.status_code) + content = ujson.loads(result.content) + self.assertEqual(len(content["filters"]), 1) + + def test_create(self): + # type: () -> None + self.login("iago@zulip.com") + data = {"pattern": "", "url_format_string": "https://realm.com/my_realm_filter/%(id)s"} + result = self.client_post("/json/realm/filters", info=data) + self.assert_json_error(result, 'This field cannot be blank.') + + data['pattern'] = '$a' + result = self.client_post("/json/realm/filters", info=data) + self.assert_json_error(result, 'Invalid filter pattern, you must use the following format PREFIX-(?P.+)') + + data['pattern'] = 'ZUL-(?P\d++)' + result = self.client_post("/json/realm/filters", info=data) + self.assert_json_error(result, 'Invalid filter pattern, you must use the following format PREFIX-(?P.+)') + + data['pattern'] = 'ZUL-(?P\d+)' + data['url_format_string'] = '$fgfg' + result = self.client_post("/json/realm/filters", info=data) + self.assert_json_error(result, 'URL format string must be in the following format: `https://example.com/%(\\w+)s`') + + data['url_format_string'] = 'https://realm.com/my_realm_filter/%(id)s' + result = self.client_post("/json/realm/filters", info=data) + self.assert_json_success(result) + + def test_not_realm_admin(self): + self.login("hamlet@zulip.com") + result = self.client_post("/json/realm/filters") + self.assert_json_error(result, 'Must be a realm administrator') + result = self.client_delete("/json/realm/filters/15") + self.assert_json_error(result, 'Must be a realm administrator') + + def test_delete(self): + # type: () -> None + self.login("iago@zulip.com") + realm = get_realm_by_string_id('zulip') + filter_id = do_add_realm_filter( + realm, + "#(?P[123])", + "https://realm.com/my_realm_filter/%(id)s") + filters_count = RealmFilter.objects.count() + result = self.client_delete("/json/realm/filters/{0}".format(filter_id + 1)) + self.assert_json_error(result, 'Filter not found') + + result = self.client_delete("/json/realm/filters/{0}".format(filter_id)) + self.assert_json_success(result) + self.assertEqual(RealmFilter.objects.count(), filters_count - 1) diff --git a/zerver/views/realm_filters.py b/zerver/views/realm_filters.py new file mode 100644 index 0000000000..24d8c30ba0 --- /dev/null +++ b/zerver/views/realm_filters.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import + +from six import text_type +from django.core.exceptions import ValidationError +from django.http import HttpRequest, HttpResponse +from django.views.decorators.csrf import csrf_exempt +from django.utils.translation import ugettext as _ + +from zerver.decorator import has_request_variables, REQ, require_realm_admin +from zerver.lib.actions import do_add_realm_filter, do_remove_realm_filter +from zerver.lib.response import json_success, json_error +from zerver.lib.rest import rest_dispatch as _rest_dispatch +from zerver.lib.validator import check_string +from zerver.models import realm_filters_for_domain, UserProfile, RealmFilter + + +# Custom realm filters +def list_filters(request, user_profile): + # type: (HttpRequest, UserProfile) -> HttpResponse + filters = realm_filters_for_domain(user_profile.realm.domain) + return json_success({'filters': filters}) + + +@require_realm_admin +@has_request_variables +def create_filter(request, user_profile, pattern=REQ(), + url_format_string=REQ()): + # type: (HttpRequest, UserProfile, text_type, text_type) -> HttpResponse + try: + filter_id = do_add_realm_filter( + realm=user_profile.realm, + pattern=pattern, + url_format_string=url_format_string + ) + return json_success({'id': filter_id}) + except ValidationError as e: + return json_error(e.messages[0], data={"errors": dict(e)}) + + +@require_realm_admin +def delete_filter(request, user_profile, filter_id): + # type: (HttpRequest, UserProfile, int) -> HttpResponse + try: + do_remove_realm_filter(realm=user_profile.realm, id=filter_id) + except RealmFilter.DoesNotExist: + return json_error(_('Filter not found')) + return json_success() diff --git a/zproject/urls.py b/zproject/urls.py index 2a0b9d74e5..827eafb626 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -162,6 +162,13 @@ v1_api_and_json_patterns = [ url(r'^realm/emoji/(?P[0-9a-zA-Z.\-_]+(? zerver.views.realm_filters + url(r'^realm/filters$', rest_dispatch, + {'GET': 'zerver.views.realm_filters.list_filters', + 'POST': 'zerver.views.realm_filters.create_filter'}), + url(r'^realm/filters/(?P\d+)$', rest_dispatch, + {'DELETE': 'zerver.views.realm_filters.delete_filter'}), + # users -> zerver.views.users url(r'^users$', rest_dispatch, {'GET': 'zerver.views.users.get_members_backend',