diff --git a/version.py b/version.py index 50366b1ddf..7f31ba9d0c 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 175 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = (233, 2) +PROVISION_VERSION = (233, 3) diff --git a/zerver/migrations/0441_backfill_realmfilter_url_template.py b/zerver/migrations/0441_backfill_realmfilter_url_template.py new file mode 100644 index 0000000000..c173a1f8a5 --- /dev/null +++ b/zerver/migrations/0441_backfill_realmfilter_url_template.py @@ -0,0 +1,48 @@ +# Generated by Django 4.0.7 on 2022-10-02 20:50 + +import re + +import uri_template +from django.db import migrations +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + + +def transform_to_url_template_syntax( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + linkifier_model = apps.get_model("zerver", "RealmFilter") + + linkifiers = linkifier_model.objects.all() + # Matches anything of the form %(variable_name)s + var_pattern = re.compile(r"(? None: - Recipient = apps.get_model("zerver", "Recipient") - Message = apps.get_model("zerver", "Message") + RealmFilter = apps.get_model("zerver", "RealmFilter") iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) - denmark_recipient = Recipient.objects.get(type=2, type_id=denmark.id) - self.msg_id = Message.objects.create( - recipient_id=denmark_recipient.id, - subject="topic 4", - sender_id=iago.id, - sending_client_id=1, - content="current message text", - date_sent=timezone_now(), - ).id + urls = [ + "http://example.com/", + "https://example.com/", + "https://user:password@example.com/", + "https://example.com/@user/thing", + "https://example.com/!path", + "https://example.com/foo.bar", + "https://example.com/foo[bar]", + "https://example.com/{foo}", + "https://example.com/{foo}{bars}", + "https://example.com/{foo}/and/{bar}", + "https://example.com/?foo={foo}", + "https://example.com/%ab", + "https://example.com/%ba", + "https://example.com/%21", + "https://example.com/words%20with%20spaces", + "https://example.com/back%20to%20{back}", + "https://example.com/encoded%2fwith%2fletters", + "https://example.com/encoded%2Fwith%2Fupper%2Fcase%2Fletters", + "https://example.com/%%", + "https://example.com/%%(", + "https://example.com/%%()", + "https://example.com/%%(foo", + "https://example.com/%%(foo)", + "https://example.com/%%(foo)s", + "https://example.com/%(foo)s", + "https://example.com/%(foo)s%(bar)s", + ] + self.linkifier_ids = [] - # topic edits contain only "prev_subject" field. - # stream edits contain only "prev_stream" field. - msg = Message.objects.filter(id=self.msg_id).first() - msg.edit_history = orjson.dumps( - [ - { - "user_id": 11, - "timestamp": 1644405050, - "prev_stream": 3, - "prev_subject": "topic 3", - }, - {"user_id": 11, "timestamp": 1644405040, "prev_stream": 2}, - { - "user_id": 11, - "timestamp": 1644405030, - "prev_content": "test content and topic edit", - "prev_rendered_content": "
test content and topic edit
", - "prev_rendered_content_version": 1, - "prev_subject": "topic 2", - }, - {"user_id": 11, "timestamp": 1644405020, "prev_subject": "topic 1"}, - { - "user_id": 11, - "timestamp": 1644405010, - "prev_content": "test content only edit", - "prev_rendered_content": "test content only edit
", - "prev_rendered_content_version": 1, - }, - ] - ).decode() - msg.save(update_fields=["edit_history"]) + for index, url in enumerate(urls): + self.linkifier_ids.append( + RealmFilter.objects.create( + realm=iago.realm, + pattern=f"dummy{index}", + url_format_string=url, + ).id + ) - def test_message_legacy_edit_history_format(self) -> None: - Message = self.apps.get_model("zerver", "Message") - Recipient = self.apps.get_model("zerver", "Recipient") + def test_converted_url_templates(self) -> None: + RealmFilter = self.apps.get_model("zerver", "RealmFilter") - iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) + expected_urls = [ + "http://example.com/", + "https://example.com/", + "https://user:password@example.com/", + "https://example.com/@user/thing", + "https://example.com/!path", + "https://example.com/foo.bar", + "https://example.com/foo[bar]", + "https://example.com/%7Bfoo%7D", + "https://example.com/%7Bfoo%7D%7Bbars%7D", + "https://example.com/%7Bfoo%7D/and/%7Bbar%7D", + "https://example.com/?foo=%7Bfoo%7D", + "https://example.com/%ab", + "https://example.com/%ba", + "https://example.com/%21", + "https://example.com/words%20with%20spaces", + "https://example.com/back%20to%20%7Bback%7D", + "https://example.com/encoded%2fwith%2fletters", + "https://example.com/encoded%2Fwith%2Fupper%2Fcase%2Fletters", + "https://example.com/%", + "https://example.com/%(", + "https://example.com/%()", + "https://example.com/%(foo", + "https://example.com/%(foo)", + "https://example.com/%(foo)s", + "https://example.com/{foo}", + "https://example.com/{foo}{bar}", + ] - msg = Message.objects.filter(id=self.msg_id).first() - msg_stream_id = Recipient.objects.get(id=msg.recipient_id).type_id - new_edit_history = orjson.loads(msg.edit_history) - - self.assert_length(new_edit_history, 5) - - # stream and topic edit entry - self.assertFalse("prev_subject" in new_edit_history[0]) - self.assertEqual(new_edit_history[0]["prev_topic"], "topic 3") - self.assertEqual(new_edit_history[0]["topic"], msg.subject) - self.assertEqual(new_edit_history[0]["prev_stream"], 3) - self.assertEqual(new_edit_history[0]["stream"], msg_stream_id) - self.assertEqual(new_edit_history[0]["stream"], denmark.id) - self.assertEqual( - set(new_edit_history[0].keys()), - {"timestamp", "prev_topic", "topic", "prev_stream", "stream", "user_id"}, - ) - - # stream only edit entry - self.assertEqual(new_edit_history[1]["prev_stream"], 2) - self.assertEqual(new_edit_history[1]["stream"], 3) - self.assertEqual( - set(new_edit_history[1].keys()), {"timestamp", "prev_stream", "stream", "user_id"} - ) - - # topic and content edit entry - self.assertFalse("prev_subject" in new_edit_history[2]) - self.assertEqual(new_edit_history[2]["prev_topic"], "topic 2") - self.assertEqual(new_edit_history[2]["topic"], "topic 3") - self.assertEqual(new_edit_history[2]["prev_content"], "test content and topic edit") - self.assertEqual( - new_edit_history[2]["prev_rendered_content"], "test content and topic edit
" - ) - self.assertEqual(new_edit_history[2]["prev_rendered_content_version"], 1) - self.assertEqual( - set(new_edit_history[2].keys()), - { - "timestamp", - "prev_topic", - "topic", - "prev_content", - "prev_rendered_content", - "prev_rendered_content_version", - "user_id", - }, - ) - - # topic only edit entry - self.assertFalse("prev_subject" in new_edit_history[3]) - self.assertEqual(new_edit_history[3]["prev_topic"], "topic 1") - self.assertEqual(new_edit_history[3]["topic"], "topic 2") - self.assertEqual( - set(new_edit_history[3].keys()), {"timestamp", "prev_topic", "topic", "user_id"} - ) - - # content only edit entry - not retested because never changes - self.assertEqual(new_edit_history[4]["prev_content"], "test content only edit") - self.assertEqual( - new_edit_history[4]["prev_rendered_content"], "test content only edit
" - ) - self.assertEqual(new_edit_history[4]["prev_rendered_content_version"], 1) - self.assertEqual( - set(new_edit_history[4].keys()), - { - "timestamp", - "prev_content", - "prev_rendered_content", - "prev_rendered_content_version", - "user_id", - }, - ) - - -@skip("Fails because newer migrations have since been merged.") # nocoverage # skipped -class MessageEditHistoryModernFormats(MigrationsTestCase): - migrate_from = "0376_set_realmemoji_author_and_reupload_realmemoji" - migrate_to = "0377_message_edit_history_format" - - msg_id: Optional[int] = None - - @use_db_models - def setUpBeforeMigration(self, apps: StateApps) -> None: - Recipient = apps.get_model("zerver", "Recipient") - Message = apps.get_model("zerver", "Message") - - iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) - denmark_recipient = Recipient.objects.get(type=2, type_id=denmark.id) - - self.msg_id = Message.objects.create( - recipient_id=denmark_recipient.id, - subject="topic 4", - sender_id=iago.id, - sending_client_id=1, - content="current message text", - date_sent=timezone_now(), - ).id - - msg = Message.objects.filter(id=self.msg_id).first() - msg_stream_id = Recipient.objects.get(id=msg.recipient_id).type_id - - # topic edits contain "topic" and "prev_topic" fields. - # stream edits contain "stream" and "prev_stream" fields. - msg.edit_history = orjson.dumps( - [ - { - "user_id": 11, - "timestamp": 1644405050, - "stream": msg_stream_id, - "prev_stream": 3, - "topic": msg.subject, - "prev_topic": "topic 3", - }, - {"user_id": 11, "timestamp": 1644405040, "prev_stream": 2, "stream": 3}, - { - "user_id": 11, - "timestamp": 1644405030, - "prev_content": "test content and topic edit", - "prev_rendered_content": "test content and topic edit
", - "prev_rendered_content_version": 1, - "prev_topic": "topic 2", - "topic": "topic 3", - }, - { - "user_id": 11, - "timestamp": 1644405020, - "prev_topic": "topic 1", - "topic": "topic 2", - }, - ] - ).decode() - msg.save(update_fields=["edit_history"]) - - def test_message_modern_edit_history_format(self) -> None: - Message = self.apps.get_model("zerver", "Message") - Recipient = self.apps.get_model("zerver", "Recipient") - - iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) - - msg = Message.objects.filter(id=self.msg_id).first() - msg_stream_id = Recipient.objects.get(id=msg.recipient_id).type_id - new_edit_history = orjson.loads(msg.edit_history) - - self.assert_length(new_edit_history, 4) - - # stream and topic edit entry - self.assertEqual(new_edit_history[0]["prev_topic"], "topic 3") - self.assertEqual(new_edit_history[0]["topic"], msg.subject) - self.assertEqual(new_edit_history[0]["prev_stream"], 3) - self.assertEqual(new_edit_history[0]["stream"], msg_stream_id) - self.assertEqual(new_edit_history[0]["stream"], denmark.id) - self.assertEqual( - set(new_edit_history[0].keys()), - {"timestamp", "prev_topic", "topic", "prev_stream", "stream", "user_id"}, - ) - - # stream only edit entry - self.assertEqual(new_edit_history[1]["prev_stream"], 2) - self.assertEqual(new_edit_history[1]["stream"], 3) - self.assertEqual( - set(new_edit_history[1].keys()), {"timestamp", "prev_stream", "stream", "user_id"} - ) - - # topic and content edit entry - self.assertEqual(new_edit_history[2]["prev_topic"], "topic 2") - self.assertEqual(new_edit_history[2]["topic"], "topic 3") - self.assertEqual(new_edit_history[2]["prev_content"], "test content and topic edit") - self.assertEqual( - new_edit_history[2]["prev_rendered_content"], "test content and topic edit
" - ) - self.assertEqual(new_edit_history[2]["prev_rendered_content_version"], 1) - self.assertEqual( - set(new_edit_history[2].keys()), - { - "timestamp", - "prev_topic", - "topic", - "prev_content", - "prev_rendered_content", - "prev_rendered_content_version", - "user_id", - }, - ) - - # topic only edit entry - self.assertEqual(new_edit_history[3]["prev_topic"], "topic 1") - self.assertEqual(new_edit_history[3]["topic"], "topic 2") - self.assertEqual( - set(new_edit_history[3].keys()), {"timestamp", "prev_topic", "topic", "user_id"} - ) - - -@skip("Fails because newer migrations have since been merged.") # nocoverage # skipped -class MessageEditHistoryIntermediateFormats(MigrationsTestCase): - migrate_from = "0376_set_realmemoji_author_and_reupload_realmemoji" - migrate_to = "0377_message_edit_history_format" - - msg_id: Optional[int] = None - - @use_db_models - def setUpBeforeMigration(self, apps: StateApps) -> None: - Recipient = apps.get_model("zerver", "Recipient") - Message = apps.get_model("zerver", "Message") - - iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) - denmark_recipient = Recipient.objects.get(type=2, type_id=denmark.id) - - self.msg_id = Message.objects.create( - recipient_id=denmark_recipient.id, - subject="topic 4", - sender_id=iago.id, - sending_client_id=1, - content="current message text", - date_sent=timezone_now(), - ).id - - msg = Message.objects.filter(id=self.msg_id).first() - msg_stream_id = Recipient.objects.get(id=msg.recipient_id).type_id - - # topic edits contain "prev_subject", "topic" and "prev_topic" fields. - # stream edits contain "stream" and "prev_stream" fields. - msg.edit_history = orjson.dumps( - [ - { - "user_id": 11, - "timestamp": 1644405050, - "stream": msg_stream_id, - "prev_stream": 3, - "topic": msg.subject, - "prev_topic": "topic 3", - "prev_subject": "topic 3", - }, - {"user_id": 11, "timestamp": 1644405040, "prev_stream": 2, "stream": 3}, - { - "user_id": 11, - "timestamp": 1644405030, - "prev_content": "test content and topic edit", - "prev_rendered_content": "test content and topic edit
", - "prev_rendered_content_version": 1, - "prev_topic": "topic 2", - "prev_subject": "topic 2", - "topic": "topic 3", - }, - { - "user_id": 11, - "timestamp": 1644405020, - "prev_topic": "topic 1", - "prev_subject": "topic 1", - "topic": "topic 2", - }, - ] - ).decode() - msg.save(update_fields=["edit_history"]) - - def test_message_temporary_edit_history_format(self) -> None: - Message = self.apps.get_model("zerver", "Message") - Recipient = self.apps.get_model("zerver", "Recipient") - - iago = self.example_user("iago") - stream_name = "Denmark" - denmark = get_stream(stream_name, iago.realm) - - msg = Message.objects.filter(id=self.msg_id).first() - msg_stream_id = Recipient.objects.get(id=msg.recipient_id).type_id - new_edit_history = orjson.loads(msg.edit_history) - - self.assert_length(new_edit_history, 4) - - # stream and topic edit entry - self.assertFalse("prev_subject" in new_edit_history[0]) - self.assertEqual(new_edit_history[0]["prev_topic"], "topic 3") - self.assertEqual(new_edit_history[0]["topic"], msg.subject) - self.assertEqual(new_edit_history[0]["prev_stream"], 3) - self.assertEqual(new_edit_history[0]["stream"], msg_stream_id) - self.assertEqual(new_edit_history[0]["stream"], denmark.id) - self.assertEqual( - set(new_edit_history[0].keys()), - {"timestamp", "prev_topic", "topic", "prev_stream", "stream", "user_id"}, - ) - - # stream only edit entry - self.assertEqual(new_edit_history[1]["prev_stream"], 2) - self.assertEqual(new_edit_history[1]["stream"], 3) - self.assertEqual( - set(new_edit_history[1].keys()), {"timestamp", "prev_stream", "stream", "user_id"} - ) - - # topic and content edit entry - self.assertFalse("prev_subject" in new_edit_history[2]) - self.assertEqual(new_edit_history[2]["prev_topic"], "topic 2") - self.assertEqual(new_edit_history[2]["topic"], "topic 3") - self.assertEqual(new_edit_history[2]["prev_content"], "test content and topic edit") - self.assertEqual( - new_edit_history[2]["prev_rendered_content"], "test content and topic edit
" - ) - self.assertEqual(new_edit_history[2]["prev_rendered_content_version"], 1) - self.assertEqual( - set(new_edit_history[2].keys()), - { - "timestamp", - "prev_topic", - "topic", - "prev_content", - "prev_rendered_content", - "prev_rendered_content_version", - "user_id", - }, - ) - - # topic only edit entry - self.assertFalse("prev_subject" in new_edit_history[3]) - self.assertEqual(new_edit_history[3]["prev_topic"], "topic 1") - self.assertEqual(new_edit_history[3]["topic"], "topic 2") - self.assertEqual( - set(new_edit_history[3].keys()), {"timestamp", "prev_topic", "topic", "user_id"} - ) + for linkifier_id, expected in zip(self.linkifier_ids, expected_urls): + linkifier = RealmFilter.objects.filter(id=linkifier_id).first() + self.assertIsNotNone(linkifier) + self.assertEqual(linkifier.url_template, expected)