Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
zulip/zerver/tests/test_link_embed.py

308 lines
13 KiB
Python
Raw Normal View History

Add oembed/Open Graph/Meta tags data retrieval from inline links. This change adds support for displaying inline open graph previews for links posted into Zulip. It is designed to interact correctly with message editing. This adds the new settings.INLINE_URL_EMBED_PREVIEW setting to control whether this feature is enabled. By default, this setting is currently disabled, so that we can burn it in for a bit before it impacts users more broadly. Eventually, we may want to make this manageable via a (set of?) per-realm settings. E.g. I can imagine a realm wanting to be able to enable/disable it for certain URLs.
2016-10-27 12:06:44 +02:00
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function
import mock
import ujson
from typing import Any
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
from requests.exceptions import ConnectionError
Add oembed/Open Graph/Meta tags data retrieval from inline links. This change adds support for displaying inline open graph previews for links posted into Zulip. It is designed to interact correctly with message editing. This adds the new settings.INLINE_URL_EMBED_PREVIEW setting to control whether this feature is enabled. By default, this setting is currently disabled, so that we can burn it in for a bit before it impacts users more broadly. Eventually, we may want to make this manageable via a (set of?) per-realm settings. E.g. I can imagine a realm wanting to be able to enable/disable it for certain URLs.
2016-10-27 12:06:44 +02:00
from django.test import override_settings
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
from zerver.models import Recipient, Message
Add oembed/Open Graph/Meta tags data retrieval from inline links. This change adds support for displaying inline open graph previews for links posted into Zulip. It is designed to interact correctly with message editing. This adds the new settings.INLINE_URL_EMBED_PREVIEW setting to control whether this feature is enabled. By default, this setting is currently disabled, so that we can burn it in for a bit before it impacts users more broadly. Eventually, we may want to make this manageable via a (set of?) per-realm settings. E.g. I can imagine a realm wanting to be able to enable/disable it for certain URLs.
2016-10-27 12:06:44 +02:00
from zerver.lib.test_classes import ZulipTestCase
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
from zerver.lib.test_helpers import MockPythonResponse
from zerver.worker.queue_processors import FetchLinksEmbedData
from zerver.lib.url_preview.preview import get_link_embed_data
Add oembed/Open Graph/Meta tags data retrieval from inline links. This change adds support for displaying inline open graph previews for links posted into Zulip. It is designed to interact correctly with message editing. This adds the new settings.INLINE_URL_EMBED_PREVIEW setting to control whether this feature is enabled. By default, this setting is currently disabled, so that we can burn it in for a bit before it impacts users more broadly. Eventually, we may want to make this manageable via a (set of?) per-realm settings. E.g. I can imagine a realm wanting to be able to enable/disable it for certain URLs.
2016-10-27 12:06:44 +02:00
from zerver.lib.url_preview.oembed import get_oembed_data
from zerver.lib.url_preview.parsers import (
OpenGraphParser, GenericParser)
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
TEST_CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'default',
},
'database': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'url-preview',
}
}
Add oembed/Open Graph/Meta tags data retrieval from inline links. This change adds support for displaying inline open graph previews for links posted into Zulip. It is designed to interact correctly with message editing. This adds the new settings.INLINE_URL_EMBED_PREVIEW setting to control whether this feature is enabled. By default, this setting is currently disabled, so that we can burn it in for a bit before it impacts users more broadly. Eventually, we may want to make this manageable via a (set of?) per-realm settings. E.g. I can imagine a realm wanting to be able to enable/disable it for certain URLs.
2016-10-27 12:06:44 +02:00
@override_settings(INLINE_URL_EMBED_PREVIEW=True)
class OembedTestCase(ZulipTestCase):
@mock.patch('pyoembed.requests.get')
def test_present_provider(self, get):
# type: (Any) -> None
get.return_value = response = mock.Mock()
response.headers = {'content-type': 'application/json'}
response.ok = True
response_data = {
'type': 'rich',
'thumbnail_url': 'https://scontent.cdninstagram.com/t51.2885-15/n.jpg',
'thumbnail_width': 640,
'thumbnail_height': 426,
'title': 'NASA',
'html': '<p>test</p>',
'version': '1.0',
'width': 658,
'height': None}
response.text = ujson.dumps(response_data)
url = 'http://instagram.com/p/BLtI2WdAymy'
data = get_oembed_data(url)
self.assertIsInstance(data, dict)
self.assertIn('title', data)
self.assertEqual(data['title'], response_data['title'])
@mock.patch('pyoembed.requests.get')
def test_error_request(self, get):
# type: (Any) -> None
get.return_value = response = mock.Mock()
response.ok = False
url = 'http://instagram.com/p/BLtI2WdAymy'
data = get_oembed_data(url)
self.assertIsNone(data)
class OpenGraphParserTestCase(ZulipTestCase):
def test_page_with_og(self):
# type: () -> None
html = """<html>
<head>
<meta property="og:title" content="The Rock" />
<meta property="og:type" content="video.movie" />
<meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
<meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
<meta property="og:description" content="The Rock film" />
</head>
</html>"""
parser = OpenGraphParser(html)
result = parser.extract_data()
self.assertIn('title', result)
self.assertEqual(result['title'], 'The Rock')
self.assertEqual(result.get('description'), 'The Rock film')
class GenericParserTestCase(ZulipTestCase):
def test_parser(self):
# type: () -> None
html = """
<html>
<head><title>Test title</title></head>
<body>
<h1>Main header</h1>
<p>Description text</p>
</body>
</html>
"""
parser = GenericParser(html)
result = parser.extract_data()
self.assertEqual(result.get('title'), 'Test title')
self.assertEqual(result.get('description'), 'Description text')
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
def test_extract_image(self):
# type: () -> None
html = """
<html>
<body>
<h1>Main header</h1>
<img src="http://test.com/test.jpg">
<div>
<p>Description text</p>
</div>
</body>
</html>
"""
parser = GenericParser(html)
result = parser.extract_data()
self.assertEqual(result.get('title'), 'Main header')
self.assertEqual(result.get('description'), 'Description text')
self.assertEqual(result.get('image'), 'http://test.com/test.jpg')
def test_extract_description(self):
# type: () -> None
html = """
<html>
<body>
<div>
<div>
<p>Description text</p>
</div>
</div>
</body>
</html>
"""
parser = GenericParser(html)
result = parser.extract_data()
self.assertEqual(result.get('description'), 'Description text')
html = """
<html>
<head><meta name="description" content="description 123"</head>
<body></body>
</html>
"""
parser = GenericParser(html)
result = parser.extract_data()
self.assertEqual(result.get('description'), 'description 123')
html = "<html><body></body></html>"
parser = GenericParser(html)
result = parser.extract_data()
self.assertIsNone(result.get('description'))
class PreviewTestCase(ZulipTestCase):
PreviewTestCase: Factor out open_graph_html variable.
2017-02-23 06:17:16 +01:00
open_graph_html = """
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
<html>
<head>
<title>Test title</title>
<meta property="og:title" content="The Rock" />
<meta property="og:type" content="video.movie" />
<meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
<meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
</head>
<body>
<h1>Main header</h1>
<p>Description text</p>
</body>
</html>
"""
PreviewTestCase: Factor out open_graph_html variable.
2017-02-23 06:17:16 +01:00
PreviewTestCase: Use the actual queued event. This makes the test significantly simpler and more faithful, but not mocking the event put into queue_json_publish.
2017-02-23 06:16:30 +01:00
@override_settings(INLINE_URL_EMBED_PREVIEW=True)
preview: Fix adding links in message editing. When you edit a message to contain links, and URL previews are enabled, previously we'd throw an exception, because the realm ID wasn't included in the event. Also adds a test so that we can have effective test coverage on this codepath, though this history is actually that I found the bug through writing this test :).
2017-02-23 06:20:01 +01:00
def test_edit_message_history(self):
# type: () -> None
self.login("hamlet@zulip.com")
msg_id = self.send_message("hamlet@zulip.com", "Scotland", Recipient.STREAM,
subject="editing", content="original")
url = 'http://test.org/'
response = MockPythonResponse(self.open_graph_html, 200)
mocked_response = mock.Mock(
side_effect=lambda k: {url: response}.get(k, MockPythonResponse('', 404)))
with mock.patch('zerver.views.messages.queue_json_publish') as patched:
result = self.client_patch("/json/messages/" + str(msg_id), {
'message_id': msg_id, 'content': url,
})
self.assert_json_success(result)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
embedded_link = '<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(url)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(embedded_link, msg.rendered_content)
@override_settings(INLINE_URL_EMBED_PREVIEW=True)
bugdown: Add domain-name to relative image-url in open graph data. fixes #4608.
2017-05-04 01:41:35 +02:00
def _send_message_with_test_org_url(self, sender_email, queue_should_run=True, relative_url=False):
# type: (str, bool, bool) -> Message
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
url = 'http://test.org/'
PreviewTestCase: Use the actual queued event. This makes the test significantly simpler and more faithful, but not mocking the event put into queue_json_publish.
2017-02-23 06:16:30 +01:00
with mock.patch('zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_message(
sender_email, "cordelia@zulip.com",
Recipient.PERSONAL, subject="url", content=url)
if queue_should_run:
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
else:
patched.assert_not_called()
# If we nothing was put in the queue, we don't need to
# run the queue processor or any of the following code
return Message.objects.select_related("sender").get(id=msg_id)
# Verify the initial message doesn't have the embedded links rendered
render_incoming_message: Require a realm object.
2017-01-22 05:55:30 +01:00
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertNotIn(
'<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(url),
msg.rendered_content)
PreviewTestCase: Use the actual queued event. This makes the test significantly simpler and more faithful, but not mocking the event put into queue_json_publish.
2017-02-23 06:16:30 +01:00
# Mock the network request result so the test can be fast without Internet
response = MockPythonResponse(self.open_graph_html, 200)
bugdown: Add domain-name to relative image-url in open graph data. fixes #4608.
2017-05-04 01:41:35 +02:00
if relative_url is True:
response = MockPythonResponse(self.open_graph_html.replace('http://ia.media-imdb.com', ''), 200)
PreviewTestCase: Use the actual queued event. This makes the test significantly simpler and more faithful, but not mocking the event put into queue_json_publish.
2017-02-23 06:16:30 +01:00
mocked_response = mock.Mock(
side_effect=lambda k: {url: response}.get(k, MockPythonResponse('', 404)))
# Run the queue processor to potentially rerender things
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
render_incoming_message: Require a realm object.
2017-01-22 05:55:30 +01:00
msg = Message.objects.select_related("sender").get(id=msg_id)
bugdown: Disable link previews when bots send messages. We do not use `get_link_embed_data` for messsages sent by bots, as bots often repeat the same URL over and over again and are generally either text-focused or have their own mechanisms to provide preview content. Fixes #2968.
2017-02-03 23:28:26 +01:00
return msg
def test_get_link_embed_data(self):
# type: () -> None
url = 'http://test.org/'
embedded_link = '<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(url)
# When humans send, we should get embedded content.
msg = self._send_message_with_test_org_url(sender_email='hamlet@zulip.com')
self.assertIn(embedded_link, msg.rendered_content)
# We don't want embedded content for bots.
PreviewTestCase: Use the actual queued event. This makes the test significantly simpler and more faithful, but not mocking the event put into queue_json_publish.
2017-02-23 06:16:30 +01:00
msg = self._send_message_with_test_org_url(sender_email='webhook-bot@zulip.com',
queue_should_run=False)
bugdown: Disable link previews when bots send messages. We do not use `get_link_embed_data` for messsages sent by bots, as bots often repeat the same URL over and over again and are generally either text-focused or have their own mechanisms to provide preview content. Fixes #2968.
2017-02-03 23:28:26 +01:00
self.assertNotIn(embedded_link, msg.rendered_content)
# Try another human to make sure bot failure was due to the
# bot sending the message and not some other reason.
msg = self._send_message_with_test_org_url(sender_email='prospero@zulip.com')
self.assertIn(embedded_link, msg.rendered_content)
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
Add realm-level settings to control inline image and url preview. This gives users more control in case they don't want previews, especially for the "previews of linked websites" feature. Fixes: #2640.
2017-03-13 14:42:03 +01:00
def test_inline_url_embed_preview(self):
# type: () -> None
with_preview = '<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>\n<div class="message_embed"><a class="message_embed_image" href="http://test.org/" style="background-image: url(http://ia.media-imdb.com/images/rock.jpg)" target="_blank"></a><div class="data-container"><div class="message_embed_title"><a href="http://test.org/" target="_blank" title="The Rock">The Rock</a></div><div class="message_embed_description">Description text</div></div></div>'
without_preview = '<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>'
msg = self._send_message_with_test_org_url(sender_email='hamlet@zulip.com')
self.assertEqual(msg.rendered_content, with_preview)
realm = msg.get_realm()
setattr(realm, 'inline_url_embed_preview', False)
realm.save()
msg = self._send_message_with_test_org_url(sender_email='prospero@zulip.com', queue_should_run=False)
self.assertEqual(msg.rendered_content, without_preview)
bugdown: Add domain-name to relative image-url in open graph data. fixes #4608.
2017-05-04 01:41:35 +02:00
def test_inline_url_embed_preview_with_relative_image_url(self):
# type: () -> None
with_preview_relative = '<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>\n<div class="message_embed"><a class="message_embed_image" href="http://test.org/" style="background-image: url(http://test.org/images/rock.jpg)" target="_blank"></a><div class="data-container"><div class="message_embed_title"><a href="http://test.org/" target="_blank" title="The Rock">The Rock</a></div><div class="message_embed_description">Description text</div></div></div>'
# Try case where the opengraph image is a relative url.
msg = self._send_message_with_test_org_url(sender_email='prospero@zulip.com', relative_url=True)
self.assertEqual(msg.rendered_content, with_preview_relative)
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
def test_http_error_get_data(self):
# type: () -> None
url = 'http://test.org/'
msg_id = self.send_message(
"hamlet@zulip.com", "cordelia@zulip.com",
Recipient.PERSONAL, subject="url", content=url)
render_incoming_message: Require a realm object.
2017-01-22 05:55:30 +01:00
msg = Message.objects.select_related("sender").get(id=msg_id)
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
event = {
'message_id': msg_id,
'urls': [url],
render_incoming_message: Require a realm object.
2017-01-22 05:55:30 +01:00
'message_realm_id': msg.sender.realm_id,
URL preview: Improve test coverage.
2016-12-13 04:20:33 +01:00
'message_content': url}
with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mock.Mock(side_effect=ConnectionError())):
with mock.patch('logging.error') as error_mock:
FetchLinksEmbedData().consume(event)
self.assertEqual(error_mock.call_count, 1)
msg = Message.objects.get(id=msg_id)
self.assertEqual(
'<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_invalid_link(self):
# type: () -> None
with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False, CACHES=TEST_CACHES):
self.assertIsNone(get_link_embed_data('com.notvalidlink'))