2016-11-16 12:37:26 +01:00
|
|
|
import ujson
|
|
|
|
from django.conf import settings
|
2017-05-25 11:50:13 +02:00
|
|
|
from django.db import close_old_connections
|
|
|
|
from django.core import signals
|
2017-10-28 01:17:38 +02:00
|
|
|
from django.test import override_settings
|
dependencies: Remove WebSockets system for sending messages.
Zulip has had a small use of WebSockets (specifically, for the code
path of sending messages, via the webapp only) since ~2013. We
originally added this use of WebSockets in the hope that the latency
benefits of doing so would allow us to avoid implementing a markdown
local echo; they were not. Further, HTTP/2 may have eliminated the
latency difference we hoped to exploit by using WebSockets in any
case.
While we’d originally imagined using WebSockets for other endpoints,
there was never a good justification for moving more components to the
WebSockets system.
This WebSockets code path had a lot of downsides/complexity,
including:
* The messy hack involving constructing an emulated request object to
hook into doing Django requests.
* The `message_senders` queue processor system, which increases RAM
needs and must be provisioned independently from the rest of the
server).
* A duplicate check_send_receive_time Nagios test specific to
WebSockets.
* The requirement for users to have their firewalls/NATs allow
WebSocket connections, and a setting to disable them for networks
where WebSockets don’t work.
* Dependencies on the SockJS family of libraries, which has at times
been poorly maintained, and periodically throws random JavaScript
exceptions in our production environments without a deep enough
traceback to effectively investigate.
* A total of about 1600 lines of our code related to the feature.
* Increased load on the Tornado system, especially around a Zulip
server restart, and especially for large installations like
zulipchat.com, resulting in extra delay before messages can be sent
again.
As detailed in
https://github.com/zulip/zulip/pull/12862#issuecomment-536152397, it
appears that removing WebSockets moderately increases the time it
takes for the `send_message` API query to return from the server, but
does not significantly change the time between when a message is sent
and when it is received by clients. We don’t understand the reason
for that change (suggesting the possibility of a measurement error),
and even if it is a real change, we consider that potential small
latency regression to be acceptable.
If we later want WebSockets, we’ll likely want to just use Django
Channels.
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2019-07-23 01:43:40 +02:00
|
|
|
from tornado.httpclient import HTTPResponse
|
2016-11-16 12:37:26 +01:00
|
|
|
|
|
|
|
from zerver.lib.test_classes import ZulipTestCase
|
|
|
|
|
dependencies: Remove WebSockets system for sending messages.
Zulip has had a small use of WebSockets (specifically, for the code
path of sending messages, via the webapp only) since ~2013. We
originally added this use of WebSockets in the hope that the latency
benefits of doing so would allow us to avoid implementing a markdown
local echo; they were not. Further, HTTP/2 may have eliminated the
latency difference we hoped to exploit by using WebSockets in any
case.
While we’d originally imagined using WebSockets for other endpoints,
there was never a good justification for moving more components to the
WebSockets system.
This WebSockets code path had a lot of downsides/complexity,
including:
* The messy hack involving constructing an emulated request object to
hook into doing Django requests.
* The `message_senders` queue processor system, which increases RAM
needs and must be provisioned independently from the rest of the
server).
* A duplicate check_send_receive_time Nagios test specific to
WebSockets.
* The requirement for users to have their firewalls/NATs allow
WebSocket connections, and a setting to disable them for networks
where WebSockets don’t work.
* Dependencies on the SockJS family of libraries, which has at times
been poorly maintained, and periodically throws random JavaScript
exceptions in our production environments without a deep enough
traceback to effectively investigate.
* A total of about 1600 lines of our code related to the feature.
* Increased load on the Tornado system, especially around a Zulip
server restart, and especially for large installations like
zulipchat.com, resulting in extra delay before messages can be sent
again.
As detailed in
https://github.com/zulip/zulip/pull/12862#issuecomment-536152397, it
appears that removing WebSockets moderately increases the time it
takes for the `send_message` API query to return from the server, but
does not significantly change the time between when a message is sent
and when it is received by clients. We don’t understand the reason
for that change (suggesting the possibility of a measurement error),
and even if it is a real change, we consider that potential small
latency regression to be acceptable.
If we later want WebSockets, we’ll likely want to just use Django
Channels.
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2019-07-23 01:43:40 +02:00
|
|
|
from tornado.testing import AsyncHTTPTestCase
|
2016-11-16 12:37:26 +01:00
|
|
|
from tornado.web import Application
|
|
|
|
|
|
|
|
from zerver.tornado.application import create_tornado_application
|
2017-05-25 11:50:13 +02:00
|
|
|
from zerver.tornado import event_queue
|
dependencies: Remove WebSockets system for sending messages.
Zulip has had a small use of WebSockets (specifically, for the code
path of sending messages, via the webapp only) since ~2013. We
originally added this use of WebSockets in the hope that the latency
benefits of doing so would allow us to avoid implementing a markdown
local echo; they were not. Further, HTTP/2 may have eliminated the
latency difference we hoped to exploit by using WebSockets in any
case.
While we’d originally imagined using WebSockets for other endpoints,
there was never a good justification for moving more components to the
WebSockets system.
This WebSockets code path had a lot of downsides/complexity,
including:
* The messy hack involving constructing an emulated request object to
hook into doing Django requests.
* The `message_senders` queue processor system, which increases RAM
needs and must be provisioned independently from the rest of the
server).
* A duplicate check_send_receive_time Nagios test specific to
WebSockets.
* The requirement for users to have their firewalls/NATs allow
WebSocket connections, and a setting to disable them for networks
where WebSockets don’t work.
* Dependencies on the SockJS family of libraries, which has at times
been poorly maintained, and periodically throws random JavaScript
exceptions in our production environments without a deep enough
traceback to effectively investigate.
* A total of about 1600 lines of our code related to the feature.
* Increased load on the Tornado system, especially around a Zulip
server restart, and especially for large installations like
zulipchat.com, resulting in extra delay before messages can be sent
again.
As detailed in
https://github.com/zulip/zulip/pull/12862#issuecomment-536152397, it
appears that removing WebSockets moderately increases the time it
takes for the `send_message` API query to return from the server, but
does not significantly change the time between when a message is sent
and when it is received by clients. We don’t understand the reason
for that change (suggesting the possibility of a measurement error),
and even if it is a real change, we consider that potential small
latency regression to be acceptable.
If we later want WebSockets, we’ll likely want to just use Django
Channels.
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2019-07-23 01:43:40 +02:00
|
|
|
from zerver.tornado.event_queue import process_event
|
2016-11-16 12:37:26 +01:00
|
|
|
|
2017-11-06 03:14:57 +01:00
|
|
|
import urllib.parse
|
2016-11-16 12:37:26 +01:00
|
|
|
|
dependencies: Remove WebSockets system for sending messages.
Zulip has had a small use of WebSockets (specifically, for the code
path of sending messages, via the webapp only) since ~2013. We
originally added this use of WebSockets in the hope that the latency
benefits of doing so would allow us to avoid implementing a markdown
local echo; they were not. Further, HTTP/2 may have eliminated the
latency difference we hoped to exploit by using WebSockets in any
case.
While we’d originally imagined using WebSockets for other endpoints,
there was never a good justification for moving more components to the
WebSockets system.
This WebSockets code path had a lot of downsides/complexity,
including:
* The messy hack involving constructing an emulated request object to
hook into doing Django requests.
* The `message_senders` queue processor system, which increases RAM
needs and must be provisioned independently from the rest of the
server).
* A duplicate check_send_receive_time Nagios test specific to
WebSockets.
* The requirement for users to have their firewalls/NATs allow
WebSocket connections, and a setting to disable them for networks
where WebSockets don’t work.
* Dependencies on the SockJS family of libraries, which has at times
been poorly maintained, and periodically throws random JavaScript
exceptions in our production environments without a deep enough
traceback to effectively investigate.
* A total of about 1600 lines of our code related to the feature.
* Increased load on the Tornado system, especially around a Zulip
server restart, and especially for large installations like
zulipchat.com, resulting in extra delay before messages can be sent
again.
As detailed in
https://github.com/zulip/zulip/pull/12862#issuecomment-536152397, it
appears that removing WebSockets moderately increases the time it
takes for the `send_message` API query to return from the server, but
does not significantly change the time between when a message is sent
and when it is received by clients. We don’t understand the reason
for that change (suggesting the possibility of a measurement error),
and even if it is a real change, we consider that potential small
latency regression to be acceptable.
If we later want WebSockets, we’ll likely want to just use Django
Channels.
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2019-07-23 01:43:40 +02:00
|
|
|
from typing import Any, Dict, Optional, List, cast
|
2016-11-16 12:37:26 +01:00
|
|
|
|
2017-05-25 11:50:13 +02:00
|
|
|
class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase):
|
2017-11-05 10:51:25 +01:00
|
|
|
def setUp(self) -> None:
|
2017-10-27 08:28:23 +02:00
|
|
|
super().setUp()
|
2017-05-25 11:50:13 +02:00
|
|
|
signals.request_started.disconnect(close_old_connections)
|
|
|
|
signals.request_finished.disconnect(close_old_connections)
|
2018-05-11 01:39:38 +02:00
|
|
|
self.session_cookie = None # type: Optional[Dict[str, str]]
|
2017-05-25 11:50:13 +02:00
|
|
|
|
2017-11-05 10:51:25 +01:00
|
|
|
def tearDown(self) -> None:
|
2017-10-27 08:28:23 +02:00
|
|
|
super().tearDown()
|
2019-07-22 22:18:25 +02:00
|
|
|
self.session_cookie = None
|
2017-05-25 11:50:13 +02:00
|
|
|
|
2017-10-28 01:17:38 +02:00
|
|
|
@override_settings(DEBUG=False)
|
2017-11-05 10:51:25 +01:00
|
|
|
def get_app(self) -> Application:
|
2018-11-03 00:09:16 +01:00
|
|
|
return create_tornado_application(9993)
|
2017-05-25 11:50:13 +02:00
|
|
|
|
2018-05-11 01:39:38 +02:00
|
|
|
def client_get(self, path: str, **kwargs: Any) -> HTTPResponse:
|
2017-05-25 11:50:13 +02:00
|
|
|
self.add_session_cookie(kwargs)
|
2020-02-25 02:53:12 +01:00
|
|
|
kwargs['skip_user_agent'] = True
|
2020-02-25 02:40:46 +01:00
|
|
|
self.set_http_headers(kwargs)
|
2017-08-26 00:40:17 +02:00
|
|
|
if 'HTTP_HOST' in kwargs:
|
|
|
|
kwargs['headers']['Host'] = kwargs['HTTP_HOST']
|
|
|
|
del kwargs['HTTP_HOST']
|
2017-05-25 11:50:13 +02:00
|
|
|
return self.fetch(path, method='GET', **kwargs)
|
|
|
|
|
2018-05-11 01:39:38 +02:00
|
|
|
def fetch_async(self, method: str, path: str, **kwargs: Any) -> None:
|
2017-05-25 11:50:13 +02:00
|
|
|
self.add_session_cookie(kwargs)
|
2020-02-25 02:53:12 +01:00
|
|
|
kwargs['skip_user_agent'] = True
|
2020-02-25 02:40:46 +01:00
|
|
|
self.set_http_headers(kwargs)
|
2017-08-26 00:40:17 +02:00
|
|
|
if 'HTTP_HOST' in kwargs:
|
|
|
|
kwargs['headers']['Host'] = kwargs['HTTP_HOST']
|
|
|
|
del kwargs['HTTP_HOST']
|
2017-05-25 11:50:13 +02:00
|
|
|
self.http_client.fetch(
|
|
|
|
self.get_url(path),
|
|
|
|
self.stop,
|
|
|
|
method=method,
|
|
|
|
**kwargs
|
|
|
|
)
|
|
|
|
|
2018-05-11 01:39:38 +02:00
|
|
|
def client_get_async(self, path: str, **kwargs: Any) -> None:
|
2020-02-25 02:53:12 +01:00
|
|
|
kwargs['skip_user_agent'] = True
|
2020-02-25 02:40:46 +01:00
|
|
|
self.set_http_headers(kwargs)
|
2017-05-25 11:50:13 +02:00
|
|
|
self.fetch_async('GET', path, **kwargs)
|
|
|
|
|
2020-03-06 18:40:46 +01:00
|
|
|
def login_user(self, *args: Any, **kwargs: Any) -> None:
|
|
|
|
super().login_user(*args, **kwargs)
|
2017-05-25 11:50:13 +02:00
|
|
|
session_cookie = settings.SESSION_COOKIE_NAME
|
|
|
|
session_key = self.client.session.session_key
|
|
|
|
self.session_cookie = {
|
|
|
|
"Cookie": "{}={}".format(session_cookie, session_key)
|
|
|
|
}
|
|
|
|
|
2018-05-11 01:39:38 +02:00
|
|
|
def get_session_cookie(self) -> Dict[str, str]:
|
2017-05-25 11:50:13 +02:00
|
|
|
return {} if self.session_cookie is None else self.session_cookie
|
|
|
|
|
2017-11-05 10:51:25 +01:00
|
|
|
def add_session_cookie(self, kwargs: Dict[str, Any]) -> None:
|
2017-05-25 11:50:13 +02:00
|
|
|
# TODO: Currently only allows session cookie
|
|
|
|
headers = kwargs.get('headers', {})
|
|
|
|
headers.update(self.get_session_cookie())
|
|
|
|
kwargs['headers'] = headers
|
|
|
|
|
2017-11-05 10:51:25 +01:00
|
|
|
def create_queue(self, **kwargs: Any) -> str:
|
2020-02-25 02:53:12 +01:00
|
|
|
response = self.client_get('/json/events?dont_block=true', subdomain="zulip",
|
|
|
|
skip_user_agent=True)
|
2017-05-25 11:50:13 +02:00
|
|
|
self.assertEqual(response.code, 200)
|
|
|
|
body = ujson.loads(response.body)
|
|
|
|
self.assertEqual(body['events'], [])
|
|
|
|
self.assertIn('queue_id', body)
|
|
|
|
return body['queue_id']
|
|
|
|
|
|
|
|
class EventsTestCase(TornadoWebTestCase):
|
2017-11-05 10:51:25 +01:00
|
|
|
def test_create_queue(self) -> None:
|
2020-03-06 18:40:46 +01:00
|
|
|
self.login_user(self.example_user('hamlet'))
|
2017-05-25 11:50:13 +02:00
|
|
|
queue_id = self.create_queue()
|
|
|
|
self.assertIn(queue_id, event_queue.clients)
|
|
|
|
|
2017-11-05 10:51:25 +01:00
|
|
|
def test_events_async(self) -> None:
|
2017-05-25 11:50:13 +02:00
|
|
|
user_profile = self.example_user('hamlet')
|
2020-03-06 18:40:46 +01:00
|
|
|
self.login_user(user_profile)
|
2017-05-25 11:50:13 +02:00
|
|
|
event_queue_id = self.create_queue()
|
|
|
|
data = {
|
|
|
|
'queue_id': event_queue_id,
|
2019-08-06 00:08:56 +02:00
|
|
|
'last_event_id': -1,
|
2017-05-25 11:50:13 +02:00
|
|
|
}
|
|
|
|
|
2017-11-06 03:14:57 +01:00
|
|
|
path = '/json/events?{}'.format(urllib.parse.urlencode(data))
|
2017-05-25 11:50:13 +02:00
|
|
|
self.client_get_async(path)
|
|
|
|
|
2017-11-05 10:51:25 +01:00
|
|
|
def process_events() -> None:
|
2017-05-25 11:50:13 +02:00
|
|
|
users = [user_profile.id]
|
|
|
|
event = dict(
|
|
|
|
type='test',
|
|
|
|
data='test data',
|
|
|
|
)
|
|
|
|
process_event(event, users)
|
|
|
|
|
|
|
|
self.io_loop.call_later(0.1, process_events)
|
|
|
|
response = self.wait()
|
|
|
|
data = ujson.loads(response.body)
|
|
|
|
events = data['events']
|
|
|
|
events = cast(List[Dict[str, Any]], events)
|
|
|
|
self.assertEqual(len(events), 1)
|
|
|
|
self.assertEqual(events[0]['data'], 'test data')
|
|
|
|
self.assertEqual(data['result'], 'success')
|