From 7acb642fa5df04249e2d072ad7f55c395807e805 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 17 Mar 2022 13:42:25 -0700 Subject: [PATCH] requirements: Upgrade to Tornado 6. Fixes #8913. Signed-off-by: Anders Kaseorg --- pyproject.toml | 1 - requirements/common.in | 2 +- requirements/dev.txt | 48 +++++++++++++++++++++--- requirements/prod.txt | 48 +++++++++++++++++++++--- tools/run-dev.py | 39 ++++++++----------- version.py | 2 +- zerver/lib/queue.py | 4 +- zerver/management/commands/runtornado.py | 6 +-- zerver/tests/test_tornado.py | 10 ++--- zerver/tornado/event_queue.py | 10 ++--- zerver/tornado/handlers.py | 13 ++----- 11 files changed, 115 insertions(+), 68 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11106a80d3..40e7d3a986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,6 @@ module = [ "sourcemap.*", "talon_core.*", "tlds.*", - "tornado.*", "twitter.*", "two_factor.*", "zxcvbn.*", diff --git a/requirements/common.in b/requirements/common.in index 6db6d0ff5d..89b79cea3d 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -84,7 +84,7 @@ redis sourcemap # Tornado used for server->client push system -tornado==4.* # https://github.com/zulip/zulip/issues/8913 +tornado # Fast JSON parser orjson diff --git a/requirements/dev.txt b/requirements/dev.txt index 9959b19898..11db9d7beb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1929,12 +1929,48 @@ tomli==2.0.1 \ # black # mypy # pep517 -tornado==4.5.3 \ - --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \ - --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \ - --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \ - --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \ - --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a +tornado==6.1 \ + --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \ + --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \ + --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \ + --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \ + --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \ + --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \ + --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \ + --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \ + --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \ + --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \ + --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \ + --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \ + --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \ + --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \ + --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \ + --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \ + --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \ + --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \ + --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \ + --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \ + --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \ + --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \ + --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \ + --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \ + --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \ + --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \ + --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \ + --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \ + --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \ + --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \ + --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \ + --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \ + --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \ + --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \ + --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \ + --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \ + --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \ + --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \ + --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \ + --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \ + --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5 # via # -r requirements/common.in # snakeviz diff --git a/requirements/prod.txt b/requirements/prod.txt index c3623aa95d..f8eecaed14 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1281,12 +1281,48 @@ tlds==2022032300 \ --hash=sha256:40ad49151d06de78c647101c6c7c79b68bf33651bbdd15f134eab46b5f458037 \ --hash=sha256:ecc7d864bfb3eb4e6d03ceba545e25b50483877f86e6d5b757ae51923260a954 # via -r requirements/common.in -tornado==4.5.3 \ - --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \ - --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \ - --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \ - --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \ - --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a +tornado==6.1 \ + --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \ + --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \ + --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \ + --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \ + --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \ + --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \ + --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \ + --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \ + --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \ + --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \ + --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \ + --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \ + --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \ + --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \ + --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \ + --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \ + --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \ + --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \ + --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \ + --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \ + --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \ + --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \ + --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \ + --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \ + --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \ + --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \ + --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \ + --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \ + --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \ + --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \ + --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \ + --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \ + --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \ + --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \ + --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \ + --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \ + --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \ + --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \ + --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \ + --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \ + --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5 # via -r requirements/common.in traitlets==5.1.1 \ --hash=sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7 \ diff --git a/tools/run-dev.py b/tools/run-dev.py index a313680cb0..068f7f7e78 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -238,22 +238,6 @@ class BaseHandler(web.RequestHandler): def delete(self) -> None: pass - def handle_response(self, response: httpclient.HTTPResponse) -> None: - if response.error and not isinstance(response.error, httpclient.HTTPError): - self.set_status(500) - self.write("Internal server error:\n" + str(response.error)) - else: - self.set_status(response.code, response.reason) - self._headers = httputil.HTTPHeaders() # clear tornado default header - - for header, v in response.headers.get_all(): - # some header appear multiple times, eg 'Set-Cookie' - if header.lower() != "transfer-encoding": - self.add_header(header, v) - if response.body: - self.write(response.body) - self.finish() - async def prepare(self) -> None: assert self.request.method is not None assert self.request.remote_ip is not None @@ -283,14 +267,21 @@ class BaseHandler(web.RequestHandler): decompress_response=False, ) response = await client.fetch(request, raise_error=False) - self.handle_response(response) - except httpclient.HTTPError as e: - if hasattr(e, "response") and e.response: - self.handle_response(e.response) - else: - self.set_status(500) - self.write("Internal server error:\n" + str(e)) - self.finish() + + self.set_status(response.code, response.reason) + self._headers = httputil.HTTPHeaders() # clear tornado default header + + for header, v in response.headers.get_all(): + # some header appear multiple times, eg 'Set-Cookie' + if header.lower() != "transfer-encoding": + self.add_header(header, v) + if response.body: + self.write(response.body) + self.finish() + except (ConnectionError, httpclient.HTTPError) as e: + self.set_status(500) + self.write("Internal server error:\n" + str(e)) + self.finish() class WebPackHandler(BaseHandler): diff --git a/version.py b/version.py index 42e5435656..750a9a06c4 100644 --- a/version.py +++ b/version.py @@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 128 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = "185.2" +PROVISION_VERSION = "186.0" diff --git a/zerver/lib/queue.py b/zerver/lib/queue.py index 7ac4ee00c7..b30ae73ade 100644 --- a/zerver/lib/queue.py +++ b/zerver/lib/queue.py @@ -302,7 +302,7 @@ class TornadoQueueClient(QueueClient[Channel]): "TornadoQueueClient couldn't connect to RabbitMQ, retrying in %d secs...", retry_secs, ) - ioloop.IOLoop.instance().call_later(retry_secs, self._reconnect) + ioloop.IOLoop.current().call_later(retry_secs, self._reconnect) def _on_connection_closed( self, connection: pika.connection.Connection, reason: Exception @@ -315,7 +315,7 @@ class TornadoQueueClient(QueueClient[Channel]): "TornadoQueueClient lost connection to RabbitMQ, reconnecting in %d secs...", retry_secs, ) - ioloop.IOLoop.instance().call_later(retry_secs, self._reconnect) + ioloop.IOLoop.current().call_later(retry_secs, self._reconnect) def _on_open(self, connection: pika.connection.Connection) -> None: assert self.connection is not None diff --git a/zerver/management/commands/runtornado.py b/zerver/management/commands/runtornado.py index 15a927fd22..a0339a2f53 100644 --- a/zerver/management/commands/runtornado.py +++ b/zerver/management/commands/runtornado.py @@ -10,7 +10,7 @@ from asgiref.sync import async_to_sync, sync_to_async from django.conf import settings from django.core.management.base import BaseCommand, CommandError, CommandParser from tornado import autoreload -from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future +from tornado.platform.asyncio import AsyncIOMainLoop settings.RUNNING_INSIDE_TORNADO = True @@ -111,9 +111,7 @@ class Command(BaseCommand): # start tornado web server in single-threaded mode http_server = httpserver.HTTPServer(application, xheaders=True) - stack.push_async_callback( - lambda: to_asyncio_future(http_server.close_all_connections()) - ) + stack.push_async_callback(http_server.close_all_connections) stack.callback(http_server.stop) http_server.listen(port, address=addr) diff --git a/zerver/tests/test_tornado.py b/zerver/tests/test_tornado.py index 78f5c0fda5..f7bf223f60 100644 --- a/zerver/tests/test_tornado.py +++ b/zerver/tests/test_tornado.py @@ -12,7 +12,7 @@ from django.db import close_old_connections from django.test import override_settings from tornado.httpclient import HTTPResponse from tornado.ioloop import IOLoop -from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future +from tornado.platform.asyncio import AsyncIOMainLoop from tornado.testing import AsyncHTTPTestCase, AsyncTestCase from tornado.web import Application from typing_extensions import ParamSpec @@ -71,9 +71,7 @@ class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): if "HTTP_HOST" in kwargs: kwargs["headers"]["Host"] = kwargs["HTTP_HOST"] del kwargs["HTTP_HOST"] - return await to_asyncio_future( - self.http_client.fetch(self.get_url(path), method="GET", **kwargs) - ) + return await self.http_client.fetch(self.get_url(path), method="GET", **kwargs) async def fetch_async(self, method: str, path: str, **kwargs: Any) -> HTTPResponse: self.add_session_cookie(kwargs) @@ -82,9 +80,7 @@ class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): if "HTTP_HOST" in kwargs: kwargs["headers"]["Host"] = kwargs["HTTP_HOST"] del kwargs["HTTP_HOST"] - return await to_asyncio_future( - self.http_client.fetch(self.get_url(path), method=method, **kwargs) - ) + return await self.http_client.fetch(self.get_url(path), method=method, **kwargs) async def client_get_async(self, path: str, **kwargs: Any) -> HTTPResponse: kwargs["skip_user_agent"] = True diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py index 714233c430..32e885f9b6 100644 --- a/zerver/tornado/event_queue.py +++ b/zerver/tornado/event_queue.py @@ -246,7 +246,7 @@ class ClientDescriptor: heartbeat_event = create_heartbeat_event() self.add_event(heartbeat_event) - ioloop = tornado.ioloop.IOLoop.instance() + ioloop = tornado.ioloop.IOLoop.current() interval = HEARTBEAT_MIN_FREQ_SECS + random.randint(0, 10) if self.client_type_name != "API: heartbeat test": self._timeout_handle = ioloop.call_later(interval, timeout_callback) @@ -265,7 +265,7 @@ class ClientDescriptor: self.current_handler_id = None self.current_client_name = None if self._timeout_handle is not None: - ioloop = tornado.ioloop.IOLoop.instance() + ioloop = tornado.ioloop.IOLoop.current() ioloop.remove_timeout(self._timeout_handle) self._timeout_handle = None @@ -601,8 +601,6 @@ def send_restart_events(immediate: bool = False) -> None: async def setup_event_queue(server: tornado.httpserver.HTTPServer, port: int) -> None: - ioloop = tornado.ioloop.IOLoop.instance() - if not settings.TEST_SUITE: load_event_queues(port) autoreload.add_reload_hook(lambda: dump_event_queues(port)) @@ -613,9 +611,7 @@ async def setup_event_queue(server: tornado.httpserver.HTTPServer, port: int) -> pass # Set up event queue garbage collection - pc = tornado.ioloop.PeriodicCallback( - lambda: gc_event_queues(port), EVENT_QUEUE_GC_FREQ_MSECS, ioloop - ) + pc = tornado.ioloop.PeriodicCallback(lambda: gc_event_queues(port), EVENT_QUEUE_GC_FREQ_MSECS) pc.start() send_restart_events(immediate=settings.DEVELOPMENT) diff --git a/zerver/tornado/handlers.py b/zerver/tornado/handlers.py index 146708f664..aaed2ad610 100644 --- a/zerver/tornado/handlers.py +++ b/zerver/tornado/handlers.py @@ -1,4 +1,3 @@ -import asyncio import logging import urllib import weakref @@ -151,9 +150,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler): async def get(self, *args: Any, **kwargs: Any) -> None: request = self.convert_tornado_request_to_django_request() - response = await asyncio.ensure_future( - sync_to_async(lambda: self.get_response(request), thread_sensitive=True)() - ) + response = await sync_to_async(lambda: self.get_response(request), thread_sensitive=True)() try: if hasattr(response, "asynchronous"): @@ -184,7 +181,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler): # the Django side; this triggers cleanup work like # resetting the urlconf and any cache/database # connections. - await asyncio.ensure_future(sync_to_async(response.close, thread_sensitive=True)()) + await sync_to_async(response.close, thread_sensitive=True)() async def head(self, *args: Any, **kwargs: Any) -> None: await self.get(*args, **kwargs) @@ -262,9 +259,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler): res_type=result_dict["result"], data=result_dict, status=self.get_status() ) - response = await asyncio.ensure_future( - sync_to_async(lambda: self.get_response(request), thread_sensitive=True)() - ) + response = await sync_to_async(lambda: self.get_response(request), thread_sensitive=True)() try: # Explicitly mark requests as varying by cookie, since the # middleware will not have seen a session access @@ -273,4 +268,4 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler): self.write_django_response_as_tornado_response(response) finally: # Tell Django we're done processing this request - await asyncio.ensure_future(sync_to_async(response.close, thread_sensitive=True)()) + await sync_to_async(response.close, thread_sensitive=True)()