requirements: Upgrade to Tornado 6.

Fixes #8913.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

pyproject.toml

@@ -93,7 +93,6 @@ module = [
     "sourcemap.*",
     "talon_core.*",
     "tlds.*",
-    "tornado.*",
     "twitter.*",
     "two_factor.*",
     "zxcvbn.*",

requirements/common.in

@@ -84,7 +84,7 @@ redis
 sourcemap
 
 # Tornado used for server->client push system
-tornado==4.*  # https://github.com/zulip/zulip/issues/8913
+tornado
 
 # Fast JSON parser
 orjson

requirements/dev.txt

@@ -1929,12 +1929,48 @@ tomli==2.0.1 \
     #   black
     #   mypy
     #   pep517
-tornado==4.5.3 \
+tornado==6.1 \
-    --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \
+    --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \
-    --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \
+    --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \
-    --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \
+    --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \
-    --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \
+    --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \
-    --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a
+    --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \
+    --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \
+    --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \
+    --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \
+    --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \
+    --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \
+    --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \
+    --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \
+    --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \
+    --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \
+    --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \
+    --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \
+    --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \
+    --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \
+    --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \
+    --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \
+    --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \
+    --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \
+    --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \
+    --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \
+    --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \
+    --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \
+    --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \
+    --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \
+    --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \
+    --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \
+    --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \
+    --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \
+    --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \
+    --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \
+    --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \
+    --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \
+    --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \
+    --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \
+    --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \
+    --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \
+    --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5
     # via
     #   -r requirements/common.in
     #   snakeviz

requirements/prod.txt

@@ -1281,12 +1281,48 @@ tlds==2022032300 \
     --hash=sha256:40ad49151d06de78c647101c6c7c79b68bf33651bbdd15f134eab46b5f458037 \
     --hash=sha256:ecc7d864bfb3eb4e6d03ceba545e25b50483877f86e6d5b757ae51923260a954
     # via -r requirements/common.in
-tornado==4.5.3 \
+tornado==6.1 \
-    --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \
+    --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \
-    --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \
+    --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \
-    --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \
+    --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \
-    --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \
+    --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \
-    --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a
+    --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \
+    --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \
+    --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \
+    --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \
+    --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \
+    --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \
+    --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \
+    --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \
+    --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \
+    --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \
+    --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \
+    --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \
+    --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \
+    --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \
+    --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \
+    --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \
+    --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \
+    --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \
+    --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \
+    --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \
+    --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \
+    --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \
+    --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \
+    --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \
+    --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \
+    --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \
+    --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \
+    --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \
+    --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \
+    --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \
+    --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \
+    --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \
+    --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \
+    --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \
+    --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \
+    --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \
+    --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5
     # via -r requirements/common.in
 traitlets==5.1.1 \
     --hash=sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7 \

tools/run-dev.py

@@ -238,22 +238,6 @@ class BaseHandler(web.RequestHandler):
     def delete(self) -> None:
         pass
 
-    def handle_response(self, response: httpclient.HTTPResponse) -> None:
-        if response.error and not isinstance(response.error, httpclient.HTTPError):
-            self.set_status(500)
-            self.write("Internal server error:\n" + str(response.error))
-        else:
-            self.set_status(response.code, response.reason)
-            self._headers = httputil.HTTPHeaders()  # clear tornado default header
-
-            for header, v in response.headers.get_all():
-                # some header appear multiple times, eg 'Set-Cookie'
-                if header.lower() != "transfer-encoding":
-                    self.add_header(header, v)
-            if response.body:
-                self.write(response.body)
-        self.finish()
-
     async def prepare(self) -> None:
         assert self.request.method is not None
         assert self.request.remote_ip is not None
@@ -283,14 +267,21 @@ class BaseHandler(web.RequestHandler):
                 decompress_response=False,
             )
             response = await client.fetch(request, raise_error=False)
-            self.handle_response(response)
+
-        except httpclient.HTTPError as e:
+            self.set_status(response.code, response.reason)
-            if hasattr(e, "response") and e.response:
+            self._headers = httputil.HTTPHeaders()  # clear tornado default header
-                self.handle_response(e.response)
+
-            else:
+            for header, v in response.headers.get_all():
-                self.set_status(500)
+                # some header appear multiple times, eg 'Set-Cookie'
-                self.write("Internal server error:\n" + str(e))
+                if header.lower() != "transfer-encoding":
-                self.finish()
+                    self.add_header(header, v)
+            if response.body:
+                self.write(response.body)
+            self.finish()
+        except (ConnectionError, httpclient.HTTPError) as e:
+            self.set_status(500)
+            self.write("Internal server error:\n" + str(e))
+            self.finish()
 
 
 class WebPackHandler(BaseHandler):

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 128
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "185.2"
+PROVISION_VERSION = "186.0"

zerver/lib/queue.py

@@ -302,7 +302,7 @@ class TornadoQueueClient(QueueClient[Channel]):
             "TornadoQueueClient couldn't connect to RabbitMQ, retrying in %d secs...",
             retry_secs,
         )
-        ioloop.IOLoop.instance().call_later(retry_secs, self._reconnect)
+        ioloop.IOLoop.current().call_later(retry_secs, self._reconnect)
 
     def _on_connection_closed(
         self, connection: pika.connection.Connection, reason: Exception
@@ -315,7 +315,7 @@ class TornadoQueueClient(QueueClient[Channel]):
             "TornadoQueueClient lost connection to RabbitMQ, reconnecting in %d secs...",
             retry_secs,
         )
-        ioloop.IOLoop.instance().call_later(retry_secs, self._reconnect)
+        ioloop.IOLoop.current().call_later(retry_secs, self._reconnect)
 
     def _on_open(self, connection: pika.connection.Connection) -> None:
         assert self.connection is not None

zerver/management/commands/runtornado.py

@@ -10,7 +10,7 @@ from asgiref.sync import async_to_sync, sync_to_async
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError, CommandParser
 from tornado import autoreload
-from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future
+from tornado.platform.asyncio import AsyncIOMainLoop
 
 settings.RUNNING_INSIDE_TORNADO = True
 
@@ -111,9 +111,7 @@ class Command(BaseCommand):
 
                 # start tornado web server in single-threaded mode
                 http_server = httpserver.HTTPServer(application, xheaders=True)
-                stack.push_async_callback(
+                stack.push_async_callback(http_server.close_all_connections)
-                    lambda: to_asyncio_future(http_server.close_all_connections())
-                )
                 stack.callback(http_server.stop)
                 http_server.listen(port, address=addr)
 

zerver/tests/test_tornado.py

@@ -12,7 +12,7 @@ from django.db import close_old_connections
 from django.test import override_settings
 from tornado.httpclient import HTTPResponse
 from tornado.ioloop import IOLoop
-from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future
+from tornado.platform.asyncio import AsyncIOMainLoop
 from tornado.testing import AsyncHTTPTestCase, AsyncTestCase
 from tornado.web import Application
 from typing_extensions import ParamSpec
@@ -71,9 +71,7 @@ class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase):
         if "HTTP_HOST" in kwargs:
             kwargs["headers"]["Host"] = kwargs["HTTP_HOST"]
             del kwargs["HTTP_HOST"]
-        return await to_asyncio_future(
+        return await self.http_client.fetch(self.get_url(path), method="GET", **kwargs)
-            self.http_client.fetch(self.get_url(path), method="GET", **kwargs)
-        )
 
     async def fetch_async(self, method: str, path: str, **kwargs: Any) -> HTTPResponse:
         self.add_session_cookie(kwargs)
@@ -82,9 +80,7 @@ class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase):
         if "HTTP_HOST" in kwargs:
             kwargs["headers"]["Host"] = kwargs["HTTP_HOST"]
             del kwargs["HTTP_HOST"]
-        return await to_asyncio_future(
+        return await self.http_client.fetch(self.get_url(path), method=method, **kwargs)
-            self.http_client.fetch(self.get_url(path), method=method, **kwargs)
-        )
 
     async def client_get_async(self, path: str, **kwargs: Any) -> HTTPResponse:
         kwargs["skip_user_agent"] = True

zerver/tornado/event_queue.py

@@ -246,7 +246,7 @@ class ClientDescriptor:
             heartbeat_event = create_heartbeat_event()
             self.add_event(heartbeat_event)
 
-        ioloop = tornado.ioloop.IOLoop.instance()
+        ioloop = tornado.ioloop.IOLoop.current()
         interval = HEARTBEAT_MIN_FREQ_SECS + random.randint(0, 10)
         if self.client_type_name != "API: heartbeat test":
             self._timeout_handle = ioloop.call_later(interval, timeout_callback)
@@ -265,7 +265,7 @@ class ClientDescriptor:
         self.current_handler_id = None
         self.current_client_name = None
         if self._timeout_handle is not None:
-            ioloop = tornado.ioloop.IOLoop.instance()
+            ioloop = tornado.ioloop.IOLoop.current()
             ioloop.remove_timeout(self._timeout_handle)
             self._timeout_handle = None
 
@@ -601,8 +601,6 @@ def send_restart_events(immediate: bool = False) -> None:
 
 
 async def setup_event_queue(server: tornado.httpserver.HTTPServer, port: int) -> None:
-    ioloop = tornado.ioloop.IOLoop.instance()
-
     if not settings.TEST_SUITE:
         load_event_queues(port)
         autoreload.add_reload_hook(lambda: dump_event_queues(port))
@@ -613,9 +611,7 @@ async def setup_event_queue(server: tornado.httpserver.HTTPServer, port: int) ->
         pass
 
     # Set up event queue garbage collection
-    pc = tornado.ioloop.PeriodicCallback(
+    pc = tornado.ioloop.PeriodicCallback(lambda: gc_event_queues(port), EVENT_QUEUE_GC_FREQ_MSECS)
-        lambda: gc_event_queues(port), EVENT_QUEUE_GC_FREQ_MSECS, ioloop
-    )
     pc.start()
 
     send_restart_events(immediate=settings.DEVELOPMENT)

zerver/tornado/handlers.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 import urllib
 import weakref
@@ -151,9 +150,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
 
     async def get(self, *args: Any, **kwargs: Any) -> None:
         request = self.convert_tornado_request_to_django_request()
-        response = await asyncio.ensure_future(
+        response = await sync_to_async(lambda: self.get_response(request), thread_sensitive=True)()
-            sync_to_async(lambda: self.get_response(request), thread_sensitive=True)()
-        )
 
         try:
             if hasattr(response, "asynchronous"):
@@ -184,7 +181,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
             # the Django side; this triggers cleanup work like
             # resetting the urlconf and any cache/database
             # connections.
-            await asyncio.ensure_future(sync_to_async(response.close, thread_sensitive=True)())
+            await sync_to_async(response.close, thread_sensitive=True)()
 
     async def head(self, *args: Any, **kwargs: Any) -> None:
         await self.get(*args, **kwargs)
@@ -262,9 +259,7 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
             res_type=result_dict["result"], data=result_dict, status=self.get_status()
         )
 
-        response = await asyncio.ensure_future(
+        response = await sync_to_async(lambda: self.get_response(request), thread_sensitive=True)()
-            sync_to_async(lambda: self.get_response(request), thread_sensitive=True)()
-        )
         try:
             # Explicitly mark requests as varying by cookie, since the
             # middleware will not have seen a session access
@@ -273,4 +268,4 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
             self.write_django_response_as_tornado_response(response)
         finally:
             # Tell Django we're done processing this request
-            await asyncio.ensure_future(sync_to_async(response.close, thread_sensitive=True)())
+            await sync_to_async(response.close, thread_sensitive=True)()