From 09e17fbe17fecb3b83ef80a0e04248e7387f328f Mon Sep 17 00:00:00 2001 From: "K.Kanakhin" Date: Wed, 12 Oct 2016 19:09:32 +0600 Subject: [PATCH] run-dev: Use tornado for dev proxy server for HTTP requests. - Use tornado as proxy server for development environment, replacing twisted (this doesn't support websockets). - Upgrade tornado version to 4.4.1 (needs to be coupled to the above since neither change works without the other) --- requirements/common.txt | 5 +- tools/run-dev.py | 222 ++++++++++++++++++++++++++++++++-------- version.py | 2 +- 3 files changed, 183 insertions(+), 46 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index 86e3997c45..7986d4e38c 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -21,6 +21,9 @@ SQLAlchemy==1.1.1 # Needed for Tornado >3.2 compatibility backports-abc==0.4 +# Needed for Tornado 4 compatibility +backports.ssl-match-hostname==3.5.0.1 + # Needed for S3 file uploads boto==2.42.0 @@ -127,7 +130,7 @@ sourcemap==0.1.8 statsd==3.2.1 # Tornado used for server->client push system -tornado==3.2.2 +tornado==4.4.2 # Needed for Python static typing typing==3.5.2.2 diff --git a/tools/run-dev.py b/tools/run-dev.py index 346852062b..5a6de95b7c 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -2,27 +2,23 @@ from __future__ import print_function import optparse -import pwd -import subprocess -import signal -import traceback -import sys import os +import pwd +import signal +import subprocess +import sys +import time +import traceback -if False: from typing import Any +from six.moves.urllib.parse import urlunparse -from twisted.internet import reactor -from twisted.web import proxy, server, resource +from tornado import httpclient +from tornado import httputil +from tornado import gen +from tornado import web +from tornado.ioloop import IOLoop -# Monkey-patch twisted.web.http to avoid request.finish exceptions -# https://trac.zulip.net/ticket/1728 -from twisted.web.http import Request -orig_finish = Request.finish -def patched_finish(self): - # type: (Any) -> None - if not self._disconnected: - orig_finish(self) -Request.finish = patched_finish +if False: from typing import Any, Callable, Generator, Optional if 'posix' in os.name and os.geteuid() == 0: raise RuntimeError("run-dev.py should not be run as root.") @@ -53,7 +49,7 @@ parser.add_option('--no-clear-memcached', action='store_false', dest='clear_memcached', default=True, help='Do not clear memcached') -(options, args) = parser.parse_args() +(options, arguments) = parser.parse_args() if options.interface is None: user_id = os.getuid() @@ -113,7 +109,7 @@ cmds = [['./tools/compile-handlebars-templates', 'forever'], ['python', '-u', 'manage.py', 'runtornado'] + manage_args + ['127.0.0.1:%d' % (tornado_port,)], ['./tools/run-dev-queue-processors'] + manage_args, - ['env', 'PGHOST=127.0.0.1', # Force password authentication using .pgpass + ['env', 'PGHOST=127.0.0.1', # Force password authentication using .pgpass './puppet/zulip/files/postgresql/process_fts_updates']] if options.test: # Webpack doesn't support 2 copies running on the same system, so @@ -126,38 +122,176 @@ else: for cmd in cmds: subprocess.Popen(cmd) -class Resource(resource.Resource): - def getChild(self, name, request): - # type: (bytes, server.Request) -> resource.Resource - # Assume an HTTP 1.1 request - proxy_host = request.requestHeaders.getRawHeaders('Host') - request.requestHeaders.setRawHeaders('X-Forwarded-Host', proxy_host) - if (request.uri.startswith(b'/json/events') or - request.uri.startswith(b'/api/v1/events') or - request.uri.startswith(b'/sockjs')): - return proxy.ReverseProxyResource('127.0.0.1', tornado_port, b'/' + name) - - elif (request.uri.startswith(b'/webpack') or - request.uri.startswith(b'/socket.io')): - return proxy.ReverseProxyResource('127.0.0.1', webpack_port, b'/' + name) - - return proxy.ReverseProxyResource('127.0.0.1', django_port, b'/'+name) +def transform_url(protocol, path, query, target_port, target_host): + # type: (str, str, str, int, str) -> str + # generate url with target host + host = ":".join((target_host, str(target_port))) + newpath = urlunparse((protocol, host, path, '', query, '')) + return newpath - # log which services/ports will be started - print("Starting Zulip services on ports: web proxy: {},".format(proxy_port), - "Django: {}, Tornado: {}".format(django_port, tornado_port), end='') - if options.test: - print("") # no webpack for --test +@gen.engine +def fetch_request(url, callback, **kwargs): + # type: (str, Any, **Any) -> Generator[Callable[..., Any], Any, None] + # use large timeouts to handle polling requests + req = httpclient.HTTPRequest(url, connect_timeout=240.0, request_timeout=240.0, **kwargs) + client = httpclient.AsyncHTTPClient() + # wait for response + response = yield gen.Task(client.fetch, req) + callback(response) + + +class BaseHandler(web.RequestHandler): + # target server ip + target_host = '127.0.0.1' # type: str + # target server port + target_port = None # type: int + + def _add_request_headers(self, exclude_lower_headers_list=None): + # type: (Optional[List[str]]) -> httputil.HTTPHeaders + exclude_lower_headers_list = exclude_lower_headers_list or [] + headers = httputil.HTTPHeaders() + for header, v in self.request.headers.get_all(): + if header.lower() not in exclude_lower_headers_list: + headers.add(header, v) + return headers + + def get(self): + # type: () -> None + pass + + def head(self): + # type: () -> None + pass + + def post(self): + # type: () -> None + pass + + def put(self): + # type: () -> None + pass + + def patch(self): + # type: () -> None + pass + + def options(self): + # type: () -> None + pass + + def delete(self): + # type: () -> None + pass + + def handle_response(self, response): + # type: (Any) -> None + if response.error and not isinstance(response.error, httpclient.HTTPError): + self.set_status(500) + self.write('Internal server error:\n' + str(response.error)) + else: + self.set_status(response.code, response.reason) + self._headers = httputil.HTTPHeaders() # clear tornado default header + + for header, v in response.headers.get_all(): + if header != 'Content-Length': + # some header appear multiple times, eg 'Set-Cookie' + self.add_header(header, v) + if response.body: + # rewrite Content-Length Header by the response + self.set_header('Content-Length', len(response.body)) + self.write(response.body) + self.finish() + + @web.asynchronous + def prepare(self): + # type: () -> None + url = transform_url( + self.request.protocol, + self.request.path, + self.request.query, + self.target_port, + self.target_host, + ) + try: + fetch_request( + url=url, + callback=self.handle_response, + method=self.request.method, + headers=self._add_request_headers(["upgrade-insecure-requests"]), + follow_redirects=False, + body=getattr(self.request, 'body'), + allow_nonstandard_methods=True + ) + except httpclient.HTTPError as e: + if hasattr(e, 'response') and e.response: + self.handle_response(e.response) + else: + self.set_status(500) + self.write('Internal server error:\n' + str(e)) + self.finish() + + +class WebPackHandler(BaseHandler): + target_port = webpack_port + + +class DjangoHandler(BaseHandler): + target_port = django_port + + +class TornadoHandler(BaseHandler): + target_port = tornado_port + + +class Application(web.Application): + def __init__(self): + # type: () -> None + handlers = [ + (r"/sockjs/.*/websocket$", TornadoHandler), + (r"/json/events.*", TornadoHandler), + (r"/api/v1/events.*", TornadoHandler), + (r"/webpack.*", WebPackHandler), + (r"/sockjs.*", TornadoHandler), + (r"/socket.io.*", WebPackHandler), + (r"/.*", DjangoHandler) + ] + super(Application, self).__init__(handlers) + + +def on_shutdown(): + # type: () -> None + IOLoop.instance().stop() + + +def shutdown_handler(*args, **kwargs): + # type: (*Any, **Any) -> None + io_loop = IOLoop.instance() + if io_loop._callbacks: + io_loop.add_timeout(time.time() + 1, shutdown_handler) else: - print(", webpack: {}".format(webpack_port)) + io_loop.stop() - print(WARNING + "Note: only port {} is exposed to the host in a Vagrant environment.".format(proxy_port) + ENDC) +# log which services/ports will be started +print("Starting Zulip services on ports: web proxy: {},".format(proxy_port), + "Django: {}, Tornado: {}".format(django_port, tornado_port), end='') +if options.test: + print("") # no webpack for --test +else: + print(", webpack: {}".format(webpack_port)) + +print("".join((WARNING, + "Note: only port {} is exposed to the host in a Vagrant environment.".format( + proxy_port), ENDC))) try: - reactor.listenTCP(proxy_port, server.Site(Resource()), interface=options.interface) - reactor.run() + app = Application() + app.listen(proxy_port) + ioloop = IOLoop.instance() + for s in (signal.SIGINT, signal.SIGTERM): + signal.signal(s, shutdown_handler) + ioloop.start() except: # Print the traceback before we get SIGTERM and die. traceback.print_exc() diff --git a/version.py b/version.py index 7236919a5e..06811990b9 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.4.1+git" -PROVISION_VERSION = '1.00' +PROVISION_VERSION = '2.0.0'