Django 1.10: Use uWSGI.

Fixes: #1121

With some tweaks by tabbott to make the number of processes configurable.

docs/architecture-overview.md

@@ -139,9 +139,9 @@ from outside.
     nginx gets the hostname for the Tornado server via
     `puppet/zulip/files/nginx/zulip-include-frontend/upstreams`.
 -   Requests to all other paths are sent to the Django app via the UNIX
-    socket `unix:/home/zulip/deployments/fastcgi-socket` (defined in
+    socket `unix:/home/zulip/deployments/uwsgi-socket` (defined in
     `puppet/zulip/files/nginx/zulip-include-frontend/upstreams`). We use
-    `zproject/wsgi.py` to implement FastCGI here (see
+    `zproject/wsgi.py` to implement uWSGI here (see
     `django.core.wsgi`).
 
 ### Supervisor

puppet/zulip/files/nginx/fastcgi_params

@@ -1,28 +0,0 @@
-fastcgi_param	QUERY_STRING		$query_string;
-fastcgi_param	REQUEST_METHOD		$request_method;
-fastcgi_param	CONTENT_TYPE		$content_type;
-fastcgi_param	CONTENT_LENGTH		$content_length;
-
-fastcgi_param	SCRIPT_FILENAME		$request_filename;
-fastcgi_param	SCRIPT_NAME		$fastcgi_script_name;
-fastcgi_param	REQUEST_URI		$uri;
-fastcgi_param	DOCUMENT_URI		$document_uri;
-fastcgi_param	DOCUMENT_ROOT		$document_root;
-fastcgi_param	SERVER_PROTOCOL		$server_protocol;
-
-fastcgi_param	GATEWAY_INTERFACE	CGI/1.1;
-fastcgi_param	SERVER_SOFTWARE		nginx/$nginx_version;
-
-fastcgi_param	REMOTE_ADDR		$remote_addr;
-fastcgi_param	REMOTE_PORT		$remote_port;
-fastcgi_param	SERVER_ADDR		$server_addr;
-fastcgi_param	SERVER_PORT		$server_port;
-fastcgi_param	SERVER_NAME		$server_name;
-
-fastcgi_param	HTTPS			$https;
-
-# PHP only, required if PHP was built with --enable-force-cgi-redirect
-fastcgi_param	REDIRECT_STATUS		200;
-
-# Ensure Zulip is safe from httpoxy security issue
-fastcgi_param HTTP_PROXY "";

puppet/zulip/files/nginx/uwsgi_params

@@ -0,0 +1,15 @@
+uwsgi_param QUERY_STRING    $query_string;
+uwsgi_param REQUEST_METHOD  $request_method;
+uwsgi_param CONTENT_TYPE    $content_type;
+uwsgi_param CONTENT_LENGTH  $content_length;
+uwsgi_param REQUEST_URI     $request_uri;
+uwsgi_param PATH_INFO       $document_uri;
+uwsgi_param DOCUMENT_ROOT   $document_root;
+uwsgi_param SERVER_PROTOCOL $server_protocol;
+uwsgi_param REQUEST_SCHEME  $scheme;
+uwsgi_param HTTPS           $https if_not_empty;
+uwsgi_param REMOTE_ADDR     $remote_addr;
+uwsgi_param REMOTE_PORT     $remote_port;
+uwsgi_param SERVER_ADDR     $server_addr;
+uwsgi_param SERVER_PORT     $server_port;
+uwsgi_param SERVER_NAME     $server_name;

puppet/zulip/files/nginx/zulip-include-frontend/app

@@ -50,15 +50,10 @@ location /sockjs {
     include /etc/nginx/zulip-include/location-sockjs;
 }
 
-# Send everything else to Django via FastCGI
+# Send everything else to Django via uWSGI
 location / {
-    include fastcgi_params;
-    fastcgi_pass django;
-    fastcgi_split_path_info ^()(.*)$;
-    # Second number set to `getconf PAGESIZE`
-    fastcgi_buffers 1024 4k;
-    fastcgi_max_temp_file_size 0;
-    fastcgi_next_upstream off;
+    include uwsgi_params;
+    uwsgi_pass django;
 }
 
 location /api/ {
@@ -66,13 +61,8 @@ location /api/ {
     add_header Access-Control-Allow-Headers Authorization;
     add_header Access-Control-Allow-Methods 'GET, POST';
 
-    include fastcgi_params;
-    fastcgi_pass django;
-    fastcgi_split_path_info ^()(.*)$;
-    # Second number set to `getconf PAGESIZE`
-    fastcgi_buffers 1024 4k;
-    fastcgi_max_temp_file_size 0;
-    fastcgi_next_upstream off;
+    include uwsgi_params;
+    uwsgi_pass  django;
 
 }
 

puppet/zulip/files/nginx/zulip-include-frontend/upstreams

@@ -1,5 +1,5 @@
 upstream django {
-    server unix:/home/zulip/deployments/fastcgi-socket;
+    server unix:/home/zulip/deployments/uwsgi-socket;
 }
 
 upstream tornado {

puppet/zulip/files/supervisor/conf.d/zulip.conf

@@ -7,40 +7,21 @@
 ; Note: shell expansion ("~" or "$HOME") is not supported.  Environment
 ; variables can be expanded using this syntax: "%(ENV_HOME)s".
 
-[fcgi-program:zulip-django]
-command=/home/zulip/deployments/current/manage.py runfcgi daemonize=False maxchildren=20 ; the program (relative uses PATH, can take args)
-;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
-;numprocs=1                    ; number of processes copies to start (def 1)
-;directory=/tmp                ; directory to cwd to before exec (def no cwd)
-;umask=022                     ; umask for process (default None)
+[program:zulip-django]
+command=/home/zulip/deployments/current/zulip-current-venv/bin/uwsgi --ini /etc/zulip/uwsgi.ini
 priority=100                   ; the relative start priority (default 999)
 autostart=true                 ; start at supervisord start (default: true)
 autorestart=true               ; whether/when to restart (default: unexpected)
-;startsecs=1                   ; number of secs prog must stay running (def. 1)
-;startretries=3                ; max # of serial start failures (default 3)
-;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
 stopsignal=HUP                 ; signal used to kill process (default TERM)
 stopwaitsecs=30                ; max num secs to wait b4 SIGKILL (default 10)
-;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
-;killasgroup=false             ; SIGKILL the UNIX process group (def false)
 user=zulip                    ; setuid to this UNIX account to run the program
 redirect_stderr=true           ; redirect proc stderr to stdout (default false)
 stdout_logfile=/var/log/zulip/django.log        ; stdout log path, NONE for none; default AUTO
 stdout_logfile_maxbytes=1GB   ; max # logfile bytes b4 rotation (default 50MB)
 stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
-;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-;stdout_events_enabled=false   ; emit events on stdout writes (default false)
-;stderr_logfile=/var/log/zulip/app.err        ; stderr log path, NONE for none; default AUTO
-;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
-;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)
-;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-;stderr_events_enabled=false   ; emit events on stderr writes (default false)
-;environment=A=1,B=2           ; process environment additions (def no adds)
-;serverurl=AUTO                ; override serverurl computation (childutils)
+stopasgroup=true              ; Without this, we leak processes every restart
+killasgroup=true              ; Without this, we leak processes every restart
 directory=/home/zulip/deployments/current/
-socket=unix:///home/zulip/deployments/fastcgi-socket
-socket_owner=zulip:zulip
-socket_mode=0700
 
 [program:zulip-tornado]
 command=env PYTHONUNBUFFERED=1 /home/zulip/deployments/current/manage.py runtornado 127.0.0.1:9993

puppet/zulip/manifests/app_frontend_base.pp

@@ -67,6 +67,18 @@ class zulip::app_frontend_base {
     source => "puppet:///modules/zulip/supervisor/conf.d/zulip.conf",
     notify => Service["supervisor"],
   }
+
+  $uwsgi_processes = zulipconf("application_server", "uwsgi_processes", "5")
+  file { "/etc/zulip/uwsgi.ini":
+    require => Package[supervisor],
+    ensure => file,
+    owner => "root",
+    group => "root",
+    mode => 644,
+    content => template("zulip/uwsgi.ini.template.erb"),
+    notify => Service["supervisor"],
+  }
+
   file { "/home/zulip/tornado":
     ensure => directory,
     owner => "zulip",

puppet/zulip/manifests/nginx.pp

@@ -24,14 +24,14 @@ class zulip::nginx {
     source => "puppet:///modules/zulip/nginx/nginx.conf",
   }
 
-  file { "/etc/nginx/fastcgi_params":
+  file { "/etc/nginx/uwsgi_params":
     require => Package["nginx-full"],
     ensure => file,
     owner  => "root",
     group  => "root",
     mode => 644,
     notify => Service["nginx"],
-    source => "puppet:///modules/zulip/nginx/fastcgi_params",
+    source => "puppet:///modules/zulip/nginx/uwsgi_params",
   }
 
   file { "/etc/nginx/sites-enabled/default":

puppet/zulip/templates/uwsgi.ini.template.erb

@@ -0,0 +1,12 @@
+[uwsgi]
+socket=/home/zulip/deployments/uwsgi-socket
+module=zproject.wsgi:application
+chdir=/home/zulip/deployments/current/
+chmod-socket=700
+chown-socket=zulip:zulip
+processes=<%= @uwsgi_processes %>
+harakiri=20
+post-buffering=4096
+env= LANG=en_US.UTF-8
+uid=zulip
+gid=zulip

requirements/prod.txt

@@ -1,3 +1,4 @@
 -r common.txt
 netifaces==0.10.5
 python-dateutil==2.5.3
+uWSGI==2.0.14

scripts/restart-server

@@ -26,7 +26,7 @@ subprocess.check_call(["./manage.py", "send_stats", "incr", "events.server_resta
 logging.info("Filling memcached caches")
 subprocess.check_call(["./manage.py", "fill_memcached_caches"])
 
-# Restart the FastCGI and related processes via supervisorctl.
+# Restart the uWSGI and related processes via supervisorctl.
 logging.info("Stopping workers")
 subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*"])
 logging.info("Stopping server core")

zproject/wsgi.py

@@ -22,6 +22,8 @@ sys.path.append(BASE_DIR)
 import scripts.lib.setup_path_on_import
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
+import django
+django.setup() # We need to call setup to load applications.
 
 # Because import_module does not correctly handle safe circular imports we
 # need to import zerver.models first before the middleware tries to import it.