Activate virtualenv in production Python code.

The manage.py change effectively switches the Zulip production server
to use the virtualenv, since all of our supervisord commands for the
various Python services go through manage.py.

Additionally, this migrates the production scripts and Nagios plugins
to use the virtualenv as well.

manage.py

@@ -4,17 +4,21 @@ import sys
 import logging
 import subprocess
 
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(BASE_DIR)
+import scripts.lib.setup_path_on_import
+
 if __name__ == "__main__":
     if 'posix' in os.name and os.geteuid() == 0:
         from django.core.management.base import CommandError
         raise CommandError("manage.py should not be run as root.")
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
-    os.environ.setdefault("PYTHONSTARTUP", os.path.join(os.path.dirname(__file__), "scripts/lib/pythonrc.py"))
+    os.environ.setdefault("PYTHONSTARTUP", os.path.join(BASE_DIR, "scripts/lib/pythonrc.py"))
 
     from django.conf import settings
 
     logger = logging.getLogger("zulip.management")
-    subprocess.check_call([os.path.join(os.path.dirname(__file__), "scripts", "lib", "log-management-command"),
+    subprocess.check_call([os.path.join(BASE_DIR, "scripts", "lib", "log-management-command"),
                            " ".join(sys.argv)])
 
     if "--no-traceback" not in sys.argv and len(sys.argv) > 1:

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_queue_worker_errors

@@ -7,6 +7,8 @@ from __future__ import print_function
 
 import sys
 sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 from zproject import settings
 
 import glob

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time

@@ -18,6 +18,9 @@ import random
 import traceback
 import os
 
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 import django
 
 def total_seconds(timedelta):

puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log

@@ -5,6 +5,10 @@ Nagios plugin to check the length of the FTS update log.
 """
 from __future__ import print_function
 
+import sys
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 import psycopg2
 
 states = {

puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup

@@ -1,11 +1,16 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
-import dateutil.parser
-import pytz
 import subprocess
 from datetime import datetime, timedelta
 
+import sys
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
+import dateutil.parser
+import pytz
+
 states = {
     "OK": 0,
     "WARNING": 1,

puppet/zulip/files/postgresql/process_fts_updates

@@ -8,6 +8,20 @@
 # search column search_tsvector in the main zerver_message.
 from __future__ import print_function
 
+import sys
+
+# We want to use a virtualenv in production, which will be in /home/zulip/deployments/current.
+# So we should add that path to sys.path and then import scripts.lib.setup_path_on_import.
+# But this file is also used in development, where the above path will not exist.
+# So `import scripts.lib.setup_path_on_import` will raise an ImportError.
+# In development, we just want to skip this step since we know that virtualenv will already be in use.
+# So catch the ImportError and do nothing.
+sys.path.append('/home/zulip/deployments/current')
+try:
+    import scripts.lib.setup_path_on_import
+except ImportError:
+    pass
+
 import psycopg2
 import psycopg2.extensions
 import select

puppet/zulip_internal/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness

@@ -13,6 +13,10 @@ import datetime
 import os
 import sys
 
+import sys
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 import django
 
 os.environ['DJANGO_SETTINGS_MODULE'] = "zproject.settings"

puppet/zulip_internal/files/postgresql/pg_backup_and_purge.py

@@ -1,6 +1,11 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
+
+import sys
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 import subprocess
 import sys
 import logging

puppet/zulip_internal/files/zulip-ec2-configure-interfaces

@@ -49,6 +49,10 @@ import logging.handlers
 import subprocess
 import re
 
+import sys
+sys.path.append('/home/zulip/deployments/current')
+import scripts.lib.setup_path_on_import
+
 import boto.utils
 import netifaces
 from six.moves import range

scripts/get-django-setting

@@ -3,9 +3,13 @@ from __future__ import absolute_import
 from __future__ import print_function
 
 import os
+from os.path import dirname, abspath
 import sys
 
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+BASE_DIR = dirname(dirname(abspath(__file__)))
+sys.path.append(BASE_DIR)
+import scripts.lib.setup_path_on_import
+
 os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings'
 from django.conf import settings
 

scripts/lib/log-management-command

@@ -2,7 +2,12 @@
 import sys
 import logging
 import os
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from os.path import dirname, abspath
+
+BASE_DIR = dirname(dirname(dirname(abspath(__file__))))
+sys.path.append(BASE_DIR)
+import scripts.lib.setup_path_on_import
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
 from django.conf import settings
 

scripts/lib/setup_path_on_import.py

@@ -0,0 +1,15 @@
+"""
+Use libraries from a virtualenv (by modifying sys.path) in production.
+Also add Zulip's root directory to sys.path
+"""
+
+import os
+from os.path import dirname, abspath
+import sys
+
+BASE_DIR = dirname(dirname(dirname(abspath(__file__))))
+activate_this = os.path.join(BASE_DIR, "zulip-venv", "bin", "activate_this.py")
+if os.path.exists(activate_this):
+    # this file will exist in production
+    exec(open(activate_this).read(), {}, dict(__file__=activate_this)) # type: ignore # https://github.com/python/mypy/issues/1577
+sys.path.append(BASE_DIR)

scripts/setup/generate_secrets.py

@@ -3,8 +3,12 @@
 
 from __future__ import print_function
 import sys, os, os.path
+from os.path import dirname, abspath
+
+BASE_DIR = dirname(dirname(dirname(abspath(__file__))))
+sys.path.append(BASE_DIR)
+import scripts.lib.setup_path_on_import
 
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
 os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings'
 
 from django.utils.crypto import get_random_string

zproject/wsgi.py

@@ -14,6 +14,12 @@ framework.
 
 """
 import os
+from os.path import dirname, abspath
+import sys
+
+BASE_DIR = dirname(dirname(abspath(__file__)))
+sys.path.append(BASE_DIR)
+import scripts.lib.setup_path_on_import
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")