mirror of https://github.com/zulip/zulip.git
provision: Rewrite using subprocess module instead of sh.
The `with sh.sudo` pattern that we were using in python-sh was deprecated, and emperically hangs on Ubuntu xenial. Since in general the use of python-sh/python-pbs caused trouble (requiring extra dependencies, confusing syntax), this just removes it. We replace it with a new zulip_tools.py library function that echoes the command line and streams the output. We do the same to install-phantomjs so we can remove that dependency.
This commit is contained in:
Tim Abbott
parent
2ac5271091
commit
52fc1c71bc
|
|
@ -97,7 +97,6 @@ running:
|
|||
|
||||
```
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python-pbs
|
||||
python /srv/zulip/provision.py
|
||||
|
||||
cd /srv/zulip
|
||||
|
|
|
|||
|
|
@ -39,8 +39,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
|||
$provision_script = <<SCRIPT
|
||||
set -x
|
||||
set -e
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python-pbs
|
||||
/usr/bin/python /srv/zulip/provision.py
|
||||
SCRIPT
|
||||
|
||||
|
|
|
|||
80
provision.py
80
provision.py
|
|
@ -5,10 +5,10 @@ import logging
|
|||
import platform
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
import sh
|
||||
except ImportError:
|
||||
import pbs as sh
|
||||
os.environ["PYTHONUNBUFFERED"] = "y"
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
from zulip_tools import run
|
||||
|
||||
SUPPORTED_PLATFORMS = {
|
||||
"Ubuntu": [
|
||||
|
|
@ -95,34 +95,22 @@ REPO_STOPWORDS_PATH = os.path.join(
|
|||
|
||||
LOUD = dict(_out=sys.stdout, _err=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
run(["sudo", "apt-get", "update"])
|
||||
run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])
|
||||
|
||||
with sh.sudo:
|
||||
sh.apt_get.update(**LOUD)
|
||||
|
||||
sh.apt_get.install(*APT_DEPENDENCIES[codename], assume_yes=True, **LOUD)
|
||||
|
||||
temp_deb_path = sh.mktemp("package_XXXXXX.deb", tmpdir=True)
|
||||
|
||||
sh.wget(
|
||||
TSEARCH_URL,
|
||||
output_document=temp_deb_path,
|
||||
**LOUD
|
||||
)
|
||||
|
||||
with sh.sudo:
|
||||
sh.dpkg("--install", temp_deb_path, **LOUD)
|
||||
temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
|
||||
run(["wget", "-O", temp_deb_path, TSEARCH_URL])
|
||||
run(["sudo", "dpkg", "--install", temp_deb_path])
|
||||
|
||||
# Install phantomjs
|
||||
os.system("./tools/install-phantomjs")
|
||||
run(["./tools/install-phantomjs"])
|
||||
|
||||
with sh.sudo:
|
||||
sh.rm("-rf", VENV_PATH, **LOUD)
|
||||
sh.mkdir("-p", VENV_PATH, **LOUD)
|
||||
sh.chown("{}:{}".format(os.getuid(), os.getgid()), VENV_PATH, **LOUD)
|
||||
run(["sudo", "rm", "-rf", VENV_PATH])
|
||||
run(["sudo", "mkdir", "-p", VENV_PATH])
|
||||
run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), VENV_PATH])
|
||||
|
||||
sh.virtualenv(VENV_PATH, **LOUD)
|
||||
run(["virtualenv", VENV_PATH])
|
||||
|
||||
# Add the ./tools and ./scripts/setup directories inside the repository root to
|
||||
# the system path; we'll reference them later.
|
||||
|
|
@ -145,36 +133,36 @@ def main():
|
|||
activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py")
|
||||
execfile(activate_this, dict(__file__=activate_this))
|
||||
|
||||
sh.pip.install(requirement=os.path.join(ZULIP_PATH, "requirements.txt"), **LOUD)
|
||||
run(["pip", "install", "--requirement",
|
||||
os.path.join(ZULIP_PATH, "requirements.txt")])
|
||||
|
||||
with sh.sudo:
|
||||
sh.cp(REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH, **LOUD)
|
||||
run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
|
||||
|
||||
# npm install and management commands expect to be run from the root of the
|
||||
# project.
|
||||
os.chdir(ZULIP_PATH)
|
||||
|
||||
os.system("tools/download-zxcvbn")
|
||||
os.system("tools/emoji_dump/build_emoji")
|
||||
os.system("generate_secrets.py -d")
|
||||
run(["tools/download-zxcvbn"])
|
||||
run(["tools/emoji_dump/build_emoji"])
|
||||
run(["scripts/setup/generate_secrets.py", "-d"])
|
||||
if "--travis" in sys.argv:
|
||||
os.system("sudo service rabbitmq-server restart")
|
||||
os.system("sudo service redis-server restart")
|
||||
os.system("sudo service memcached restart")
|
||||
run(["sudo", "service", "rabbitmq-server", "restart"])
|
||||
run(["sudo", "service", "redis-server", "restart"])
|
||||
run(["sudo", "service", "memcached", "restart"])
|
||||
elif "--docker" in sys.argv:
|
||||
os.system("sudo service rabbitmq-server restart")
|
||||
os.system("sudo pg_dropcluster --stop %s main" % (POSTGRES_VERSION,))
|
||||
os.system("sudo pg_createcluster -e utf8 --start %s main" % (POSTGRES_VERSION,))
|
||||
os.system("sudo service redis-server restart")
|
||||
os.system("sudo service memcached restart")
|
||||
sh.configure_rabbitmq(**LOUD)
|
||||
sh.postgres_init_dev_db(**LOUD)
|
||||
sh.do_destroy_rebuild_database(**LOUD)
|
||||
sh.postgres_init_test_db(**LOUD)
|
||||
sh.do_destroy_rebuild_test_database(**LOUD)
|
||||
run(["sudo", "service", "rabbitmq-server", "restart"])
|
||||
run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
|
||||
run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
|
||||
run(["sudo", "service", "redis-server", "restart"])
|
||||
run(["sudo", "service", "memcached", "restart"])
|
||||
run(["scripts/setup/configure-rabbitmq"])
|
||||
run(["tools/postgres-init-dev-db"])
|
||||
run(["tools/do-destroy-rebuild-database"])
|
||||
run(["tools/postgres-init-test-db"])
|
||||
run(["tools/do-destroy-rebuild-test-database"])
|
||||
# Run npm install last because it can be flaky, and that way one
|
||||
# only needs to rerun `npm install` to fix the installation.
|
||||
sh.npm.install(**LOUD)
|
||||
run(["npm", "install"])
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -3,28 +3,27 @@ from __future__ import print_function
|
|||
import os
|
||||
import sys
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
import sh
|
||||
except ImportError:
|
||||
import pbs as sh
|
||||
|
||||
LOUD = dict(_out=sys.stdout, _err=sys.stderr)
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
from zulip_tools import run
|
||||
|
||||
if platform.architecture()[0] == '64bit':
|
||||
phantomjs_arch = 'x86_64'
|
||||
elif platform.architecture()[0] == '32bit':
|
||||
phantomjs_arch = 'i686'
|
||||
|
||||
with sh.sudo:
|
||||
PHANTOMJS_PATH = "/srv/phantomjs"
|
||||
PHANTOMJS_BASENAME = "phantomjs-1.9.8-linux-%s" % (phantomjs_arch,)
|
||||
PHANTOMJS_TARBALL_BASENAME = PHANTOMJS_BASENAME + ".tar.bz2"
|
||||
PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, PHANTOMJS_TARBALL_BASENAME)
|
||||
PHANTOMJS_URL = "https://github.com/zulip/zulip-dist-phantomjs/blob/master/%s?raw=true" % (PHANTOMJS_TARBALL_BASENAME,)
|
||||
sh.mkdir("-p", PHANTOMJS_PATH, **LOUD)
|
||||
if not os.path.exists(PHANTOMJS_TARBALL):
|
||||
sh.curl('-J', '-L', PHANTOMJS_URL, o=PHANTOMJS_TARBALL, **LOUD)
|
||||
sh.tar("xj", directory=PHANTOMJS_PATH, file=PHANTOMJS_TARBALL, **LOUD)
|
||||
sh.ln("-sf", os.path.join(PHANTOMJS_PATH, PHANTOMJS_BASENAME, "bin", "phantomjs"),
|
||||
"/usr/local/bin/phantomjs", **LOUD)
|
||||
PHANTOMJS_PATH = "/srv/phantomjs"
|
||||
PHANTOMJS_BASENAME = "phantomjs-1.9.8-linux-%s" % (phantomjs_arch,)
|
||||
PHANTOMJS_TARBALL_BASENAME = PHANTOMJS_BASENAME + ".tar.bz2"
|
||||
PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, PHANTOMJS_TARBALL_BASENAME)
|
||||
PHANTOMJS_URL = "https://github.com/zulip/zulip-dist-phantomjs/blob/master/%s?raw=true" % (PHANTOMJS_TARBALL_BASENAME,)
|
||||
|
||||
run(["sudo", "mkdir", "-p", PHANTOMJS_PATH])
|
||||
if not os.path.exists(PHANTOMJS_TARBALL):
|
||||
run(["sudo", "curl", '-J', '-L', PHANTOMJS_URL, "-o", PHANTOMJS_TARBALL])
|
||||
run(["sudo", "tar", "-xj", "--directory", PHANTOMJS_PATH,
|
||||
"--file", PHANTOMJS_TARBALL])
|
||||
run(["sudo", "ln", "-sf", os.path.join(PHANTOMJS_PATH, PHANTOMJS_BASENAME,
|
||||
"bin", "phantomjs"),
|
||||
"/usr/local/bin/phantomjs"])
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
set -x
|
||||
pip install pbs
|
||||
python provision.py --travis
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ set -x
|
|||
# to conflicts over which version of postgres should be running.
|
||||
sudo apt-get remove postgresql-9.1 postgresql-9.2 postgresql-9.4 -y
|
||||
|
||||
pip install pbs
|
||||
python provision.py --travis
|
||||
source /srv/zulip-venv/bin/activate
|
||||
./tools/build-release-tarball travis
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import errno
|
|||
import os
|
||||
import pwd
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
|
@ -67,3 +68,12 @@ def get_deployment_lock(error_rerun_script):
|
|||
|
||||
def release_deployment_lock():
|
||||
shutil.rmtree(LOCK_DIR)
|
||||
|
||||
def run(args):
|
||||
# Output what we're doing in the `set -x` style
|
||||
print("+ %s" % (" ".join(args)))
|
||||
process = subprocess.Popen(args)
|
||||
rc = process.wait()
|
||||
if rc:
|
||||
raise subprocess.CalledProcessError(rc, args)
|
||||
return 0
|
||||
|
|
|
|||
Loading…
Reference in New Issue