From 789ae8648ada471cf0f51e18c8f1b7b941e397c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20H=C3=B6nig?= Date: Sat, 14 Jan 2017 10:19:26 +0000 Subject: [PATCH] Add wrapper and log file output for provisioning. Before this commit, provisioning was done by executing provision.py, which printed the log directly to stdout, making debugging harder. This commit creates a wrapper bash script 'provision' in tools, which calls 'zulip/scripts/tools/provision_vm.py' (the new location of provision.py) and prints all the output to 'zulip/var/log/zulip/zulip_provision.log' via 'tee'. Travis tests and docs have been modified accordingly. --- Vagrantfile | 4 ++-- docs/architecture-overview.md | 4 ++-- docs/dev-env-first-time-contributors.md | 20 ++++++++-------- docs/dev-setup-non-vagrant.md | 4 ++-- tools/{ => lib}/provision.py | 2 +- tools/lib/test_script.py | 6 ++--- tools/lint-all | 2 +- tools/minify-js | 2 +- tools/provision | 31 +++++++++++++++++++++++++ tools/travis/setup-backend | 3 ++- tools/travis/setup-production | 3 ++- zerver/tests/tests.py | 2 +- 12 files changed, 58 insertions(+), 25 deletions(-) rename tools/{ => lib}/provision.py (99%) create mode 100755 tools/provision diff --git a/Vagrantfile b/Vagrantfile index e180932aca..2c8bfe6e96 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -100,11 +100,11 @@ if [ -d "/sys/fs/selinux" ]; then sudo mount -o remount,ro /sys/fs/selinux fi ln -nsf /srv/zulip ~/zulip -/srv/zulip/tools/provision.py | sudo tee -a /var/log/zulip_provision.log +/srv/zulip/tools/provision SCRIPT config.vm.provision "shell", - # We want provision.py to be run with the permissions of the vagrant user. + # We want provision to be run with the permissions of the vagrant user. privileged: false, inline: $provision_script end diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index f6e600dc57..3a9a779b37 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -222,10 +222,10 @@ list of stopwords used by a Postgresql extension. In a development environment, configuration of that postgresql extension is handled by `tools/postgres-init-dev-db` (invoked by -`tools/provision.py`). That file also manages setting up the +`tools/provision`). That file also manages setting up the development postgresql user. -`tools/provision.py` also invokes `tools/do-destroy-rebuild-database` +`tools/provision also invokes `tools/do-destroy-rebuild-database` to create the actual database with its schema. ### Nagios diff --git a/docs/dev-env-first-time-contributors.md b/docs/dev-env-first-time-contributors.md index 094fbe0493..5dc72065c2 100644 --- a/docs/dev-env-first-time-contributors.md +++ b/docs/dev-env-first-time-contributors.md @@ -32,7 +32,7 @@ When reporting your issue, please include the following information: * installation method (Vagrant or direct) * whether or not you are using a proxy * a copy of Zulip's `vagrant` provisioning logs, available in - `/var/log/zulip_provision.log` on your virtual machine + `/var/log/provision.log` on your virtual machine ### Requirements @@ -268,7 +268,7 @@ does the following: - configures this virtual machine/container for use with Zulip, - creates a shared directory mapping your clone of the Zulip code inside the virtual machine/container at `~/zulip` -- runs the `tools/provision.py` script inside the virtual machine/container, which +- runs the `tools/provision` script inside the virtual machine/container, which downloads all required dependencies, sets up the python environment for the Zulip development server, and initializes a default test database. @@ -435,7 +435,7 @@ is likely because we've recently merged changes to the development environment provisioning process that you need to apply to your development environment. To update your environment, you'll need to re-provision your vagrant machine using `vagrant provision` (this just -runs `tools/provision.py` from your Zulip checkout inside the Vagrant +runs `tools/provision` from your Zulip checkout inside the Vagrant guest); this should complete in about a minute. After provisioning, you'll want to @@ -540,7 +540,7 @@ When reporting your issue, please include the following information: * installation method (Vagrant or direct) * whether or not you are using a proxy * a copy of Zulip's `vagrant` provisioning logs, available in - `/var/log/zulip_provision.log` on your virtual machine + `/var/log/provision.log` on your virtual machine #### The box 'ubuntu/trusty64' could not be found @@ -682,16 +682,16 @@ The `vagrant up` command basically does the following: * Downloads an Ubuntu image and starts it using a Vagrant provider. * Uses `vagrant ssh` to connect to that Ubuntu guest, and then runs - `tools/provision.py`, which has a lot of subcommands that are + `tools/provision`, which has a lot of subcommands that are executed via Python's `subprocess` module. These errors mean that one of those subcommands failed. To debug such errors, you can log in to the Vagrant guest machine by running `vagrant ssh`, which should present you with a standard shell prompt. You can debug interactively by using e.g. `cd zulip && -./tools/provision.py`, and then running the individual subcommands +./tools/provision`, and then running the individual subcommands that failed. Once you've resolved the problem, you can rerun -`tools/provision.py` to proceed; the provisioning system is designed +`tools/provision` to proceed; the provisioning system is designed to recover well from failures. The zulip provisioning system is generally highly reliable; the most common @@ -727,16 +727,16 @@ the VM. ##### npm install errors -The `tools/provision.py` script may encounter an error related to `npm install` +The `tools/provision` script may encounter an error related to `npm install` that looks something like: ``` ==> default: + npm install ==> default: Traceback (most recent call last): -==> default: File "/srv/zulip/tools/provision.py", line 195, in +==> default: File "/srv/zulip/tools/provision", line 195, in ==> default: ==> default: sys.exit(main()) -==> default: File "/srv/zulip/tools/provision.py", line 191, in main +==> default: File "/srv/zulip/tools/provision", line 191, in main ==> default: ==> default: run(["npm", "install"]) ==> default: File "/srv/zulip/scripts/lib/zulip_tools.py", line 78, in run diff --git a/docs/dev-setup-non-vagrant.md b/docs/dev-setup-non-vagrant.md index bd6b85045e..8091c159a6 100644 --- a/docs/dev-setup-non-vagrant.md +++ b/docs/dev-setup-non-vagrant.md @@ -17,7 +17,7 @@ can do that by just running: ``` # From a clone of zulip.git -./tools/provision.py +./tools/provision source /srv/zulip-venv/bin/activate ./tools/run-dev.py # starts the development server ``` @@ -395,7 +395,7 @@ docker build -t user/zulipdev . Commit and tag the provisioned images. The below will install Zulip's dependencies: ``` docker run -itv $(pwd):/srv/zulip -p 9991:9991 user/zulipdev /bin/bash -$ /usr/bin/python /srv/zulip/tools/provision.py --docker +$ /usr/bin/python /srv/zulip/tools/provision --docker docker ps -af ancestor=user/zulipdev docker commit -m "Zulip installed" user/zulipdev:v2 ``` diff --git a/tools/provision.py b/tools/lib/provision.py similarity index 99% rename from tools/provision.py rename to tools/lib/provision.py index f36a87d7a9..04e5cd1ab8 100755 --- a/tools/provision.py +++ b/tools/lib/provision.py @@ -11,7 +11,7 @@ os.environ["PYTHONUNBUFFERED"] = "y" PY2 = sys.version_info[0] == 2 -ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import run, subprocess_text_output, OKBLUE, ENDC, WARNING diff --git a/tools/lib/test_script.py b/tools/lib/test_script.py index feea962c61..56234b1070 100644 --- a/tools/lib/test_script.py +++ b/tools/lib/test_script.py @@ -33,7 +33,7 @@ version of dependencies than the version you provisioned last. This may be ok, but it's likely that you either want to rebase your branch on top of upstream/master or re-provision your VM. -Do this: `./tools/provision.py` +Do this: `./tools/provision` ''' NEED_TO_UPGRADE = ''' @@ -41,7 +41,7 @@ It looks like you checked out a branch that has added dependencies beyond what you last provisioned. Your command is likely to fail until you add dependencies by provisioning. -Do this: `./tools/provision.py` +Do this: `./tools/provision` ''' def get_provisioning_status(): @@ -52,7 +52,7 @@ def get_provisioning_status(): # If the developer doesn't have a version_file written by # a previous provision, then we don't do any safety checks # here on the assumption that the developer is managing - # their own dependencies and not running provision.py. + # their own dependencies and not running provision. return True, None version = open(version_file).read().strip() diff --git a/tools/lint-all b/tools/lint-all index cb6a9084b4..65f03555d3 100755 --- a/tools/lint-all +++ b/tools/lint-all @@ -347,7 +347,7 @@ def build_custom_checkers(by_lang): # This rule might give false positives in virtualenv setup files which should be excluded, # and comments which should be rewritten to avoid use of "python2", "python3", etc. {'pattern': 'python[23]', - 'exclude': set(['tools/provision.py', + 'exclude': set(['tools/lib/provision.py', 'tools/setup/setup_venvs.py', 'scripts/lib/setup_venv.py', 'tools/lint-all']), diff --git a/tools/minify-js b/tools/minify-js index 45eaccfe3f..a980bc39c9 100755 --- a/tools/minify-js +++ b/tools/minify-js @@ -88,7 +88,7 @@ CLOSURE_BINARY = '/usr/bin/closure-compiler' if not os.path.exists(CLOSURE_BINARY): CLOSURE_BINARY = 'tools/closure-compiler/run' if not os.path.exists(CLOSURE_BINARY): - print("closure-compiler not installed; the Vagrant tools/provision.py installs it via apt " + print("closure-compiler not installed; the Vagrant tools/provision installs it via apt " "or you can manually unpack http://dl.google.com/closure-compiler/compiler-latest.zip to " "tools/closure-compiler") sys.exit(1) diff --git a/tools/provision b/tools/provision new file mode 100755 index 0000000000..1effcd077b --- /dev/null +++ b/tools/provision @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +if [ "$EUID" -eq 0 ]; then + echo "Error: The provision script must not be run as root" >&2 + exit 1 +fi + +#Make the script independent of the location from where it is +#executed +PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE}")" ; pwd -P ) +cd "$PARENT_PATH" +mkdir -p ../var/log +LOG_PATH="../var/log/provision.log" +PROVISION_PATH="lib/provision.py" + +echo "PROVISIONING STARTING." >> $LOG_PATH + +python "$PROVISION_PATH" $@ 2>&1 | tee -a "$LOG_PATH" +failed=${PIPESTATUS[0]} + +if [ $failed = 1 ]; then + echo -e "\033[0;31m" + echo "Provisioning failed!" + echo + echo -n "The provision process is designed to be idempotent, so you can retry " + echo -n "after resolving whatever issue caused the failure (there should be a traceback above). " + echo -n "A log of this installation is available in zulip/log/provision.log" + echo -e "\033[0m" + exit 1 +fi +exit 0 diff --git a/tools/travis/setup-backend b/tools/travis/setup-backend index 68d65185da..92dd59e4b5 100755 --- a/tools/travis/setup-backend +++ b/tools/travis/setup-backend @@ -1,6 +1,7 @@ #!/bin/bash set -e set -x -tools/provision.py --travis + +tools/provision --travis sudo mkdir -p /var/lib/nagios_state sudo chown travis /var/lib/nagios_state diff --git a/tools/travis/setup-production b/tools/travis/setup-production index c88eabb88c..233dc2181f 100755 --- a/tools/travis/setup-production +++ b/tools/travis/setup-production @@ -11,7 +11,8 @@ sudo chmod a+rX /home/travis # to conflicts over which version of postgres should be running. sudo apt-get remove postgresql-9.1 postgresql-client-9.1 postgresql-9.1-postgis-2.2 postgresql-contrib-9.1 postgresql-9.1-postgis-2.2-scripts postgresql-9.2 postgresql-client-9.2 postgresql-9.2-postgis-2.2 postgresql-9.2-postgis-2.2-scripts postgresql-contrib-9.2 postgresql-9.4 postgresql-9.4-postgis-2.2-scripts postgresql-client-9.4 postgresql-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-2.2-scripts postgresql-contrib-9.5 postgresql-client-9.5 postgresql-9.6 postgresql-9.6-postgis-2.2 postgresql-9.6-postgis-2.2-scripts postgresql-contrib-9.6 postgresql-client-9.6 -y -tools/provision.py --travis --production-travis +tools/provision --travis --production-travis + cp -a tools/travis/success-http-headers.txt ~/ source tools/travis/activate-venv diff --git a/zerver/tests/tests.py b/zerver/tests/tests.py index c9f6eb4a04..0f2314ac7d 100644 --- a/zerver/tests/tests.py +++ b/zerver/tests/tests.py @@ -2134,7 +2134,7 @@ class HomeTest(ZulipTestCase): class AuthorsPageTest(ZulipTestCase): def setUp(self): # type: () -> None - """ Manual installation which did not execute `tools/provision.py` + """ Manual installation which did not execute `tools/provision` would not have the `static/generated/github-contributors.json` fixture file. """