From 360c27deddac9077f5bd8515627474af33e619b1 Mon Sep 17 00:00:00 2001
From: Greg Price <gnprice@gmail.com>
Date: Mon, 24 Jul 2017 19:46:15 -0700
Subject: [PATCH] tests: Bring all the top-level test scripts into alignment on
 what to test.

Notably, this adds our checks on translated message strings to
`tools/test-all`, so that they don't cause surprise failures in
CI after a branch is pushed.  (Alternately they could have gone
in `tools/lint` to accomplish the same goal, but `makemessages`
which they depend on is quite slow -- on my machine it takes 7s,
compared to 10s for all of `tools/lint`.)
---
 tools/test-all               |  8 +++++++-
 tools/travis/backend         | 20 +++++++++-----------
 tools/travis/frontend        |  4 ++++
 tools/travis/static-analysis | 10 ++++++++--
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/tools/test-all b/tools/test-all
index b305b42dbe..7a0bff7ed0 100755
--- a/tools/test-all
+++ b/tools/test-all
@@ -36,9 +36,14 @@ run ./tools/clean-repo
 
 # travis/static-analysis
 run ./tools/run-mypy
+# Not running check-py3 because it demands a clean Git worktree.
+# run ./tools/check-py3
 
 # travis/backend
 run ./tools/lint --pep8 $FORCEARG
+run python manage.py makemessages --locale en
+run python -W ignore tools/check-capitalization --no-generate
+run python -W ignore tools/check-frontend-i18n --no-generate
 run ./tools/test-tools
 run ./tools/test-backend $FORCEARG
 run ./tools/test-migrations
@@ -48,7 +53,8 @@ run ./tools/test-migrations
 # run ./tools/test-documentation
 run ./tools/test-help-documentation.py $FORCEARG
 run ./tools/test-api
-
+# Not running run-dev tests locally; we never have
+# run ./tools/test-run-dev
 # Not running queue worker reload tests since it's low-churn code
 # run ./tools/test-queue-worker-reload
 
diff --git a/tools/travis/backend b/tools/travis/backend
index 4692d1787e..67ee5a4563 100755
--- a/tools/travis/backend
+++ b/tools/travis/backend
@@ -1,8 +1,4 @@
 #!/bin/bash
-# This script is very similar to tools/test-all (what one runs
-# locally).  Possibly they should be merged, though it's worth noting,
-# they are intentionally different (basically some slow stuff is not
-# worth running in `test-all`).
 
 source tools/travis/activate-venv
 
@@ -10,6 +6,10 @@ set -e
 set -x
 
 ./tools/lint --pep8 # Include the slow and thus non-default pep8 linter check
+python manage.py makemessages --locale en
+python -W ignore tools/check-capitalization --no-generate
+python -W ignore tools/check-frontend-i18n --no-generate
+
 ./tools/test-tools
 ./tools/test-backend --coverage
 ./tools/test-migrations
@@ -17,11 +17,9 @@ set -x
 ./tools/test-documentation
 ./tools/test-help-documentation.py
 ./tools/test-api
+#./tools/test-run-dev  # Disabled in CI because flaky.
+#./tools/test-queue-worker-reload  # Disabled in CI because flaky.
 
-# Some test suites disabled in CI for being flaky
-#./tools/test-run-dev
-#./tools/test-queue-worker-reload
-
-python manage.py makemessages --locale en
-python -W ignore tools/check-capitalization --no-generate
-python -W ignore tools/check-frontend-i18n --no-generate
+# NB: Everything here should be in `tools/test-all`.  If there's a
+# reason not to run it there, it should be there as a comment
+# explaining why.
diff --git a/tools/travis/frontend b/tools/travis/frontend
index 091da738e4..bdda8bb9da 100755
--- a/tools/travis/frontend
+++ b/tools/travis/frontend
@@ -7,3 +7,7 @@ set -x
 
 ./tools/test-js-with-node --coverage
 ./tools/test-js-with-casper
+
+# NB: Everything here should be in `tools/test-all`.  If there's a
+# reason not to run it there, it should be there as a comment
+# explaining why.
diff --git a/tools/travis/static-analysis b/tools/travis/static-analysis
index 6bf27bf85e..f943cfca5a 100755
--- a/tools/travis/static-analysis
+++ b/tools/travis/static-analysis
@@ -1,11 +1,12 @@
 #!/bin/bash
 
-retcode=0
+set -e
 set -x
+
+retcode=0
 ./tools/run-mypy --py2 --linecoverage-report || retcode=1
 ./tools/run-mypy --py3 || retcode=1
 set +x
-
 if [ "$retcode" == "0" ]; then
     echo "The mypy static type checker for python detected no errors!"
 else
@@ -16,5 +17,10 @@ else
     echo "on mypy, how Zulip is using mypy, and how to debug common issues."
     exit "$retcode"
 fi
+set -x
 
 tools/check-py3
+
+# NB: Everything here should be in `tools/test-all`.  If there's a
+# reason not to run it there, it should be there as a comment
+# explaining why.