[manual] Humbug testing databases now backed by postgres

Currently our test database is backed by sqlite; this commit moves
us to using postgres for our all database needs. This, in conjunction
with the patched django on github, allow us to have fewer hacks and
more true-to-life tests. It also sets the stage for testing the bulk_create
and schema search_path patches made to django.

Developers will need to run:
./tools/postgres-init-test-db
./tools/do-destroy-rebuild-test-database

this is assuming that they have already run:
./tools/postgres-init-db
./tools/do-destroy-rebuild-database

at some point on this pg_cluster. (The ordering is important; it will other-
wise complain about the south_migration table).

(imported from commit c56c6f27e13df7ae10b2e643e65d669dde61af3d)

humbug/test_settings.py

@@ -2,9 +2,15 @@ from settings import *
 import os
 import logging
 
-DATABASES["default"] = {"NAME": "zephyr/tests/zephyrdb.test",
+DATABASES["default"] = {"NAME": "humbug_test",
-                        "ENGINE": "django.db.backends.sqlite3",
+                        "USER": "humbug_test",
-                        "OPTIONS": { "timeout": 20, },}
+                        "PASSWORD": "xxxxxxxxxxxx",
+                        "HOST": "localhost",
+                        "SCHEMA": "humbug",
+                        "ENGINE": "django.db.backends.postgresql_psycopg2",
+                        "TEST_NAME": "django_humbug_tests",
+                        "OPTIONS": { },}
+
 
 if "TORNADO_SERVER" in os.environ:
     TORNADO_SERVER = os.environ["TORNADO_SERVER"]

tools/do-destroy-rebuild-test-database

@@ -0,0 +1,5 @@
+#!/bin/sh -xe
+
+# This is a really simple wrapper script, pretty much for documenting clarity
+`dirname $0`/../zephyr/tests/generate-fixtures --force
+

tools/postgres-init-db

@@ -1,17 +1,39 @@
 #/bin/sh -xe
 
-PASSWORD=xxxxxxxxxxxx
+if [[ $# == 0 ]]; then
+    USERNAME=humbug
+    PASSWORD=xxxxxxxxxxxx
+    DBNAME=humbug
+    SEARCH_PATH="$USERNAME",public
+elif [[ $# == 4 ]]; then
+    USERNAME=$1
+    PASSWORD=$2
+    DBNAME=$3
+    SEARCH_PATH=$4
+else
+    echo "Usage Instructions"
+    echo "Run with either no arguments (sets up devel db for local deploy--humbug with user humbug)"
+    echo "or specify <db-username> <password> <db-name> <user-schema-search-path>"
+    exit
+fi
 
 sudo -u postgres psql << EOF
+CREATE USER $USERNAME WITH PASSWORD '$PASSWORD';
+ALTER USER $USERNAME CREATEDB;
+ALTER ROLE $USERNAME SET search_path TO $SEARCH_PATH;
+EOF
+
+umask go-rw
+echo "*:*:*:$USERNAME:$PASSWORD" >> ~/.pgpass
+chmod go-rw ~/.pgpass
+
+psql -h localhost postgres $USERNAME <<EOF
+CREATE DATABASE $DBNAME;
+EOF
+
+sudo -u postgres psql $DBNAME << EOF
 DROP SCHEMA public CASCADE;
-CREATE DATABASE humbug;
-CREATE USER humbug WITH PASSWORD '$PASSWORD';
-GRANT ALL ON DATABASE humbug TO humbug;
-ALTER USER humbug CREATEDB;
 EOF
 
 echo "Database created"
 
-umask go-rw
-echo "*:*:*:humbug:$PASSWORD" >> ~/.pgpass
-chmod go-rw ~/.pgpass

tools/postgres-init-test-db

@@ -0,0 +1,4 @@
+#/bin/sh -xe
+
+$(dirname $0)/postgres-init-db humbug_test xxxxxxxxxxxx humbug_test humbug,public
+

tools/test-backend

@@ -10,4 +10,5 @@ case "$1" in
 esac
 
 cd "$(dirname "$0")"/..
+./zephyr/tests/generate-fixtures
 ./manage.py test "$target" --settings=humbug.test_settings "$@"

zephyr/tests.py

@@ -2824,13 +2824,7 @@ class PivotalHookTests(AuthedTestCase):
 
 
 class Runner(DjangoTestSuiteRunner):
-    option_list = (
+    option_list = ()
-        optparse.make_option('--skip-generate',
-            dest='generate', default=True, action='store_false',
-            help='Skip generating test fixtures')
-    ,)
 
-    def __init__(self, generate, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
-        if generate:
-            subprocess.check_call("zephyr/tests/generate-fixtures");
         DjangoTestSuiteRunner.__init__(self, *args, **kwargs)

zephyr/tests/frontend/tests/06-unread.js

@@ -81,6 +81,10 @@ casper.then(function () {
     scroll_to(200);
     scroll_to(300);
     scroll_to(400);
+    scroll_to(500);
+    scroll_to(600);
+    scroll_to(700);
+    scroll_to(800);
     var sidebar_end = get_sidebar_num();
     casper.test.assert(sidebar_end < sidebar_initial, "Unread count in sidebar decreases after scrolling");
 

zephyr/tests/generate-fixtures

@@ -4,24 +4,30 @@ function migration_status {
     ./manage.py migrate --list | sed 's/*/ /' > "$1"
 }
 
-if [ -e zephyr/tests/zephyrdb.test.pristine ]; then
+template_grep_error_code=$(echo "SELECT 1 from pg_database WHERE datname='humbug_test_template';" | python manage.py dbshell --settings=humbug.test_settings | grep -q "1 row"; echo $?)
+
+if [ $template_grep_error_code == "0" ]; then
     migration_status zephyr/fixtures/available-migrations
     if [ -e zephyr/fixtures/migration-status ] &&
         cmp -s zephyr/fixtures/available-migrations zephyr/fixtures/migration-status &&
         [ "$1" != "--force" ]; then
-        cp zephyr/tests/zephyrdb.test.pristine zephyr/tests/zephyrdb.test
+            psql -h localhost postgres humbug_test << EOF
+DROP DATABASE humbug_test;
+CREATE DATABASE humbug_test TEMPLATE humbug_test_template;
+COMMIT;
+EOF
         exit 0
     fi
 fi
 
 mkdir -p zephyr/fixtures
-rm -f zephyr/tests/zephyrdb.test
-rm -f zephyr/tests/zephyrdb.test.pristine
 
 # Remove time.pyc to try to prevent it from screwing people importing
 # time from inside zephyr.lib.  We can drop this hack after a bit.
 rm -f zephyr/lib/time.pyc
 
+
+echo "DROP SCHEMA humbug CASCADE; CREATE SCHEMA humbug;" | python manage.py dbshell --settings=humbug.test_settings
 python manage.py syncdb --noinput --settings=humbug.test_settings
 python manage.py migrate --settings=humbug.test_settings --all
 migration_status "zephyr/fixtures/migration-status"
@@ -35,4 +41,9 @@ python manage.py dumpdata  --settings=humbug.test_settings \
     zephyr.UserMessage zephyr.Client \
     zephyr.DefaultStream > zephyr/fixtures/messages.json
 
-cp zephyr/tests/zephyrdb.test zephyr/tests/zephyrdb.test.pristine
+# create pristine template database, for fast fixture restoration after tests are run.
+psql -h localhost postgres humbug_test << EOF
+DROP DATABASE humbug_test_template;
+CREATE DATABASE humbug_test_template TEMPLATE humbug_test;
+COMMIT;
+EOF