Complete rewrite of test/run.sh

Output is more useful (IMO). Everything is silenced for the duration of
the tests but a count of tests/failures and any errors are output at the
end. Pass --verbose to watch stdout during the tests.

Logic is also seperated into core setup/runners and individual test
definitions. Should be more easily extensible.

yesod/test/run.sh

@@ -1,26 +1,97 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 #
-# A wrapper for the shelltest test. Passes along options to shelltest.
+# Runs test/scaffold.sh with a variety of inputs. Hides all output
+# besides failure details.
 #
-#   cabal install shelltestrunner
+###
 
-cabal clean && cabal install && cabal sdist
+[[ "$1" =~ -v|--verbose ]] && stdout=/dev/stdout || stdout=/dev/null
 
-# I am not that good at shell scripting
-# this for loop only operates on 1 file (as per tail -1)
-for f in $(ls -1rt dist/*.tar.gz | tail -1)
-do
-  tar -xzvf $f && cd `basename $f .tar.gz`
+tmp='/tmp'
+pwd="$PWD"
 
-  # shelltest is designed to show you the diff of an expected stdout/stdin. We don't care about that. If it compiles, it compiles
-  # shelltest ../test/scaffold.shelltest --color --diff --all $@ -- --hide-successes
+pkg=
+dir=
 
-  ../test/scaffold.sh < ../test/sqlite-input.txt &&
-  ../test/scaffold.sh < ../test/postgresql-input.txt &&
-  ../test/scaffold.sh < ../test/tiny-input.txt &&
-  ../test/scaffold.sh < ../test/mongodb-input.txt ||
-    (echo "FAILED" && exit 1)
-  cd ..
-  rm -r `basename $f .tar.gz`
-done
-echo "PASSED"
+failures=()
+n_tested=0
+n_failed=0
+
+# runs the function named by $1, silencing stdout and redirecting stderr
+# to /tmp/function.errors. failures are tracked to be reported on during
+# cleanup
+run_test() { # {{{
+  local test_function="$*"
+
+  n_tested=$((n_tested+1))
+
+  if $test_function >"$stdout" 2>"$tmp/$test_function.errors"; then
+    echo -n '.'
+    [[ -f "$tmp/$test_function.errors" ]] && rm "$tmp/$test_function.errors"
+  else
+    echo -n 'F'
+    failures+=( "$test_function" )
+    n_failed=$((n_failed+1))
+  fi
+}
+# }}}
+
+# changes back to the original directory, removes the dist file and
+# outputs a report of tests and failures
+cleanup() { # {{{
+  cd "$pwd"
+  [[ -d "$dir" ]] && rm -r "$dir"
+
+  echo
+  echo
+  echo "Tests: $n_tested, Failures: $n_failed."
+  echo
+
+  [[ $n_failed -eq 0 ]] && return 0
+
+  for test in ${failures[@]}; do
+    echo "Failure: $test"
+    echo 'details:'
+    echo
+
+    if [[ -f "$tmp/$test.errors" ]]; then
+      cat "$tmp/$test.errors"
+      rm  "$tmp/$test.errors"
+    else
+      echo '<no stderr captured>'
+    fi
+
+    echo
+  done
+
+  return $n_failed
+}
+# }}}
+
+# compilation is test #1, sets global variable dir. other tests are run
+# from within this directory and it is removed as part of cleanup
+test_compile() {
+  cabal clean
+  cabal install
+  cabal sdist
+
+  read -r pkg < <(find dist/ -type f -name '*.tar.gz' | sort -rV)
+  dir="$(basename "$pkg" .tar.gz)"
+
+  tar -xzf "$pkg" && cd "$dir"
+}
+
+test_sqlite()     { ../test/scaffold.sh < ../test/sqlite-input.txt    ; }
+test_postgresql() { ../test/scaffold.sh < ../test/postgresql-input.txt; }
+test_mongodb()    { ../test/scaffold.sh < ../test/mongodb-input.txt   ; }
+test_tiny()       { ../test/scaffold.sh < ../test/tiny-input.txt      ; }
+
+echo 'Started'
+run_test 'test_compile'
+run_test 'test_sqlite'
+run_test 'test_postgresql'
+run_test 'test_mongodb'
+run_test 'test_tiny'
+cleanup
+
+exit $?