fixed make test reporting success when provision fails

We were getting build test 'pass' on the build farm when the provision
is failing. The cause was the use of filter pipelines in the selftest
code in the makefile. A bourne shell will by default report the error
status of the last element of the pipeline, which means when provision
failed, make thought everything had passed.

We could use the bash 'pipefail' option to fix this, but it isn't
portable, so instead I've modified the makefile to touch st/st_done
when the first element in the pipeline completes successfully. That
means we now get errors in make test, at the expense of making the
makefile even more horrible than it was.

diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 51aa7712199e3f016d92fb142e0c02a2a66285be..9d4fb4b91e2564a631178364b34d37bf78262d25 100644 (file)
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -1,10 +1,16 @@
 TEST_FORMAT = plain
 
-SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
+SELFTEST = ( rm -f $(selftest_prefix)/st_done && $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
-    $(TEST_OPTIONS) 
+    $(TEST_OPTIONS)
+# this strange code is to handle the fact that the bash pipefail option is 
+# not portable. When we run selftest we use a pipeline, and the first element
+# of that pipeline might abort. We need to catch this and mark the test
+# as having failed
+ST_TOUCH = && touch $(selftest_prefix)/st_done ) 
+ST_DONE_TEST = @test -f $(selftest_prefix)/st_done || { echo "SELFTEST FAILED"; exit 1; }
 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
@@ -12,56 +18,71 @@ FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(sr
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
 test-subunit:: everything
-       $(SELFTEST) --socket-wrapper $(TESTS)
+       $(SELFTEST) --socket-wrapper $(TESTS) $(ST_TOUCH)
+       $(ST_DONE_TEST)
 
 slowtest:: everything
-       $(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 ifeq ($(RUN_FROM_BUILD_FARM),yes)
 test:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FILTER_XFAIL) --strip-passed-output
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FILTER_XFAIL) --strip-passed-output
+       $(ST_DONE_TEST)
 else
 test:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 endif
 
 kvmtest:: everything
        $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) \
-               --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) --immediate 
+               --target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 kvmquicktest:: everything
        $(SELFTEST) $(DEFAULT_TEST_OPTIONS) \
-               $(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+               $(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 testone:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+       $(ST_DONE_TEST)
 
 test-swrap:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 test-swrap-pcap:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 test-swrap-keep-pcap:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 test-noswrap:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 quicktest:: all
-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 quicktest-subunit:: all
-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS)
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH)
+       $(ST_DONE_TEST)
 
 quicktestone:: all
-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+       $(ST_DONE_TEST)
 
 testenv:: everything
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
 
 testenv-%:: everything
-       SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)
 
 test-%:: 
        $(MAKE) test TESTS=$*
@@ -71,29 +92,35 @@ valgrindtest:: valgrindtest-all
 valgrindtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 valgrindtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 valgrindtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)
 
 gdbtest:: gdbtest-all
 
 gdbtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 gdbtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)
 
 gdbtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)