fixed make test reporting success when provision fails

We were getting build test 'pass' on the build farm when the provision
is failing. The cause was the use of filter pipelines in the selftest
code in the makefile. A bourne shell will by default report the error
status of the last element of the pipeline, which means when provision
failed, make thought everything had passed.

We could use the bash 'pipefail' option to fix this, but it isn't
portable, so instead I've modified the makefile to touch st/st_done
when the first element in the pipeline completes successfully. That
means we now get errors in make test, at the expense of making the
makefile even more horrible than it was.

diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 51aa7712199e3f016d92fb142e0c02a2a66285be..9d4fb4b91e2564a631178364b34d37bf78262d25 100644 (file)
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -1,10 +1,16 @@
 TEST_FORMAT = plain
 
 TEST_FORMAT = plain
 

-SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
+SELFTEST = ( rm -f $(selftest_prefix)/st_done && $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \

     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \

-    $(TEST_OPTIONS) 
+    $(TEST_OPTIONS)
+# this strange code is to handle the fact that the bash pipefail option is 
+# not portable. When we run selftest we use a pipeline, and the first element
+# of that pipeline might abort. We need to catch this and mark the test
+# as having failed
+ST_TOUCH = && touch $(selftest_prefix)/st_done ) 
+ST_DONE_TEST = @test -f $(selftest_prefix)/st_done || { echo "SELFTEST FAILED"; exit 1; }

 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick


@@ -12,56 +18,71 @@ FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(sr
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
 test-subunit:: everything
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
 test-subunit:: everything

-       $(SELFTEST) --socket-wrapper $(TESTS)
+       $(SELFTEST) --socket-wrapper $(TESTS) $(ST_TOUCH)
+       $(ST_DONE_TEST)

 
 slowtest:: everything
 
 slowtest:: everything

-       $(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 ifeq ($(RUN_FROM_BUILD_FARM),yes)
 test:: everything
 
 ifeq ($(RUN_FROM_BUILD_FARM),yes)
 test:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FILTER_XFAIL) --strip-passed-output
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FILTER_XFAIL) --strip-passed-output
+       $(ST_DONE_TEST)

 else
 test:: everything
 else
 test:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 endif
 
 kvmtest:: everything
        $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) \
 endif
 
 kvmtest:: everything
        $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) \

-               --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) --immediate 
+               --target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 kvmquicktest:: everything
        $(SELFTEST) $(DEFAULT_TEST_OPTIONS) \
 
 kvmquicktest:: everything
        $(SELFTEST) $(DEFAULT_TEST_OPTIONS) \

-               $(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+               $(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 testone:: everything
 
 testone:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+       $(ST_DONE_TEST)

 
 test-swrap:: everything
 
 test-swrap:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 test-swrap-pcap:: everything
 
 test-swrap-pcap:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 test-swrap-keep-pcap:: everything
 
 test-swrap-keep-pcap:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 test-noswrap:: everything
 
 test-noswrap:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 quicktest:: all
 
 quicktest:: all

-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 quicktest-subunit:: all
 
 quicktest-subunit:: all

-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS)
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH)
+       $(ST_DONE_TEST)

 
 quicktestone:: all
 
 quicktestone:: all

-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT)
+       $(ST_DONE_TEST)

 
 testenv:: everything
 
 testenv:: everything

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)

 
 testenv-%:: everything
 
 testenv-%:: everything

-       SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       SELFTEST_TESTENV=$* $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)

 
 test-%:: 
        $(MAKE) test TESTS=$*
 
 test-%:: 
        $(MAKE) test TESTS=$*


@@ -71,29 +92,35 @@ valgrindtest:: valgrindtest-all
 valgrindtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
 valgrindtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \

-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 valgrindtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
 
 valgrindtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 valgrindtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
 
 valgrindtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
        VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)

 
 gdbtest:: gdbtest-all
 
 gdbtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
 
 gdbtest:: gdbtest-all
 
 gdbtest-quick:: all
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \

-       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 gdbtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
 
 gdbtest-all:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) $(ST_TOUCH) | $(FORMAT_TEST_OUTPUT) --immediate 
+       $(ST_DONE_TEST)

 
 gdbtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
 
 gdbtest-env:: everything
        SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \

-       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv
+       $(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --testenv $(ST_TOUCH)
+       $(ST_DONE_TEST)