selftest: Split out flapping tests into a separate file.

diff --git a/selftest/flapping b/selftest/flapping
new file mode 100644 (file)
index 0000000..8628936
--- /dev/null
+++ b/selftest/flapping
@@ -0,0 +1,19 @@
+# This file contains a list of regular expressions matching the names of
+# tests that are flapping. In other words, they sometimes succeed and
+# sometimes fail, depending on external factors.
+#
+# "make test" will not report failures or successes for tests listed here.
+#
+# DO NOT ADD TESTS HERE UNLESS THEY ARE ACTUALLY FLAPPING
+#
+# It is much better to add known failing tests to 'knownfail', so the
+# test system can wwarn when they actually start passing.
+^samba3.raw.mux.* #This test is flaky on the async lock time
+^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
+^samba4.nbt.winsreplication.owned # fails sometimes, timing related
+^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
+.*printer.*print_test_extended # fails on some hosts due to timing issues ?
+.*printer.*print_test # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
+^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure

diff --git a/selftest/knownfail b/selftest/knownfail
index 6f838818680266598f4eed75a2a8660dc8bb8ebf..c49746c0a805e4a294b59b3e0568726c1864a2e8 100644 (file)
--- a/selftest/knownfail
+++ b/selftest/knownfail
@@ -5,15 +5,7 @@
 # a successful run for any of these tests an error.
 
 ^samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
-.*printer.*print_test_extended # fails on some hosts due to timing issues ?
-.*printer.*print_test # fails on some hosts due to timing issues ?
-^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
-^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
-^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
-^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
 .*driver.add_driver_timestamps # we only can store dates, not timestamps
-^samba3.raw.mux.* #This test is flaky on the async lock time
-^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
 ^samba3.smbtorture_s3.LOCAL-MEMCACHE #fails
 ^samba3.smbtorture_s3.LOCAL-TALLOC-DICT #fails
 ^samba3.posix_s3.nbt.dgram.*netlogon2
@@ -104,7 +96,6 @@
 ^samba4.smb2.compound.*.invalid2
 ^samba4.ldap.acl.*.search.* # ACL search behaviour not enabled by default
 ^samba4.ldap.acl.*.ntSecurityDescriptor.* # ACL extended checks on search not enabled by default
-^samba4.nbt.winsreplication.owned # fails sometimes, timing related
 ^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.test_dirsync_deleted_items
 #^samba4.ldap.dirsync.python.dc..__main__.ExtendedDirsyncTests.*
 ^samba4.drs.fsmo.python

diff --git a/selftest/wscript b/selftest/wscript
index 5a2cbd1c0304721682e978f3261ea2950b762416..e1308e16b0bfac4cbf4c00059802a9cf36cfbded 100644 (file)
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -88,7 +88,7 @@ def cmd_testonly(opt):
     env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
     if not env.SUBUNIT_FORMATTER:
         env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
-    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail'
+    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping'
 
     if Options.options.FAIL_IMMEDIATELY:
         env.FILTER_XFAIL += ' --fail-immediately'

diff --git a/source3/selftest/s3-selftest.sh b/source3/selftest/s3-selftest.sh
index 4943b1e2a62403710bcce0fdcac776cc3e22764a..db0a8a0ac15e36729f8127cc5051fc9946a1bd1e 100755 (executable)
--- a/source3/selftest/s3-selftest.sh
+++ b/source3/selftest/s3-selftest.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail"
+FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --expected-failures=${SELFTESTDIR}/flapping"
 if [ "x${SUBUNIT_FORMATTER}" = x"" ]; then
        SUBUNIT_FORMATTER="${PYTHON} -u ${SELFTESTDIR}/format-subunit --prefix=${SELFTESTPREFIX} --immediate"
 fi