import subunithelper
parser = optparse.OptionParser("filter-subunit [options] < instream > outstream")
-parser.add_option("--expected-failures", type="string",
- help="File containing list of regexes matching tests to consider known "
- "failures")
-parser.add_option("--flapping", type="string",
- help="File containing list of flapping tests, of which to ignore results.")
+parser.add_option("--expected-failures", type="string", action="append",
+ help=("File or directory containing lists of regexes matching tests "
+ "to consider known failures"))
+parser.add_option("--flapping", type="string", action="append",
+ help=("File or directory containing lists of flapping tests, "
+ "of which to ignore results."))
parser.add_option("--strip-passed-output", action="store_true",
help="Whether to strip output from tests that passed")
parser.add_option("--fail-immediately", action="store_true",
sys.exit(1)
if opts.expected_failures:
- expected_failures = subunithelper.read_test_regexes(opts.expected_failures)
+ expected_failures = subunithelper.read_test_regexes(*opts.expected_failures)
else:
expected_failures = {}
if opts.flapping:
- flapping = subunithelper.read_test_regexes(opts.flapping)
+ flapping = subunithelper.read_test_regexes(*opts.flapping)
else:
flapping = {}
--- /dev/null
+# Files in this directory contain lists of regular expressions
+# matching the names of tests that are that are flapping. In other
+# words, they sometimes succeed and sometimes fail, depending on
+# external factors.
+#
+# "make test" will not report failures or successes for tests listed here.
+#
+# DO NOT ADD TESTS HERE UNLESS THEY ARE ACTUALLY FLAPPING
+#
+# It is much better to add known failing tests to 'knownfail', so the
+# test system can warn when they actually start passing.
+#
+# Empty lines and lines begining with '#' are ignored.
+# Please don't add tests to this README!
--- /dev/null
+# Files in this directory contain lists of regular expressions
+# matching the names of tests that are temporarily expected to fail.
+#
+# "make test" will not report failures for tests listed here and will consider
+# a successful run for any of these tests an error.
+#
+# Empty lines and lines begining with '#' are ignored.
+# Please don't add tests to this README!
import datetime
import re
import sys
+import os
from samba import subunit
from samba.subunit.run import TestProtocolClient
from samba.subunit import iso8601
self._stream.write(msg)
-def read_test_regexes(name):
+def read_test_regexes(*names):
ret = {}
- f = open(name, 'r')
- try:
- for l in f:
- l = l.strip()
- if l == "" or l[0] == "#":
- continue
- if "#" in l:
- (regex, reason) = l.split("#", 1)
- ret[regex.strip()] = reason.strip()
- else:
- ret[l] = None
- finally:
- f.close()
+ files = []
+ for name in names:
+ # if we are given a directory, we read all the files it contains
+ # (except the ones that end with "~").
+ if os.path.isdir(name):
+ files.extend([os.path.join(name, x)
+ for x in os.listdir(name)
+ if x[-1] != '~'])
+ else:
+ files.append(name)
+
+ for filename in files:
+ f = open(filename, 'r')
+ try:
+ for l in f:
+ l = l.strip()
+ if l == "" or l[0] == "#":
+ continue
+ if "#" in l:
+ (regex, reason) = l.split("#", 1)
+ ret[regex.strip()] = reason.strip()
+ else:
+ ret[l] = None
+ finally:
+ f.close()
return ret
env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit-json --prefix=${SELFTEST_PREFIX}'
else:
env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
- env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
+ env.FILTER_XFAIL = ('${PYTHON} -u ${srcdir}/selftest/filter-subunit '
+ '--expected-failures=${srcdir}/selftest/knownfail '
+ '--expected-failures=${srcdir}/selftest/knownfail.d '
+ '--flapping=${srcdir}/selftest/flapping '
+ '--flapping=${srcdir}/selftest/flapping.d')
if Options.options.FAIL_IMMEDIATELY:
env.FILTER_XFAIL += ' --fail-immediately'