selftest: Use filter-xfail for known failures

diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index f7284d3f55a5b460d9380fbdb23f2ef60ba4bf3a..0eafc44146ff17dd6a124076d541ba30d7825a1f 100644 (file)
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -23,9 +23,9 @@ require Exporter;
 
 use strict;
 
-sub parse_results($$$$$)
+sub parse_results($$$$)
 {
-       my ($msg_ops, $statistics, $fh, $expecting_failure, $open_tests) = @_;
+       my ($msg_ops, $statistics, $fh, $open_tests) = @_;
        my $unexpected_ok = 0;
        my $expected_fail = 0;
        my $unexpected_fail = 0;
@@ -39,7 +39,7 @@ sub parse_results($$$$$)
                        push (@$open_tests, $1);
                } elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
                        $msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
-               } elsif (/^(success|successful|failure|fail|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) {
+               } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
                        $msg_ops->control_msg($_);
                        my $reason = undef;
                        if ($3) {
@@ -60,29 +60,18 @@ sub parse_results($$$$$)
                        my $result = $1;
                        if ($1 eq "success" or $1 eq "successful") {
                                pop(@$open_tests); #FIXME: Check that popped value == $2
-                               if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-                                       $statistics->{TESTS_UNEXPECTED_OK}++;
-                                       $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-                                       $unexpected_ok++;
-                               } else {
-                                       $statistics->{TESTS_EXPECTED_OK}++;
-                                       $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-                               }
-                       } elsif ($1 eq "failure" or $1 eq "fail") {
-                               pop(@$open_tests); #FIXME: Check that popped value == $2
-                               if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-                                       $statistics->{TESTS_EXPECTED_FAIL}++;
-                                       $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-                                       $expected_fail++;
-                               } else {
-                                       $statistics->{TESTS_UNEXPECTED_FAIL}++;
-                                       $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-                                       $unexpected_fail++;
-                               }
-                       } elsif ($1 eq "knownfail") {
+                               $statistics->{TESTS_EXPECTED_OK}++;
+                               $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+                       } elsif ($1 eq "xfail" or $1 eq "knownfail") {
                                pop(@$open_tests); #FIXME: Check that popped value == $2
                                $statistics->{TESTS_EXPECTED_FAIL}++;
                                $msg_ops->end_test($open_tests, $2, $1, 0, $reason);
+                               $expected_fail++;
+                       } elsif ($1 eq "failure" or $1 eq "fail") {
+                               pop(@$open_tests); #FIXME: Check that popped value == $2
+                               $statistics->{TESTS_UNEXPECTED_FAIL}++;
+                               $msg_ops->end_test($open_tests, $2, $1, 1, $reason);
+                               $unexpected_fail++;
                        } elsif ($1 eq "skip") {
                                $statistics->{TESTS_SKIP}++;
                                pop(@$open_tests); #FIXME: Check that popped value == $2

diff --git a/selftest/filter-xfail.pl b/selftest/filter-xfail.pl
index 0e0c4e22bcafd920d921d826a9a81b79f8081479..edf0e050a7159989fccc7fc0bd74ef5b38cadbe2 100755 (executable)
--- a/selftest/filter-xfail.pl
+++ b/selftest/filter-xfail.pl
@@ -157,6 +157,6 @@ sub end_test($$$$$)
 my $msg_ops = {};
 bless $msg_ops;
 
-parse_results($msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+parse_results($msg_ops, $statistics, *STDIN, []);
 
 0;

diff --git a/selftest/format-subunit.pl b/selftest/format-subunit.pl
index 000346a9affd2b5f75b2cac92f248b4d76bd4cbf..c60902f8d755ac028bdec1ee50033b196d792a6e 100755 (executable)
--- a/selftest/format-subunit.pl
+++ b/selftest/format-subunit.pl
@@ -61,7 +61,7 @@ if ($opt_format eq "buildfarm") {
 }
 
 my $expected_ret = parse_results(
-       $msg_ops, $statistics, *STDIN, sub { return 0; }, []);
+       $msg_ops, $statistics, *STDIN, []);
 
 $msg_ops->summary();
 

diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index b786c9a50f549d1bd15f0b3b080c4619696be25d..8a2c8804bc6380e6772e422e4c4d6e2db2848366 100755 (executable)
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -77,23 +77,10 @@ when the server is running locally.
 Will prevent TCP and UDP ports being opened on the local host but 
 (transparently) redirects these calls to use unix domain sockets.
 
-=item I<--expected-failures>
-
-Specify a file containing a list of tests that are expected to fail. Failures for 
-these tests will be counted as successes, successes will be counted as failures.
-
-The format for the file is, one entry per line:
-
-TESTSUITE-NAME.TEST-NAME
-
-The reason for a test can also be specified, by adding a hash sign (#) and the reason 
-after the test name.
-
 =item I<--exclude>
 
 Specify a file containing a list of tests that should be skipped. Possible 
-candidates are tests that segfault the server, flip or don't end. The format of this file is the same as 
-for the --expected-failures flag.
+candidates are tests that segfault the server, flip or don't end. 
 
 =item I<--include>
 
@@ -157,7 +144,6 @@ my $opt_socket_wrapper_pcap = undef;
 my $opt_socket_wrapper_keep_pcap = undef;
 my $opt_one = 0;
 my $opt_immediate = 0;
-my $opt_expected_failures = undef;
 my @opt_exclude = ();
 my @opt_include = ();
 my $opt_verbose = 0;
@@ -176,7 +162,6 @@ my $builddir = ".";
 my $exeext = "";
 my $prefix = "./st";
 
-my @expected_failures = ();
 my @includes = ();
 my @excludes = ();
 
@@ -205,12 +190,6 @@ sub find_in_list($$)
        return undef;
 }
 
-sub expecting_failure($)
-{
-       my ($name) = @_;
-       return find_in_list(\@expected_failures, $name);
-}
-
 sub skip($)
 {
        my ($name) = @_;
@@ -265,7 +244,7 @@ sub run_testsuite($$$$$$)
        }
 
        my $expected_ret = parse_results(
-               $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
+               $msg_ops, $statistics, *RESULT, [$name]);
 
        my $envlog = getlog_env($envname);
        $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
@@ -323,7 +302,6 @@ Target Specific:
                             failed
  --socket-wrapper           enable socket wrapper
  --bindir=PATH              path to target binaries
- --expected-failures=FILE   specify list of tests that is guaranteed to fail
 
 Samba4 Specific:
  --ldap=openldap|fedora-ds  back samba onto specified ldap server
@@ -351,7 +329,6 @@ my $result = GetOptions (
                'quick' => \$opt_quick,
                'one' => \$opt_one,
                'immediate' => \$opt_immediate,
-               'expected-failures=s' => \$opt_expected_failures,
                'exclude=s' => \@opt_exclude,
                'include=s' => \@opt_include,
                'srcdir=s' => \$srcdir,
@@ -537,10 +514,6 @@ sub read_test_regexes($)
        return @ret;
 }
 
-if (defined($opt_expected_failures)) {
-       @expected_failures = read_test_regexes($opt_expected_failures);
-}
-
 foreach (@opt_exclude) {
        push (@excludes, read_test_regexes($_));
 }

diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 033fd7938d09208ff110b840d9b8d0352b8b5266..0b1f340f3219ca909ba9ad0a174e24a77d9d86e9 100644 (file)
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -3,14 +3,17 @@ TEST_FORMAT = plain
 SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
     $(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
     --builddir=$(builddir) --srcdir=$(srcdir) \
-    --expected-failures=$(srcdir)/selftest/knownfail \
        --format=subunit \
     --exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
     $(TEST_OPTIONS) 
 
 SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
 SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
-FORMAT_TEST_OUTPUT = $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+FILTER_XFAIL = $(PERL) $(selftestdir)/filter-xfail.pl --expected-failures=$(srcdir)/selftest/knownfail
+FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
+
+subunittest:: everything
+       $(SELFTEST) $(TESTS)
 
 slowtest:: everything
        $(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate