2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
61 Show errors as soon as they happen rather than at the end of the test run.
63 =item I<--target samba4|samba3|win|kvm>
65 Specify test target against which to run. Default is 'samba4'.
69 Run only a limited number of tests. Intended to run in about 30 seconds on
70 moderately recent systems.
72 =item I<--socket-wrapper>
74 Use socket wrapper library for communication with server. Only works
75 when the server is running locally.
77 Will prevent TCP and UDP ports being opened on the local host but
78 (transparently) redirects these calls to use unix domain sockets.
80 =item I<--expected-failures>
82 Specify a file containing a list of tests that are expected to fail. Failures for
83 these tests will be counted as successes, successes will be counted as failures.
85 The format for the file is, one entry per line:
87 TESTSUITE-NAME.TEST-NAME
89 The reason for a test can also be specified, by adding a hash sign (#) and the reason
94 Specify a file containing a list of tests that should be skipped. Possible
95 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
96 for the --expected-failures flag.
100 Specify a file containing a list of tests that should be run. Same format
101 as the --exclude flag.
103 Not includes specified means all tests will be run.
107 Abort as soon as one test fails.
111 Load a list of tests from the specified location.
119 =item I<SMBD_VALGRIND>
121 =item I<TORTURE_MAXTIME>
133 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
143 use FindBin qw($RealBin $Script);
147 use Cwd qw(abs_path);
149 use Subunit qw(parse_results);
153 my $opt_target = "samba4";
155 my $opt_socket_wrapper = 0;
156 my $opt_socket_wrapper_pcap = undef;
157 my $opt_socket_wrapper_keep_pcap = undef;
159 my $opt_immediate = 0;
160 my $opt_expected_failures = undef;
161 my @opt_exclude = ();
162 my @opt_include = ();
164 my $opt_image = undef;
167 my $opt_analyse_cmd = undef;
168 my $opt_resetup_env = undef;
169 my $opt_bindir = undef;
170 my $opt_no_lazy_setup = undef;
171 my $opt_format = "plain";
179 my @expected_failures = ();
186 TESTS_UNEXPECTED_OK => 0,
187 TESTS_EXPECTED_OK => 0,
188 TESTS_UNEXPECTED_FAIL => 0,
189 TESTS_EXPECTED_FAIL => 0,
196 my ($list, $fullname) = @_;
199 if ($fullname =~ /$$_[0]/) {
200 return ($$_[1]) if ($$_[1]);
201 return "NO REASON SPECIFIED";
208 sub expecting_failure($)
211 return find_in_list(\@expected_failures, $name);
218 return find_in_list(\@excludes, $name);
227 return unless ($opt_socket_wrapper_pcap);
228 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
231 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
233 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
235 SocketWrapper::setup_pcap($pcap_file);
240 sub cleanup_pcap($$$)
242 my ($pcap_file, $expected_ret, $ret) = @_;
244 return unless ($opt_socket_wrapper_pcap);
245 return if ($opt_socket_wrapper_keep_pcap);
246 return unless ($expected_ret == $ret);
247 return unless defined($pcap_file);
252 sub run_testsuite($$$$$$)
254 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
255 my $pcap_file = setup_pcap($name);
257 $msg_ops->start_test([], $name);
259 unless (open(RESULT, "$cmd 2>&1|")) {
260 $statistics->{TESTS_ERROR}++;
261 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
262 $statistics->{SUITES_FAIL}++;
266 my $expected_ret = parse_results(
267 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
269 my $envlog = getlog_env($envname);
270 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
272 $msg_ops->output_msg("CMD: $cmd\n");
274 my $ret = close(RESULT);
275 $ret = 0 unless $ret == 1;
277 my $exitcode = $? >> 8;
280 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
282 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
285 cleanup_pcap($pcap_file, $expected_ret, $ret);
287 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
288 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
291 if ($ret != $expected_ret) {
292 $statistics->{SUITES_FAIL}++;
293 exit(1) if ($opt_one);
296 return ($ret == $expected_ret);
301 print "Samba test runner
302 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
303 Copyright (C) Stefan Metzmacher <metze\@samba.org>
305 Usage: $Script [OPTIONS] TESTNAME-REGEX
308 --help this help page
309 --target=samba[34]|win|kvm Samba version to target
310 --testlist=FILE file to read available tests from
313 --prefix=DIR prefix to run tests in [st]
314 --srcdir=DIR source directory [.]
315 --builddir=DIR output directory [.]
316 --exeext=EXT executable extention []
319 --socket-wrapper-pcap save traffic to pcap directories
320 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
322 --socket-wrapper enable socket wrapper
323 --bindir=PATH path to target binaries
324 --expected-failures=FILE specify list of tests that is guaranteed to fail
327 --ldap=openldap|fedora-ds back samba onto specified ldap server
330 --image=PATH path to KVM image
333 --quick run quick overall test
334 --one abort when the first test fails
335 --immediate print test output for failed tests during run
337 --analyse-cmd CMD command to run after each test
342 my $result = GetOptions (
343 'help|h|?' => \$opt_help,
344 'target=s' => \$opt_target,
345 'prefix=s' => \$prefix,
346 'socket-wrapper' => \$opt_socket_wrapper,
347 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
348 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
349 'quick' => \$opt_quick,
351 'immediate' => \$opt_immediate,
352 'expected-failures=s' => \$opt_expected_failures,
353 'exclude=s' => \@opt_exclude,
354 'include=s' => \@opt_include,
355 'srcdir=s' => \$srcdir,
356 'builddir=s' => \$builddir,
357 'exeext=s' => \$exeext,
358 'verbose' => \$opt_verbose,
359 'testenv' => \$opt_testenv,
361 'analyse-cmd=s' => \$opt_analyse_cmd,
362 'no-lazy-setup' => \$opt_no_lazy_setup,
363 'resetup-environment' => \$opt_resetup_env,
364 'bindir:s' => \$opt_bindir,
365 'format=s' => \$opt_format,
366 'image=s' => \$opt_image,
367 'testlist=s' => \@testlists
370 exit(1) if (not $result);
372 ShowHelp() if ($opt_help);
376 # quick hack to disable rpc validation when using valgrind - its way too slow
377 unless (defined($ENV{VALGRIND})) {
378 $ENV{VALIDATE} = "validate";
379 $ENV{MALLOC_CHECK_} = 2;
382 my $bindir = ($opt_bindir or "$builddir/bin");
383 my $bindir_abs = abs_path($bindir);
385 # Backwards compatibility:
386 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
387 if (defined($ENV{FEDORA_DS_ROOT})) {
394 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
397 $torture_maxtime *= 2;
404 die("using an empty prefix isn't allowed") unless $prefix ne "";
406 #Ensure we have the test prefix around
407 mkdir($prefix, 0777) unless -d $prefix;
409 my $prefix_abs = abs_path($prefix);
410 my $srcdir_abs = abs_path($srcdir);
411 my $builddir_abs = abs_path($builddir);
413 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
414 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
416 $ENV{PREFIX} = $prefix;
417 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
418 $ENV{PREFIX_ABS} = $prefix_abs;
419 $ENV{SRCDIR} = $srcdir;
420 $ENV{SRCDIR_ABS} = $srcdir_abs;
421 $ENV{BUILDDIR} = $builddir;
422 $ENV{BUILDDIR_ABS} = $builddir_abs;
423 $ENV{EXEEXT} = $exeext;
425 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
426 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
427 $opt_format = "buildfarm";
430 my $tls_enabled = not $opt_quick;
431 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
432 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
433 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
434 sub prefix_pathvar($$)
436 my ($name, $newpath) = @_;
437 if (defined($ENV{$name})) {
438 $ENV{$name} = "$newpath:$ENV{$name}";
440 $ENV{$name} = $newpath;
443 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
444 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
446 if ($opt_socket_wrapper_keep_pcap) {
447 # Socket wrapper keep pcap implies socket wrapper pcap
448 $opt_socket_wrapper_pcap = 1;
451 if ($opt_socket_wrapper_pcap) {
452 # Socket wrapper pcap implies socket wrapper
453 $opt_socket_wrapper = 1;
456 my $socket_wrapper_dir;
457 if ($opt_socket_wrapper) {
458 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
459 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
462 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
467 my $testenv_default = "none";
469 if ($opt_target eq "samba4") {
470 $testenv_default = "member";
471 require target::Samba4;
472 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
473 } elsif ($opt_target eq "samba3") {
474 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
475 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
477 $testenv_default = "member";
478 require target::Samba3;
479 $target = new Samba3($bindir);
480 } elsif ($opt_target eq "win") {
481 die("Windows tests will not run with socket wrapper enabled.")
482 if ($opt_socket_wrapper);
483 $testenv_default = "dc";
484 require target::Windows;
485 $target = new Windows();
486 } elsif ($opt_target eq "kvm") {
487 die("Kvm tests will not run with socket wrapper enabled.")
488 if ($opt_socket_wrapper);
490 die("No image specified") unless ($opt_image);
491 $target = new Kvm($opt_image, undef);
495 # Start a Virtual Distributed Ethernet Switch
496 # Returns the pid of the switch.
498 sub start_vde_switch($)
502 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
504 open(PID, "$path/vde.pid");
512 # Stop a Virtual Distributed Ethernet Switch
513 sub stop_vde_switch($)
519 sub read_test_regexes($)
523 open(LF, "<$name") or die("unable to read $name: $!");
527 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
528 push (@ret, [$1, $4]);
530 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
531 push (@ret, [$_, undef]);
538 if (defined($opt_expected_failures)) {
539 @expected_failures = read_test_regexes($opt_expected_failures);
542 foreach (@opt_exclude) {
543 push (@excludes, read_test_regexes($_));
546 foreach (@opt_include) {
547 push (@includes, read_test_regexes($_));
550 my $interfaces = join(',', ("127.0.0.6/8",
557 my $conffile = "$prefix_abs/client/client.conf";
558 $ENV{SMB_CONF_PATH} = $conffile;
560 sub write_clientconf($$)
562 my ($conffile, $vars) = @_;
564 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
566 if ( -d "$prefix/client/private" ) {
567 unlink <$prefix/client/private/*>;
569 mkdir("$prefix/client/private", 0777);
572 if ( -d "$prefix/client/lock" ) {
573 unlink <$prefix/client/lockdir/*>;
575 mkdir("$prefix/client/lockdir", 0777);
578 open(CF, ">$conffile");
579 print CF "[global]\n";
580 if (defined($ENV{VALGRIND})) {
581 print CF "\ticonv:native = true\n";
583 print CF "\ticonv:native = false\n";
585 print CF "\tnetbios name = client\n";
586 if (defined($vars->{DOMAIN})) {
587 print CF "\tworkgroup = $vars->{DOMAIN}\n";
589 if (defined($vars->{REALM})) {
590 print CF "\trealm = $vars->{REALM}\n";
592 if ($opt_socket_wrapper) {
593 print CF "\tinterfaces = $interfaces\n";
596 private dir = $prefix_abs/client/private
597 lock dir = $prefix_abs/client/lockdir
598 name resolve order = bcast
599 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
601 notify:inotify = false
603 system:anonymous = true
604 client lanman auth = Yes
605 torture:basedir = $prefix_abs/client
606 #We don't want to pass our self-tests if the PAC code is wrong
607 gensec:require_pac = true
608 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
615 my $testsdir = "$srcdir/selftest";
617 my %required_envs = ();
624 open(IN, $filename) or die("Unable to open $filename: $!");
627 if ($_ eq "-- TEST --\n") {
634 if (not defined($tests) or $name =~ /$tests/) {
635 $required_envs{$env} = 1;
636 push (@ret, [$name, $env, $cmdline]);
642 close(IN) or die("Error creating recipe");
646 if ($#testlists == -1) {
647 die("No testlists specified");
650 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
651 if ($opt_socket_wrapper) {
652 $ENV{SELFTEST_INTERFACES} = $interfaces;
654 $ENV{SELFTEST_INTERFACES} = "";
657 $ENV{SELFTEST_VERBOSE} = "1";
659 $ENV{SELFTEST_VERBOSE} = "";
662 $ENV{SELFTEST_QUICK} = "1";
664 $ENV{SELFTEST_QUICK} = "";
666 $ENV{SELFTEST_TARGET} = $opt_target;
667 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
670 foreach my $fn (@testlists) {
671 foreach (read_testlist($fn)) {
673 next if (@includes and not find_in_list(\@includes, $name));
674 push (@available, $_);
679 if ($opt_format eq "buildfarm") {
680 require output::buildfarm;
681 $msg_ops = new output::buildfarm($statistics);
682 } elsif ($opt_format eq "plain") {
683 require output::plain;
684 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
685 } elsif ($opt_format eq "html") {
686 require output::html;
687 mkdir("test-results", 0777);
688 $msg_ops = new output::html("test-results", $statistics);
689 } elsif ($opt_format eq "subunit") {
690 require output::subunit;
691 $msg_ops = new output::subunit();
693 die("Invalid output format '$opt_format'");
697 foreach (@available) {
699 my $skipreason = skip($name);
701 $msg_ops->skip_testsuite($name, $skipreason);
708 print STDERR "No tests to run\n";
712 my $suitestotal = $#todo + 1;
716 my %running_envs = ();
718 sub get_running_env($)
726 return $running_envs{$envname};
729 my @exported_envvars = (
734 # domain controller stuff
754 "WINBINDD_SOCKET_DIR",
755 "WINBINDD_PRIV_PIPE_DIR"
758 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
760 teardown_env($_) foreach(keys %running_envs);
761 die("Received signal $signame");
768 my $testenv_vars = undef;
774 $option =~ s/^[^:]*//;
777 $option = "client" if $option eq "";
779 if ($envname eq "none") {
781 } elsif (defined(get_running_env($envname))) {
782 $testenv_vars = get_running_env($envname);
783 if (not $target->check_env($testenv_vars)) {
784 $testenv_vars = undef;
787 $testenv_vars = $target->setup_env($envname, $prefix);
790 return undef unless defined($testenv_vars);
792 $running_envs{$envname} = $testenv_vars;
794 if ($option eq "local") {
795 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
796 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
797 } elsif ($option eq "client") {
798 SocketWrapper::set_default_iface(6);
799 write_clientconf($conffile, $testenv_vars);
800 $ENV{SMB_CONF_PATH} = $conffile;
802 die("Unknown option[$option] for envname[$envname]");
805 foreach (@exported_envvars) {
806 if (defined($testenv_vars->{$_})) {
807 $ENV{$_} = $testenv_vars->{$_};
813 return $testenv_vars;
816 sub exported_envvars_str($)
818 my ($testenv_vars) = @_;
821 foreach (@exported_envvars) {
822 next unless defined($testenv_vars->{$_});
823 $out .= $_."=".$testenv_vars->{$_}."\n";
832 return "" if ($envname eq "none");
833 return $target->getlog_env(get_running_env($envname));
839 return 1 if ($envname eq "none");
840 return $target->check_env(get_running_env($envname));
846 return if ($envname eq "none");
847 $target->teardown_env(get_running_env($envname));
848 delete $running_envs{$envname};
851 if ($opt_no_lazy_setup) {
852 setup_env($_) foreach (keys %required_envs);
856 my $testenv_name = $ENV{SELFTEST_TESTENV};
857 $testenv_name = $testenv_default unless defined($testenv_name);
859 my $testenv_vars = setup_env($testenv_name);
861 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
863 my $envvarstr = exported_envvars_str($testenv_vars);
865 my $term = ($ENV{TERM} or "xterm");
866 system("$term -e 'echo -e \"
867 Welcome to the Samba4 Test environment '$testenv_name'
869 This matches the client environment used in make test
870 server is pid `cat \$PIDDIR/samba.pid`
872 Some useful environment variables:
873 TORTURE_OPTIONS=\$TORTURE_OPTIONS
874 SMB_CONF_PATH=\$SMB_CONF_PATH
877 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
878 teardown_env($testenv_name);
883 $cmd =~ s/([\(\)])/\\$1/g;
885 my $envname = $$_[1];
887 my $envvars = setup_env($envname);
888 if (not defined($envvars)) {
889 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
893 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
896 if (defined($opt_analyse_cmd)) {
897 system("$opt_analyse_cmd \"$name\"");
900 teardown_env($envname) if ($opt_resetup_env);
906 teardown_env($_) foreach (keys %running_envs);
914 # if there were any valgrind failures, show them
915 foreach (<$prefix/valgrind.log*>) {
917 system("grep DWARF2.CFI.reader $_ > /dev/null");
919 print "VALGRIND FAILURE\n";
925 if ($opt_format eq "buildfarm") {
926 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
929 exit $statistics->{SUITES_FAIL};