2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
44 Change directory to run tests in. Default is 'st'.
48 Show errors as soon as they happen rather than at the end of the test run.
50 =item I<--target samba4|samba3|win|kvm>
52 Specify test target against which to run. Default is 'samba4'.
56 Run only a limited number of tests. Intended to run in about 30 seconds on
57 moderately recent systems.
59 =item I<--socket-wrapper>
61 Use socket wrapper library for communication with server. Only works
62 when the server is running locally.
64 Will prevent TCP and UDP ports being opened on the local host but
65 (transparently) redirects these calls to use unix domain sockets.
67 =item I<--expected-failures>
69 Specify a file containing a list of tests that are expected to fail. Failures for
70 these tests will be counted as successes, successes will be counted as failures.
72 The format for the file is, one entry per line:
74 TESTSUITE-NAME.TEST-NAME
76 The reason for a test can also be specified, by adding a hash sign (#) and the reason
81 Specify a file containing a list of tests that should be skipped. Possible
82 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
83 for the --expected-failures flag.
87 Specify a file containing a list of tests that should be run. Same format
88 as the --exclude flag.
90 Not includes specified means all tests will be run.
94 Abort as soon as one test fails.
98 Load a list of tests from the specified location.
106 =item I<SMBD_VALGRIND>
108 =item I<TORTURE_MAXTIME>
120 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
130 use FindBin qw($RealBin $Script);
134 use Cwd qw(abs_path);
136 use Subunit qw(parse_results);
140 my $opt_target = "samba4";
142 my $opt_socket_wrapper = 0;
143 my $opt_socket_wrapper_pcap = undef;
144 my $opt_socket_wrapper_keep_pcap = undef;
146 my $opt_immediate = 0;
147 my $opt_expected_failures = undef;
148 my @opt_exclude = ();
149 my @opt_include = ();
151 my $opt_image = undef;
154 my $opt_analyse_cmd = undef;
155 my $opt_resetup_env = undef;
156 my $opt_bindir = undef;
157 my $opt_no_lazy_setup = undef;
158 my $opt_format = "plain";
166 my @expected_failures = ();
173 TESTS_UNEXPECTED_OK => 0,
174 TESTS_EXPECTED_OK => 0,
175 TESTS_UNEXPECTED_FAIL => 0,
176 TESTS_EXPECTED_FAIL => 0,
183 my ($list, $fullname) = @_;
186 if ($fullname =~ /$$_[0]/) {
187 return ($$_[1]) if ($$_[1]);
188 return "NO REASON SPECIFIED";
195 sub expecting_failure($)
198 return find_in_list(\@expected_failures, $name);
205 return find_in_list(\@excludes, $name);
214 return unless ($opt_socket_wrapper_pcap);
215 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
218 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
220 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
222 SocketWrapper::setup_pcap($pcap_file);
227 sub cleanup_pcap($$$)
229 my ($pcap_file, $expected_ret, $ret) = @_;
231 return unless ($opt_socket_wrapper_pcap);
232 return if ($opt_socket_wrapper_keep_pcap);
233 return unless ($expected_ret == $ret);
234 return unless defined($pcap_file);
239 sub run_testsuite($$$$$$)
241 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
242 my $pcap_file = setup_pcap($name);
244 $msg_ops->start_test([], $name);
246 unless (open(RESULT, "$cmd 2>&1|")) {
247 $statistics->{TESTS_ERROR}++;
248 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
249 $statistics->{SUITES_FAIL}++;
253 my $expected_ret = parse_results(
254 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
256 my $envlog = getlog_env($envname);
257 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
259 $msg_ops->output_msg("CMD: $cmd\n");
261 my $ret = close(RESULT);
262 $ret = 0 unless $ret == 1;
264 my $exitcode = $? >> 8;
267 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
269 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
272 cleanup_pcap($pcap_file, $expected_ret, $ret);
274 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
275 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
278 if ($ret != $expected_ret) {
279 $statistics->{SUITES_FAIL}++;
280 exit(1) if ($opt_one);
283 return ($ret == $expected_ret);
288 print "Samba test runner
289 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
291 Usage: $Script [OPTIONS] TESTNAME-REGEX
294 --help this help page
295 --target=samba[34]|win|kvm Samba version to target
296 --testlist=FILE file to read available tests from
299 --prefix=DIR prefix to run tests in [st]
300 --srcdir=DIR source directory [.]
301 --builddir=DIR output directory [.]
302 --exeext=EXT executable extention []
305 --socket-wrapper-pcap save traffic to pcap directories
306 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
308 --socket-wrapper enable socket wrapper
309 --bindir=PATH path to target binaries
310 --expected-failures=FILE specify list of tests that is guaranteed to fail
313 --ldap=openldap|fedora-ds back samba onto specified ldap server
316 --image=PATH path to KVM image
319 --quick run quick overall test
320 --one abort when the first test fails
321 --immediate print test output for failed tests during run
323 --analyse-cmd CMD command to run after each test
328 my $result = GetOptions (
329 'help|h|?' => \$opt_help,
330 'target=s' => \$opt_target,
331 'prefix=s' => \$prefix,
332 'socket-wrapper' => \$opt_socket_wrapper,
333 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
334 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
335 'quick' => \$opt_quick,
337 'immediate' => \$opt_immediate,
338 'expected-failures=s' => \$opt_expected_failures,
339 'exclude=s' => \@opt_exclude,
340 'include=s' => \@opt_include,
341 'srcdir=s' => \$srcdir,
342 'builddir=s' => \$builddir,
343 'exeext=s' => \$exeext,
344 'verbose' => \$opt_verbose,
345 'testenv' => \$opt_testenv,
347 'analyse-cmd=s' => \$opt_analyse_cmd,
348 'no-lazy-setup' => \$opt_no_lazy_setup,
349 'resetup-environment' => \$opt_resetup_env,
350 'bindir:s' => \$opt_bindir,
351 'format=s' => \$opt_format,
352 'image=s' => \$opt_image,
353 'testlist=s' => \@testlists
356 exit(1) if (not $result);
358 ShowHelp() if ($opt_help);
362 # quick hack to disable rpc validation when using valgrind - its way too slow
363 unless (defined($ENV{VALGRIND})) {
364 $ENV{VALIDATE} = "validate";
365 $ENV{MALLOC_CHECK_} = 2;
368 my $old_pwd = "$RealBin/..";
370 # Backwards compatibility:
371 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
372 if (defined($ENV{FEDORA_DS_ROOT})) {
379 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
382 $torture_maxtime *= 2;
389 die("using an empty prefix isn't allowed") unless $prefix ne "";
391 #Ensure we have the test prefix around
392 mkdir($prefix, 0777) unless -d $prefix;
394 my $prefix_abs = abs_path($prefix);
395 my $srcdir_abs = abs_path($srcdir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX} = $prefix;
401 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS} = $prefix_abs;
403 $ENV{SRCDIR} = $srcdir;
404 $ENV{SRCDIR_ABS} = $srcdir_abs;
406 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
407 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
408 $opt_format = "buildfarm";
411 my $tls_enabled = not $opt_quick;
412 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
413 $ENV{LDB_MODULES_PATH} = "$old_pwd/source4/bin/modules/ldb";
414 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/source4/bin/modules";
415 sub prefix_pathvar($$)
417 my ($name, $newpath) = @_;
418 if (defined($ENV{$name})) {
419 $ENV{$name} = "$newpath:$ENV{$name}";
421 $ENV{$name} = $newpath;
424 prefix_pathvar("PKG_CONFIG_PATH", "$old_pwd/source4/bin/pkgconfig");
425 prefix_pathvar("PYTHONPATH", "$old_pwd/source4/bin/python");
427 if ($opt_socket_wrapper_keep_pcap) {
428 # Socket wrapper keep pcap implies socket wrapper pcap
429 $opt_socket_wrapper_pcap = 1;
432 if ($opt_socket_wrapper_pcap) {
433 # Socket wrapper pcap implies socket wrapper
434 $opt_socket_wrapper = 1;
437 my $socket_wrapper_dir;
438 if ($opt_socket_wrapper) {
439 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
440 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
443 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
448 my $testenv_default = "none";
450 if ($opt_target eq "samba4") {
451 $testenv_default = "member";
452 require target::Samba4;
453 $target = new Samba4($opt_bindir or "$builddir/bin",
454 $ldap, "$srcdir/setup", $exeext);
455 } elsif ($opt_target eq "samba3") {
456 my $bindir = ($opt_bindir or "$builddir/bin");
457 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
458 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
460 $testenv_default = "member";
461 require target::Samba3;
462 $target = new Samba3($bindir);
463 } elsif ($opt_target eq "win") {
464 die("Windows tests will not run with socket wrapper enabled.")
465 if ($opt_socket_wrapper);
466 $testenv_default = "dc";
467 require target::Windows;
468 $target = new Windows();
469 } elsif ($opt_target eq "kvm") {
470 die("Kvm tests will not run with socket wrapper enabled.")
471 if ($opt_socket_wrapper);
473 die("No image specified") unless ($opt_image);
474 $target = new Kvm($opt_image, undef);
478 # Start a Virtual Distributed Ethernet Switch
479 # Returns the pid of the switch.
481 sub start_vde_switch($)
485 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
487 open(PID, "$path/vde.pid");
495 # Stop a Virtual Distributed Ethernet Switch
496 sub stop_vde_switch($)
502 sub read_test_regexes($)
506 open(LF, "<$name") or die("unable to read $name: $!");
510 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
511 push (@ret, [$1, $4]);
513 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
514 push (@ret, [$_, undef]);
521 if (defined($opt_expected_failures)) {
522 @expected_failures = read_test_regexes($opt_expected_failures);
525 foreach (@opt_exclude) {
526 push (@excludes, read_test_regexes($_));
529 foreach (@opt_include) {
530 push (@includes, read_test_regexes($_));
533 my $interfaces = join(',', ("127.0.0.6/8",
540 my $conffile = "$prefix_abs/client/client.conf";
541 $ENV{SMB_CONF_PATH} = $conffile;
543 sub write_clientconf($$)
545 my ($conffile, $vars) = @_;
547 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
549 if ( -d "$prefix/client/private" ) {
550 unlink <$prefix/client/private/*>;
552 mkdir("$prefix/client/private", 0777);
555 if ( -d "$prefix/client/lock" ) {
556 unlink <$prefix/client/lockdir/*>;
558 mkdir("$prefix/client/lockdir", 0777);
561 open(CF, ">$conffile");
562 print CF "[global]\n";
563 if (defined($ENV{VALGRIND})) {
564 print CF "\ticonv:native = true\n";
566 print CF "\ticonv:native = false\n";
568 print CF "\tnetbios name = client\n";
569 if (defined($vars->{DOMAIN})) {
570 print CF "\tworkgroup = $vars->{DOMAIN}\n";
572 if (defined($vars->{REALM})) {
573 print CF "\trealm = $vars->{REALM}\n";
575 if ($opt_socket_wrapper) {
576 print CF "\tinterfaces = $interfaces\n";
579 private dir = $prefix_abs/client/private
580 lock dir = $prefix_abs/client/lockdir
581 name resolve order = bcast
582 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
584 notify:inotify = false
586 system:anonymous = true
587 client lanman auth = Yes
588 torture:basedir = $prefix_abs/client
589 #We don't want to pass our self-tests if the PAC code is wrong
590 gensec:require_pac = true
591 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
598 my $testsdir = "$srcdir/selftest";
600 my %required_envs = ();
607 open(IN, $filename) or die("Unable to open $filename: $!");
610 if ($_ eq "-- TEST --\n") {
617 if (not defined($tests) or $name =~ /$tests/) {
618 $required_envs{$env} = 1;
619 push (@ret, [$name, $env, $cmdline]);
625 close(IN) or die("Error creating recipe");
629 if ($#testlists == -1) {
630 die("No testlists specified");
633 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
634 if ($opt_socket_wrapper) {
635 $ENV{SELFTEST_INTERFACES} = $interfaces;
637 $ENV{SELFTEST_INTERFACES} = "";
640 $ENV{SELFTEST_VERBOSE} = "1";
642 $ENV{SELFTEST_VERBOSE} = "";
645 $ENV{SELFTEST_QUICK} = "1";
647 $ENV{SELFTEST_QUICK} = "";
649 $ENV{SELFTEST_TARGET} = $opt_target;
650 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
651 $ENV{SELFTEST_CONFFILE} = $conffile;
654 foreach my $fn (@testlists) {
655 foreach (read_testlist($fn)) {
657 next if (@includes and not find_in_list(\@includes, $name));
658 push (@available, $_);
663 if ($opt_format eq "buildfarm") {
664 require output::buildfarm;
665 $msg_ops = new output::buildfarm($statistics);
666 } elsif ($opt_format eq "plain") {
667 require output::plain;
668 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
669 } elsif ($opt_format eq "html") {
670 require output::html;
671 mkdir("test-results", 0777);
672 $msg_ops = new output::html("test-results", $statistics);
674 die("Invalid output format '$opt_format'");
678 foreach (@available) {
680 my $skipreason = skip($name);
682 $msg_ops->skip_testsuite($name, $skipreason);
689 print STDERR "No tests to run\n";
693 my $suitestotal = $#todo + 1;
697 my %running_envs = ();
699 sub get_running_env($)
707 return $running_envs{$envname};
710 my @exported_envvars = (
715 # domain controller stuff
735 "WINBINDD_SOCKET_DIR",
736 "WINBINDD_PRIV_PIPE_DIR"
739 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
741 teardown_env($_) foreach(keys %running_envs);
742 die("Received signal $signame");
749 my $testenv_vars = undef;
755 $option =~ s/^[^:]*//;
758 $option = "client" if $option eq "";
760 if ($envname eq "none") {
762 } elsif (defined(get_running_env($envname))) {
763 $testenv_vars = get_running_env($envname);
764 if (not $target->check_env($testenv_vars)) {
765 $testenv_vars = undef;
768 $testenv_vars = $target->setup_env($envname, $prefix);
771 return undef unless defined($testenv_vars);
773 $running_envs{$envname} = $testenv_vars;
775 if ($option eq "local") {
776 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
777 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
778 } elsif ($option eq "client") {
779 SocketWrapper::set_default_iface(6);
780 write_clientconf($conffile, $testenv_vars);
781 $ENV{SMB_CONF_PATH} = $conffile;
783 die("Unknown option[$option] for envname[$envname]");
786 foreach (@exported_envvars) {
787 if (defined($testenv_vars->{$_})) {
788 $ENV{$_} = $testenv_vars->{$_};
794 return $testenv_vars;
797 sub exported_envvars_str($)
799 my ($testenv_vars) = @_;
802 foreach (@exported_envvars) {
803 next unless defined($testenv_vars->{$_});
804 $out .= $_."=".$testenv_vars->{$_}."\n";
813 return "" if ($envname eq "none");
814 return $target->getlog_env(get_running_env($envname));
820 return 1 if ($envname eq "none");
821 return $target->check_env(get_running_env($envname));
827 return if ($envname eq "none");
828 $target->teardown_env(get_running_env($envname));
829 delete $running_envs{$envname};
832 if ($opt_no_lazy_setup) {
833 setup_env($_) foreach (keys %required_envs);
837 my $testenv_name = $ENV{SELFTEST_TESTENV};
838 $testenv_name = $testenv_default unless defined($testenv_name);
840 my $testenv_vars = setup_env($testenv_name);
842 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
844 my $envvarstr = exported_envvars_str($testenv_vars);
846 my $term = ($ENV{TERM} or "xterm");
847 system("$term -e 'echo -e \"
848 Welcome to the Samba4 Test environment '$testenv_name'
850 This matches the client environment used in make test
851 server is pid `cat \$PIDDIR/samba.pid`
853 Some useful environment variables:
854 TORTURE_OPTIONS=\$TORTURE_OPTIONS
855 CONFIGURATION=\$CONFIGURATION
858 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
859 teardown_env($testenv_name);
864 $cmd =~ s/([\(\)])/\\$1/g;
866 my $envname = $$_[1];
868 my $envvars = setup_env($envname);
869 if (not defined($envvars)) {
870 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
874 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
877 if (defined($opt_analyse_cmd)) {
878 system("$opt_analyse_cmd \"$name\"");
881 teardown_env($envname) if ($opt_resetup_env);
887 teardown_env($_) foreach (keys %running_envs);
895 # if there were any valgrind failures, show them
896 foreach (<$prefix/valgrind.log*>) {
898 system("grep DWARF2.CFI.reader $_ > /dev/null");
900 print "VALGRIND FAILURE\n";
906 if ($opt_format eq "buildfarm") {
907 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
910 exit $statistics->{SUITES_FAIL};