2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
44 Change directory to run tests in. Default is 'st'.
48 Show errors as soon as they happen rather than at the end of the test run.
50 =item I<--target samba4|samba3|win|kvm>
52 Specify test target against which to run. Default is 'samba4'.
56 Run only a limited number of tests. Intended to run in about 30 seconds on
57 moderately recent systems.
59 =item I<--socket-wrapper>
61 Use socket wrapper library for communication with server. Only works
62 when the server is running locally.
64 Will prevent TCP and UDP ports being opened on the local host but
65 (transparently) redirects these calls to use unix domain sockets.
67 =item I<--expected-failures>
69 Specify a file containing a list of tests that are expected to fail. Failures for
70 these tests will be counted as successes, successes will be counted as failures.
72 The format for the file is, one entry per line:
74 TESTSUITE-NAME.TEST-NAME
76 The reason for a test can also be specified, by adding a hash sign (#) and the reason
81 Specify a file containing a list of tests that should be skipped. Possible
82 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
83 for the --expected-failures flag.
87 Specify a file containing a list of tests that should be run. Same format
88 as the --exclude flag.
90 Not includes specified means all tests will be run.
94 Abort as soon as one test fails.
98 Load a list of tests from the specified location.
106 =item I<SMBD_VALGRIND>
108 =item I<TORTURE_MAXTIME>
120 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
130 use FindBin qw($RealBin $Script);
134 use Cwd qw(abs_path);
136 use Subunit qw(parse_results);
140 my $opt_target = "samba4";
142 my $opt_socket_wrapper = 0;
143 my $opt_socket_wrapper_pcap = undef;
144 my $opt_socket_wrapper_keep_pcap = undef;
146 my $opt_immediate = 0;
147 my $opt_expected_failures = undef;
148 my @opt_exclude = ();
149 my @opt_include = ();
151 my $opt_image = undef;
154 my $opt_analyse_cmd = undef;
155 my $opt_resetup_env = undef;
156 my $opt_bindir = undef;
157 my $opt_no_lazy_setup = undef;
158 my $opt_format = "plain";
166 my @expected_failures = ();
173 TESTS_UNEXPECTED_OK => 0,
174 TESTS_EXPECTED_OK => 0,
175 TESTS_UNEXPECTED_FAIL => 0,
176 TESTS_EXPECTED_FAIL => 0,
183 my ($list, $fullname) = @_;
186 if ($fullname =~ /$$_[0]/) {
187 return ($$_[1]) if ($$_[1]);
188 return "NO REASON SPECIFIED";
195 sub expecting_failure($)
198 return find_in_list(\@expected_failures, $name);
205 return find_in_list(\@excludes, $name);
214 return unless ($opt_socket_wrapper_pcap);
215 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
218 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
220 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
222 SocketWrapper::setup_pcap($pcap_file);
227 sub cleanup_pcap($$$)
229 my ($pcap_file, $expected_ret, $ret) = @_;
231 return unless ($opt_socket_wrapper_pcap);
232 return if ($opt_socket_wrapper_keep_pcap);
233 return unless ($expected_ret == $ret);
234 return unless defined($pcap_file);
239 sub run_testsuite($$$$$$)
241 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
242 my $pcap_file = setup_pcap($name);
244 $msg_ops->start_test([], $name);
246 unless (open(RESULT, "$cmd 2>&1|")) {
247 $statistics->{TESTS_ERROR}++;
248 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
249 $statistics->{SUITES_FAIL}++;
253 my $expected_ret = parse_results(
254 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
256 my $envlog = getlog_env($envname);
257 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
259 $msg_ops->output_msg("CMD: $cmd\n");
261 my $ret = close(RESULT);
262 $ret = 0 unless $ret == 1;
264 my $exitcode = $? >> 8;
267 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
269 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
272 cleanup_pcap($pcap_file, $expected_ret, $ret);
274 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
275 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
278 if ($ret != $expected_ret) {
279 $statistics->{SUITES_FAIL}++;
280 exit(1) if ($opt_one);
283 return ($ret == $expected_ret);
288 print "Samba test runner
289 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
291 Usage: $Script [OPTIONS] TESTNAME-REGEX
294 --help this help page
295 --target=samba[34]|win|kvm Samba version to target
296 --testlist=FILE file to read available tests from
299 --prefix=DIR prefix to run tests in [st]
300 --srcdir=DIR source directory [.]
301 --builddir=DIR output directory [.]
302 --exeext=EXT executable extention []
305 --socket-wrapper-pcap save traffic to pcap directories
306 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
308 --socket-wrapper enable socket wrapper
309 --bindir=PATH path to target binaries
310 --expected-failures=FILE specify list of tests that is guaranteed to fail
313 --ldap=openldap|fedora-ds back samba onto specified ldap server
316 --image=PATH path to KVM image
319 --quick run quick overall test
320 --one abort when the first test fails
321 --immediate print test output for failed tests during run
323 --analyse-cmd CMD command to run after each test
328 my $result = GetOptions (
329 'help|h|?' => \$opt_help,
330 'target=s' => \$opt_target,
331 'prefix=s' => \$prefix,
332 'socket-wrapper' => \$opt_socket_wrapper,
333 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
334 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
335 'quick' => \$opt_quick,
337 'immediate' => \$opt_immediate,
338 'expected-failures=s' => \$opt_expected_failures,
339 'exclude=s' => \@opt_exclude,
340 'include=s' => \@opt_include,
341 'srcdir=s' => \$srcdir,
342 'builddir=s' => \$builddir,
343 'exeext=s' => \$exeext,
344 'verbose' => \$opt_verbose,
345 'testenv' => \$opt_testenv,
347 'analyse-cmd=s' => \$opt_analyse_cmd,
348 'no-lazy-setup' => \$opt_no_lazy_setup,
349 'resetup-environment' => \$opt_resetup_env,
350 'bindir:s' => \$opt_bindir,
351 'format=s' => \$opt_format,
352 'image=s' => \$opt_image,
353 'testlist=s' => \@testlists
356 exit(1) if (not $result);
358 ShowHelp() if ($opt_help);
362 # quick hack to disable rpc validation when using valgrind - its way too slow
363 unless (defined($ENV{VALGRIND})) {
364 $ENV{VALIDATE} = "validate";
365 $ENV{MALLOC_CHECK_} = 2;
368 my $bindir = ($opt_bindir or "$builddir/bin");
369 my $bindir_abs = abs_path($bindir);
371 # Backwards compatibility:
372 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
373 if (defined($ENV{FEDORA_DS_ROOT})) {
380 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
383 $torture_maxtime *= 2;
390 die("using an empty prefix isn't allowed") unless $prefix ne "";
392 #Ensure we have the test prefix around
393 mkdir($prefix, 0777) unless -d $prefix;
395 my $prefix_abs = abs_path($prefix);
396 my $srcdir_abs = abs_path($srcdir);
397 my $builddir_abs = abs_path($builddir);
399 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
400 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
402 $ENV{PREFIX} = $prefix;
403 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
404 $ENV{PREFIX_ABS} = $prefix_abs;
405 $ENV{SRCDIR} = $srcdir;
406 $ENV{SRCDIR_ABS} = $srcdir_abs;
407 $ENV{BUILDDIR} = $builddir;
408 $ENV{BUILDDIR_ABS} = $builddir_abs;
409 $ENV{EXEEXT} = $exeext;
411 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
412 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
413 $opt_format = "buildfarm";
416 my $tls_enabled = not $opt_quick;
417 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
418 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
419 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
420 sub prefix_pathvar($$)
422 my ($name, $newpath) = @_;
423 if (defined($ENV{$name})) {
424 $ENV{$name} = "$newpath:$ENV{$name}";
426 $ENV{$name} = $newpath;
429 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
430 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
432 if ($opt_socket_wrapper_keep_pcap) {
433 # Socket wrapper keep pcap implies socket wrapper pcap
434 $opt_socket_wrapper_pcap = 1;
437 if ($opt_socket_wrapper_pcap) {
438 # Socket wrapper pcap implies socket wrapper
439 $opt_socket_wrapper = 1;
442 my $socket_wrapper_dir;
443 if ($opt_socket_wrapper) {
444 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
445 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
448 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
453 my $testenv_default = "none";
455 if ($opt_target eq "samba4") {
456 $testenv_default = "member";
457 require target::Samba4;
458 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
459 } elsif ($opt_target eq "samba3") {
460 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
461 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
463 $testenv_default = "member";
464 require target::Samba3;
465 $target = new Samba3($bindir);
466 } elsif ($opt_target eq "win") {
467 die("Windows tests will not run with socket wrapper enabled.")
468 if ($opt_socket_wrapper);
469 $testenv_default = "dc";
470 require target::Windows;
471 $target = new Windows();
472 } elsif ($opt_target eq "kvm") {
473 die("Kvm tests will not run with socket wrapper enabled.")
474 if ($opt_socket_wrapper);
476 die("No image specified") unless ($opt_image);
477 $target = new Kvm($opt_image, undef);
481 # Start a Virtual Distributed Ethernet Switch
482 # Returns the pid of the switch.
484 sub start_vde_switch($)
488 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
490 open(PID, "$path/vde.pid");
498 # Stop a Virtual Distributed Ethernet Switch
499 sub stop_vde_switch($)
505 sub read_test_regexes($)
509 open(LF, "<$name") or die("unable to read $name: $!");
513 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
514 push (@ret, [$1, $4]);
516 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
517 push (@ret, [$_, undef]);
524 if (defined($opt_expected_failures)) {
525 @expected_failures = read_test_regexes($opt_expected_failures);
528 foreach (@opt_exclude) {
529 push (@excludes, read_test_regexes($_));
532 foreach (@opt_include) {
533 push (@includes, read_test_regexes($_));
536 my $interfaces = join(',', ("127.0.0.6/8",
543 my $conffile = "$prefix_abs/client/client.conf";
544 $ENV{SMB_CONF_PATH} = $conffile;
546 sub write_clientconf($$)
548 my ($conffile, $vars) = @_;
550 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
552 if ( -d "$prefix/client/private" ) {
553 unlink <$prefix/client/private/*>;
555 mkdir("$prefix/client/private", 0777);
558 if ( -d "$prefix/client/lock" ) {
559 unlink <$prefix/client/lockdir/*>;
561 mkdir("$prefix/client/lockdir", 0777);
564 open(CF, ">$conffile");
565 print CF "[global]\n";
566 if (defined($ENV{VALGRIND})) {
567 print CF "\ticonv:native = true\n";
569 print CF "\ticonv:native = false\n";
571 print CF "\tnetbios name = client\n";
572 if (defined($vars->{DOMAIN})) {
573 print CF "\tworkgroup = $vars->{DOMAIN}\n";
575 if (defined($vars->{REALM})) {
576 print CF "\trealm = $vars->{REALM}\n";
578 if ($opt_socket_wrapper) {
579 print CF "\tinterfaces = $interfaces\n";
582 private dir = $prefix_abs/client/private
583 lock dir = $prefix_abs/client/lockdir
584 name resolve order = bcast
585 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
587 notify:inotify = false
589 system:anonymous = true
590 client lanman auth = Yes
591 torture:basedir = $prefix_abs/client
592 #We don't want to pass our self-tests if the PAC code is wrong
593 gensec:require_pac = true
594 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
601 my $testsdir = "$srcdir/selftest";
603 my %required_envs = ();
610 open(IN, $filename) or die("Unable to open $filename: $!");
613 if ($_ eq "-- TEST --\n") {
620 if (not defined($tests) or $name =~ /$tests/) {
621 $required_envs{$env} = 1;
622 push (@ret, [$name, $env, $cmdline]);
628 close(IN) or die("Error creating recipe");
632 if ($#testlists == -1) {
633 die("No testlists specified");
636 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
637 if ($opt_socket_wrapper) {
638 $ENV{SELFTEST_INTERFACES} = $interfaces;
640 $ENV{SELFTEST_INTERFACES} = "";
643 $ENV{SELFTEST_VERBOSE} = "1";
645 $ENV{SELFTEST_VERBOSE} = "";
648 $ENV{SELFTEST_QUICK} = "1";
650 $ENV{SELFTEST_QUICK} = "";
652 $ENV{SELFTEST_TARGET} = $opt_target;
653 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
656 foreach my $fn (@testlists) {
657 foreach (read_testlist($fn)) {
659 next if (@includes and not find_in_list(\@includes, $name));
660 push (@available, $_);
665 if ($opt_format eq "buildfarm") {
666 require output::buildfarm;
667 $msg_ops = new output::buildfarm($statistics);
668 } elsif ($opt_format eq "plain") {
669 require output::plain;
670 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
671 } elsif ($opt_format eq "html") {
672 require output::html;
673 mkdir("test-results", 0777);
674 $msg_ops = new output::html("test-results", $statistics);
676 die("Invalid output format '$opt_format'");
680 foreach (@available) {
682 my $skipreason = skip($name);
684 $msg_ops->skip_testsuite($name, $skipreason);
691 print STDERR "No tests to run\n";
695 my $suitestotal = $#todo + 1;
699 my %running_envs = ();
701 sub get_running_env($)
709 return $running_envs{$envname};
712 my @exported_envvars = (
717 # domain controller stuff
737 "WINBINDD_SOCKET_DIR",
738 "WINBINDD_PRIV_PIPE_DIR"
741 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
743 teardown_env($_) foreach(keys %running_envs);
744 die("Received signal $signame");
751 my $testenv_vars = undef;
757 $option =~ s/^[^:]*//;
760 $option = "client" if $option eq "";
762 if ($envname eq "none") {
764 } elsif (defined(get_running_env($envname))) {
765 $testenv_vars = get_running_env($envname);
766 if (not $target->check_env($testenv_vars)) {
767 $testenv_vars = undef;
770 $testenv_vars = $target->setup_env($envname, $prefix);
773 return undef unless defined($testenv_vars);
775 $running_envs{$envname} = $testenv_vars;
777 if ($option eq "local") {
778 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
779 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
780 } elsif ($option eq "client") {
781 SocketWrapper::set_default_iface(6);
782 write_clientconf($conffile, $testenv_vars);
783 $ENV{SMB_CONF_PATH} = $conffile;
785 die("Unknown option[$option] for envname[$envname]");
788 foreach (@exported_envvars) {
789 if (defined($testenv_vars->{$_})) {
790 $ENV{$_} = $testenv_vars->{$_};
796 return $testenv_vars;
799 sub exported_envvars_str($)
801 my ($testenv_vars) = @_;
804 foreach (@exported_envvars) {
805 next unless defined($testenv_vars->{$_});
806 $out .= $_."=".$testenv_vars->{$_}."\n";
815 return "" if ($envname eq "none");
816 return $target->getlog_env(get_running_env($envname));
822 return 1 if ($envname eq "none");
823 return $target->check_env(get_running_env($envname));
829 return if ($envname eq "none");
830 $target->teardown_env(get_running_env($envname));
831 delete $running_envs{$envname};
834 if ($opt_no_lazy_setup) {
835 setup_env($_) foreach (keys %required_envs);
839 my $testenv_name = $ENV{SELFTEST_TESTENV};
840 $testenv_name = $testenv_default unless defined($testenv_name);
842 my $testenv_vars = setup_env($testenv_name);
844 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
846 my $envvarstr = exported_envvars_str($testenv_vars);
848 my $term = ($ENV{TERM} or "xterm");
849 system("$term -e 'echo -e \"
850 Welcome to the Samba4 Test environment '$testenv_name'
852 This matches the client environment used in make test
853 server is pid `cat \$PIDDIR/samba.pid`
855 Some useful environment variables:
856 TORTURE_OPTIONS=\$TORTURE_OPTIONS
857 SMB_CONF_PATH=\$SMB_CONF_PATH
860 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
861 teardown_env($testenv_name);
866 $cmd =~ s/([\(\)])/\\$1/g;
868 my $envname = $$_[1];
870 my $envvars = setup_env($envname);
871 if (not defined($envvars)) {
872 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
876 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
879 if (defined($opt_analyse_cmd)) {
880 system("$opt_analyse_cmd \"$name\"");
883 teardown_env($envname) if ($opt_resetup_env);
889 teardown_env($_) foreach (keys %running_envs);
897 # if there were any valgrind failures, show them
898 foreach (<$prefix/valgrind.log*>) {
900 system("grep DWARF2.CFI.reader $_ > /dev/null");
902 print "VALGRIND FAILURE\n";
908 if ($opt_format eq "buildfarm") {
909 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
912 exit $statistics->{SUITES_FAIL};