source3/selftest/tests.py

   1 #!/usr/bin/python
   2 # This script generates a list of testsuites that should be run as part of
   3 # the Samba 3 test suite.
   4
   5 # The output of this script is parsed by selftest.pl, which then decides
   6 # which of the tests to actually run. It will, for example, skip all tests
   7 # listed in selftest/skip or only run a subset during "make quicktest".
   8
   9 # The idea is that this script outputs all of the tests of Samba 3, not
  10 # just those that are known to pass, and list those that should be skipped
  11 # or are known to fail in selftest/skip or selftest/samba4-knownfail. This makes it
  12 # very easy to see what functionality is still missing in Samba 3 and makes
  13 # it possible to run the testsuite against other servers, such as Samba 4 or
  14 # Windows that have a different set of features.
  15
  16 # The syntax for a testsuite is "-- TEST --" on a single line, followed
  17 # by the name of the test, the environment it needs and the command to run, all
  18 # three separated by newlines. All other lines in the output are considered
  19 # comments.
  20
  21 import os, sys
  22 sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "../../selftest")))
  23 from selftesthelpers import *
  24 import subprocess
  25
  26 smb4torture = binpath("smbtorture4")
  27 samba3srcdir = srcdir() + "/source3"
  28 configuration = "--configfile=$SMB_CONF_PATH"
  29 scriptdir=os.path.join(samba3srcdir, "../script/tests")
  30
  31 torture_options = [configuration, "--maximum-runtime=$SELFTEST_MAXTIME",
  32                    "--target=samba3", "--basedir=$SELFTEST_TMPDIR",
  33                    '--option="torture:winbindd_netbios_name=$SERVER"',
  34                    '--option="torture:winbindd_netbios_domain=$DOMAIN"',
  35                    '--option=torture:sharedelay=100000']
  36
  37 if not os.getenv("SELFTEST_VERBOSE"):
  38     torture_options.append("--option=torture:progress=no")
  39 torture_options.append("--format=subunit")
  40 if os.getenv("SELFTEST_QUICK"):
  41     torture_options.append("--option=torture:quick=yes")
  42 smb4torture += " " + " ".join(torture_options)
  43
  44 def plansmbtorturetestsuite(name, env, options, description=''):
  45     modname = "samba3.posix_s3.%s %s" % (name, description)
  46     cmdline = "%s $LISTOPT %s %s" % (valgrindify(smb4torture), options, name)
  47     plantestsuite_loadlist(modname, env, cmdline)
  48
  49 plantestsuite("samba3.blackbox.success", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_success.sh")])
  50 plantestsuite("samba3.blackbox.failure", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_failure.sh")])
  51
  52 plantestsuite("samba3.local_s3", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_local_s3.sh")])
  53
  54 plantestsuite("samba3.blackbox.registry.upgrade", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_registry_upgrade.sh"), binpath('net'), binpath('dbwrap_tool')])
  55
  56 tests=[ "FDPASS", "LOCK1", "LOCK2", "LOCK3", "LOCK4", "LOCK5", "LOCK6", "LOCK7", "LOCK9",
  57         "UNLINK", "BROWSE", "ATTR", "TRANS2", "TORTURE",
  58         "OPLOCK1", "OPLOCK2", "OPLOCK4", "STREAMERROR",
  59         "DIR", "DIR1", "DIR-CREATETIME", "TCON", "TCONDEV", "RW1", "RW2", "RW3", "RW-SIGNING",
  60         "OPEN", "XCOPY", "RENAME", "DELETE", "DELETE-LN", "PROPERTIES", "W2K",
  61         "TCON2", "IOCTL", "CHKPATH", "FDSESS", "LOCAL-SUBSTITUTE", "CHAIN1", "CHAIN2",
  62         "GETADDRINFO", "POSIX", "UID-REGRESSION-TEST", "SHORTNAME-TEST",
  63         "LOCAL-BASE64", "LOCAL-GENCACHE", "POSIX-APPEND",
  64         "CASE-INSENSITIVE-CREATE", "SMB2-BASIC", "NTTRANS-FSCTL", "SMB2-NEGPROT",
  65         "BAD-NBT-SESSION"]
  66
  67 for t in tests:
  68     plantestsuite("samba3.smbtorture_s3.plain(s3dc).%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "", "-l $LOCAL_PATH"])
  69     plantestsuite("samba3.smbtorture_s3.crypt(s3dc).%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "-e", "-l $LOCAL_PATH"])
  70
  71 local_tests=[
  72         "LOCAL-SUBSTITUTE",
  73         "LOCAL-GENCACHE",
  74         "LOCAL-TALLOC-DICT",
  75         "LOCAL-BASE64",
  76         "LOCAL-RBTREE",
  77         "LOCAL-MEMCACHE",
  78         "LOCAL-STREAM-NAME",
  79         "LOCAL-WBCLIENT",
  80         "LOCAL-string_to_sid",
  81         "LOCAL-binary_to_sid",
  82         "LOCAL-DBTRANS",
  83         "LOCAL-TEVENT-SELECT",
  84         "LOCAL-CONVERT-STRING",
  85         "LOCAL-CONV-AUTH-INFO",
  86         "LOCAL-sprintf_append"]
  87
  88 for t in local_tests:
  89     plantestsuite("samba3.smbtorture_s3.%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "-e"])
  90
  91 tests=["--ping", "--separator",
  92        "--own-domain",
  93        "--all-domains",
  94        "--trusted-domains",
  95        "--domain-info=BUILTIN",
  96        "--domain-info=$DOMAIN",
  97        "--online-status",
  98        "--online-status --domain=BUILTIN",
  99        "--online-status --domain=$DOMAIN",
 100        "--check-secret --domain=$DOMAIN",
 101        "--change-secret --domain=$DOMAIN",
 102        "--check-secret --domain=$DOMAIN",
 103        "--online-status --domain=$DOMAIN",
 104        #Didn't pass yet# "--domain-users",
 105        "--domain-groups",
 106        "--name-to-sid=$USERNAME",
 107        "--name-to-sid=$DOMAIN\\\\$USERNAME",
 108      #Didn't pass yet# "--user-info=$USERNAME",
 109        "--user-groups=$DOMAIN\\\\$USERNAME",
 110        "--allocate-uid",
 111        "--allocate-gid"]
 112
 113 for t in tests:
 114     plantestsuite("samba3.wbinfo_s3.(s3dc:local).%s" % t, "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_wbinfo_s3.sh"), t])
 115     plantestsuite("samba3.wbinfo_s3.(member:local).%s" % t, "member:local", [os.path.join(samba3srcdir, "script/tests/test_wbinfo_s3.sh"), t])
 116
 117 plantestsuite(
 118     "samba3.wbinfo_sids2xids.(member:local)", "member:local",
 119     [os.path.join(samba3srcdir, "script/tests/test_wbinfo_sids2xids.sh")])
 120
 121 plantestsuite("samba3.ntlm_auth.(s3dc:local)", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_ntlm_auth_s3.sh"), valgrindify(python), samba3srcdir, binpath('ntlm_auth3'), configuration])
 122
 123 for env in ["s3dc", "member"]:
 124     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$DC_USERNAME', '$DC_PASSWORD', binpath('smbclient3'), configuration])
 125
 126 for env in ["secserver"]:
 127     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) domain creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN\\\\$DC_USERNAME', '$DC_PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no"])
 128
 129 for env in ["member"]:
 130     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$SERVER\\\\$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration])
 131
 132 for env in ["maptoguest", "secshare", "secserver"]:
 133     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) local creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no --option=clientlanmanauth=yes"])
 134
 135 env = "maptoguest"
 136 plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) bad username" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', 'notmy$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no --option=clientlanmanauth=yes"])
 137
 138 # plain
 139 for env in ["s3dc"]:
 140     plantestsuite("samba3.blackbox.smbclient_s3.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration])
 141
 142 for env in ["member"]:
 143     plantestsuite("samba3.blackbox.smbclient_s3.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration])
 144
 145 for env in ["s3dc"]:
 146     plantestsuite("samba3.blackbox.smbclient_s3.sign (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "--signing=required"])
 147
 148 for env in ["member"]:
 149     plantestsuite("samba3.blackbox.smbclient_s3.sign (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "--signing=required"])
 150
 151 # encrypted
 152 for env in ["s3dc"]:
 153     plantestsuite("samba3.blackbox.smbclient_s3.crypt (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "-e"])
 154
 155 #TODO encrypted against member, with member creds, and with DC creds
 156 plantestsuite("samba3.blackbox.net.misc", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_misc.sh"),
 157                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration])
 158 plantestsuite("samba3.blackbox.net.local.registry", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_registry.sh"),
 159                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration])
 160 plantestsuite("samba3.blackbox.net.rpc.registry", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_registry.sh"),
 161                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
 162
 163 plantestsuite("samba3.blackbox.net.local.registry.roundtrip", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_registry_roundtrip.sh"),
 164                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration,])
 165 plantestsuite("samba3.blackbox.net.rpc.registry.roundtrip", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_registry_roundtrip.sh"),
 166                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
 167
 168 plantestsuite("samba3.blackbox.net.local.conf", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_conf.sh"),
 169                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration,])
 170 plantestsuite("samba3.blackbox.net.rpc.conf", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_conf.sh"),
 171                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
 172
 173
 174 plantestsuite("samba3.blackbox.testparm", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_testparm_s3.sh"),
 175                                                        "$LOCAL_PATH"])
 176
 177 plantestsuite(
 178     "samba3.pthreadpool", "s3dc",
 179     [os.path.join(samba3srcdir, "script/tests/test_pthreadpool.sh")])
 180
 181 #smbtorture4 tests
 182
 183 base = ["base.attr", "base.charset", "base.chkpath", "base.defer_open", "base.delaywrite", "base.delete",
 184         "base.deny1", "base.deny2", "base.deny3", "base.denydos", "base.dir1", "base.dir2",
 185         "base.disconnect", "base.fdpass", "base.lock",
 186         "base.mangle", "base.negnowait", "base.ntdeny1",
 187         "base.ntdeny2", "base.open", "base.openattr", "base.properties", "base.rename", "base.rw1",
 188         "base.secleak", "base.tcon", "base.tcondev", "base.trans2", "base.unlink", "base.vuid",
 189         "base.xcopy", "base.samba3error"]
 190
 191 raw = ["raw.acls", "raw.chkpath", "raw.close", "raw.composite", "raw.context", "raw.eas",
 192        "raw.ioctl", "raw.lock", "raw.mkdir", "raw.mux", "raw.notify", "raw.open", "raw.oplock"
 193        "raw.qfileinfo", "raw.qfsinfo", "raw.read", "raw.rename", "raw.search", "raw.seek",
 194        "raw.sfileinfo.base", "raw.sfileinfo.bug", "raw.streams", "raw.unlink", "raw.write",
 195        "raw.samba3hide", "raw.samba3badpath", "raw.sfileinfo.rename",
 196        "raw.samba3caseinsensitive", "raw.samba3posixtimedlock",
 197        "raw.samba3rootdirfid", "raw.sfileinfo.end-of-file",
 198        "raw.bench-oplock", "raw.bench-lock", "raw.bench-open", "raw.bench-tcon",
 199        "raw.samba3checkfsp", "raw.samba3closeerr", "raw.samba3oplocklogoff"]
 200
 201 smb2 = ["smb2.lock", "smb2.read", "smb2.compound", "smb2.connect", "smb2.scan", "smb2.scanfind",
 202         "smb2.bench-oplock"]
 203
 204 rpc = ["rpc.authcontext", "rpc.samba3.bind", "rpc.samba3.srvsvc", "rpc.samba3.sharesec",
 205        "rpc.samba3.spoolss", "rpc.samba3.wkssvc", "rpc.samba3.winreg",
 206        "rpc.samba3.getaliasmembership-0",
 207        "rpc.samba3.netlogon", "rpc.samba3.sessionkey", "rpc.samba3.getusername",
 208        "rpc.svcctl", "rpc.ntsvcs", "rpc.winreg", "rpc.eventlog",
 209        "rpc.spoolss.printserver", "rpc.spoolss.win", "rpc.spoolss.notify", "rpc.spoolss.printer",
 210        "rpc.spoolss.driver",
 211        "rpc.lsa", "rpc.lsa-getuser", "rpc.lsa.lookupsids", "rpc.lsa.lookupnames",
 212        "rpc.lsa.privileges", "rpc.lsa.secrets",
 213        "rpc.samr", "rpc.samr.users", "rpc.samr.users.privileges", "rpc.samr.passwords",
 214        "rpc.samr.passwords.pwdlastset", "rpc.samr.large-dc", "rpc.samr.machine.auth",
 215        "rpc.samr.priv",
 216        "rpc.netlogon.admin",
 217        "rpc.schannel", "rpc.schannel2", "rpc.bench-schannel1", "rpc.join", "rpc.bind"]
 218
 219 local = ["local.nss-wrapper", "local.ndr"]
 220
 221 winbind = ["winbind.struct", "winbind.wbclient"]
 222
 223 rap = ["rap.basic", "rap.rpc", "rap.printing", "rap.sam"]
 224
 225 unix = ["unix.info2", "unix.whoami"]
 226
 227 nbt = ["nbt.dgram" ]
 228
 229 libsmbclient = ["libsmbclient"]
 230
 231 tests= base + raw + smb2 + rpc + unix + local + winbind + rap + nbt + libsmbclient
 232
 233 sub = subprocess.Popen("%s --version 2> /dev/null" % smb4torture, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=True)
 234 sub.communicate("")
 235
 236 if sub.returncode == 0:
 237     for t in tests:
 238         if t == "base.delaywrite":
 239             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --maximum-runtime=900')
 240         elif t == "rap.sam":
 241             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --option=doscharset=ISO-8859-1')
 242         elif t == "unix.whoami":
 243             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD')
 244         elif t == "raw.samba3posixtimedlock":
 245             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD --option=torture:localdir=$SELFTEST_PREFIX/s3dc/share')
 246         elif t == "raw.chkpath":
 247             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpcase -U$USERNAME%$PASSWORD')
 248         else:
 249             plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD')
 250
 251
 252     test = 'rpc.lsa.lookupsids'
 253     auth_options = ["", "ntlm", "spnego" ]
 254     signseal_options = ["", ",connect", ",sign", ",seal"]
 255     smb_options = ["", ",smb2"]
 256     endianness_options = ["", ",bigendian"]
 257     for z in smb_options:
 258         for s in signseal_options:
 259             for e in endianness_options:
 260                 for a in auth_options:
 261                     binding_string = "ncacn_np:$SERVER[%s%s%s%s]" % (a, s, z, e)
 262                     options = binding_string + " -U$USERNAME%$PASSWORD"
 263                     plansmbtorturetestsuite(test, "s3dc", options, 'over ncacn_np with [%s%s%s%s] ' % (a, s, z, e))
 264
 265             # We should try more combinations in future, but this is all
 266             # the pre-calculated credentials cache supports at the moment
 267             e = ""
 268             a = ""
 269             binding_string = "ncacn_np:$SERVER[%s%s%s%s]" % (a, s, z, e)
 270             options = binding_string + " -k yes --krb5-ccache=$PREFIX/ktest/krb5_ccache-2"
 271             plansmbtorturetestsuite(test, "ktest", options, 'over kerberos with old ccache ncacn_np with [%s%s%s%s] ' % (a, s, z, e))
 272
 273             options = binding_string + " -k yes --krb5-ccache=$PREFIX/ktest/krb5_ccache-3"
 274             plansmbtorturetestsuite(test, "ktest", options, 'over kerberos ncacn_np with [%s%s%s%s] ' % (a, s, z, e))
 275
 276
 277
 278     for e in endianness_options:
 279         for a in auth_options:
 280             for s in signseal_options:
 281                 binding_string = "ncacn_ip_tcp:$SERVER_IP[%s%s%s]" % (a, s, e)
 282                 options = binding_string + " -U$USERNAME%$PASSWORD"
 283                 plansmbtorturetestsuite(test, "s3dc", options, 'over ncacn_ip_tcp with [%s%s%s] ' % (a, s, e))
 284
 285     test = 'rpc.epmapper'
 286     env = 's3dc:local'
 287     binding_string = 'ncalrpc:'
 288     options = binding_string + " -U$USERNAME%$PASSWORD"
 289
 290     plansmbtorturetestsuite(test, env, options, 'over ncalrpc')