s3-selftest: Remove .posix_s3 from s3 test names
[samba.git] / source3 / selftest / tests.py
1 #!/usr/bin/python
2 # This script generates a list of testsuites that should be run as part of 
3 # the Samba 3 test suite.
4
5 # The output of this script is parsed by selftest.pl, which then decides 
6 # which of the tests to actually run. It will, for example, skip all tests 
7 # listed in selftest/skip or only run a subset during "make quicktest".
8
9 # The idea is that this script outputs all of the tests of Samba 3, not 
10 # just those that are known to pass, and list those that should be skipped 
11 # or are known to fail in selftest/skip or selftest/samba4-knownfail. This makes it 
12 # very easy to see what functionality is still missing in Samba 3 and makes 
13 # it possible to run the testsuite against other servers, such as Samba 4 or 
14 # Windows that have a different set of features.
15
16 # The syntax for a testsuite is "-- TEST --" on a single line, followed 
17 # by the name of the test, the environment it needs and the command to run, all 
18 # three separated by newlines. All other lines in the output are considered 
19 # comments.
20
21 import os, sys
22 sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "../../selftest")))
23 from selftesthelpers import *
24 import subprocess
25 smb4torture = binpath("smbtorture4")
26 samba3srcdir = srcdir() + "/source3"
27 configuration = "--configfile=$SMB_CONF_PATH"
28 scriptdir=os.path.join(samba3srcdir, "../script/tests")
29
30 # see if we support ADS on the Samba3 side
31 try:
32     config_h = os.environ["CONFIG_H"]
33 except KeyError:
34     config_h = os.path.join(samba3srcdir, "include/autoconf/config.h")
35
36 torture_options = [configuration, "--maximum-runtime=$SELFTEST_MAXTIME", 
37                    "--target=samba3", "--basedir=$SELFTEST_TMPDIR",
38                    '--option="torture:winbindd_netbios_name=$SERVER"',
39                    '--option="torture:winbindd_netbios_domain=$DOMAIN"', 
40                    '--option=torture:sharedelay=100000']
41
42 if not os.getenv("SELFTEST_VERBOSE"):
43     torture_options.append("--option=torture:progress=no")
44 torture_options.append("--format=subunit")
45 if os.getenv("SELFTEST_QUICK"):
46     torture_options.append("--option=torture:quick=yes")
47 smb4torture += " " + " ".join(torture_options)
48
49 sub = subprocess.Popen("%s --version 2> /dev/null" % smb4torture, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=True)
50 sub.communicate("")
51 smb4torture_possible = (sub.returncode == 0)
52
53 def plansmbtorturetestsuite(name, env, options, description=''):
54     modname = "samba3.%s %s" % (name, description)
55     cmdline = "%s $LISTOPT %s %s" % (valgrindify(smb4torture), options, name)
56     if smb4torture_possible:
57         plantestsuite_loadlist(modname, env, cmdline)
58
59 plantestsuite("samba3.blackbox.success", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_success.sh")])
60 plantestsuite("samba3.blackbox.failure", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_failure.sh")])
61
62 plantestsuite("samba3.local_s3", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_local_s3.sh")])
63
64 plantestsuite("samba3.blackbox.registry.upgrade", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_registry_upgrade.sh"), binpath('net'), binpath('dbwrap_tool')])
65
66 tests=[ "FDPASS", "LOCK1", "LOCK2", "LOCK3", "LOCK4", "LOCK5", "LOCK6", "LOCK7", "LOCK9",
67         "UNLINK", "BROWSE", "ATTR", "TRANS2", "TORTURE",
68         "OPLOCK1", "OPLOCK2", "OPLOCK4", "STREAMERROR",
69         "DIR", "DIR1", "DIR-CREATETIME", "TCON", "TCONDEV", "RW1", "RW2", "RW3", "RW-SIGNING",
70         "OPEN", "XCOPY", "RENAME", "DELETE", "DELETE-LN", "PROPERTIES", "W2K",
71         "TCON2", "IOCTL", "CHKPATH", "FDSESS", "LOCAL-SUBSTITUTE", "CHAIN1", "CHAIN2",
72         "GETADDRINFO", "POSIX", "UID-REGRESSION-TEST", "SHORTNAME-TEST",
73         "LOCAL-BASE64", "LOCAL-GENCACHE", "POSIX-APPEND",
74         "CASE-INSENSITIVE-CREATE", "SMB2-BASIC", "NTTRANS-FSCTL", "SMB2-NEGPROT",
75         "CLEANUP1",
76         "CLEANUP2",
77         "BAD-NBT-SESSION"]
78
79 for t in tests:
80     plantestsuite("samba3.smbtorture_s3.plain(s3dc).%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "", "-l $LOCAL_PATH"])
81     plantestsuite("samba3.smbtorture_s3.crypt(s3dc).%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "-e", "-l $LOCAL_PATH"])
82
83 local_tests=[
84         "LOCAL-SUBSTITUTE",
85         "LOCAL-GENCACHE",
86         "LOCAL-TALLOC-DICT",
87         "LOCAL-BASE64",
88         "LOCAL-RBTREE",
89         "LOCAL-MEMCACHE",
90         "LOCAL-STREAM-NAME",
91         "LOCAL-WBCLIENT",
92         "LOCAL-string_to_sid",
93         "LOCAL-binary_to_sid",
94         "LOCAL-DBTRANS",
95         "LOCAL-TEVENT-SELECT",
96         "LOCAL-CONVERT-STRING",
97         "LOCAL-CONV-AUTH-INFO",
98         "LOCAL-hex_encode_buf",
99         "LOCAL-sprintf_append"]
100
101 for t in local_tests:
102     plantestsuite("samba3.smbtorture_s3.%s" % t, "s3dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', binpath('smbtorture3'), "-e"])
103
104 tests=["--ping", "--separator",
105        "--own-domain",
106        "--all-domains",
107        "--trusted-domains",
108        "--domain-info=BUILTIN",
109        "--domain-info=$DOMAIN",
110        "--online-status",
111        "--online-status --domain=BUILTIN",
112        "--online-status --domain=$DOMAIN",
113        "--check-secret --domain=$DOMAIN",
114        "--change-secret --domain=$DOMAIN",
115        "--check-secret --domain=$DOMAIN",
116        "--online-status --domain=$DOMAIN",
117        #Didn't pass yet# "--domain-users",
118        "--domain-groups",
119        "--name-to-sid=$DC_USERNAME",
120        "--name-to-sid=$DOMAIN\\\\$DC_USERNAME",
121      #Didn't pass yet# "--user-info=$USERNAME",
122        "--user-groups=$DOMAIN\\\\$DC_USERNAME",
123        "--allocate-uid",
124        "--allocate-gid"]
125
126 for t in tests:
127     plantestsuite("samba3.wbinfo_s3.(s3dc:local).%s" % t, "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_wbinfo_s3.sh"), t])
128     plantestsuite("samba3.wbinfo_s3.(member:local).%s" % t, "member:local", [os.path.join(samba3srcdir, "script/tests/test_wbinfo_s3.sh"), t])
129
130 plantestsuite(
131     "samba3.wbinfo_sids2xids.(member:local)", "member:local",
132     [os.path.join(samba3srcdir, "script/tests/test_wbinfo_sids2xids.sh")])
133
134 plantestsuite("samba3.ntlm_auth.(s3dc:local)", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_ntlm_auth_s3.sh"), valgrindify(python), samba3srcdir, binpath('ntlm_auth3'), configuration])
135
136 for env in ["s3dc", "member"]:
137     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$DC_USERNAME', '$DC_PASSWORD', binpath('smbclient3'), configuration])
138
139 for env in ["secserver"]:
140     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) domain creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN\\\\$DC_USERNAME', '$DC_PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no"])
141
142 for env in ["member"]:
143     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$SERVER\\\\$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration])
144
145 for env in ["maptoguest", "secshare", "secserver"]:
146     plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) local creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', '$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no --option=clientlanmanauth=yes"])
147
148 env = "maptoguest"
149 plantestsuite("samba3.blackbox.smbclient_auth.plain (%s) bad username" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_auth.sh"), '$SERVER', '$SERVER_IP', 'notmy$USERNAME', '$PASSWORD', binpath('smbclient3'), configuration + " --option=clientntlmv2auth=no --option=clientlanmanauth=yes"])
150
151 # plain
152 for env in ["s3dc"]:
153     plantestsuite("samba3.blackbox.smbclient_s3.plain (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration])
154
155 for env in ["member"]:
156     plantestsuite("samba3.blackbox.smbclient_s3.plain (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration])
157
158 for env in ["s3dc"]:
159     plantestsuite("samba3.blackbox.smbclient_s3.sign (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$DC_USERNAME', '$DC_PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "--signing=required"])
160
161 for env in ["member"]:
162     plantestsuite("samba3.blackbox.smbclient_s3.sign (%s) member creds" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$SERVER', '$SERVER\\\\$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "--signing=required"])
163
164 # encrypted
165 for env in ["s3dc"]:
166     plantestsuite("samba3.blackbox.smbclient_s3.crypt (%s)" % env, env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_s3.sh"), '$SERVER', '$SERVER_IP', '$DOMAIN', '$USERNAME', '$PASSWORD', '$USERID', '$LOCAL_PATH', '$PREFIX', binpath('smbclient3'), binpath('wbinfo'), configuration, "-e"])
167
168 #TODO encrypted against member, with member creds, and with DC creds
169 plantestsuite("samba3.blackbox.net.misc", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_misc.sh"),
170                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration])
171 plantestsuite("samba3.blackbox.net.local.registry", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_registry.sh"),
172                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration])
173 plantestsuite("samba3.blackbox.net.rpc.registry", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_registry.sh"),
174                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
175
176 plantestsuite("samba3.blackbox.net.local.registry.roundtrip", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_registry_roundtrip.sh"),
177                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration,])
178 plantestsuite("samba3.blackbox.net.rpc.registry.roundtrip", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_registry_roundtrip.sh"),
179                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
180
181 plantestsuite("samba3.blackbox.net.local.conf", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_net_conf.sh"),
182                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration,])
183 plantestsuite("samba3.blackbox.net.rpc.conf", "s3dc", [os.path.join(samba3srcdir, "script/tests/test_net_conf.sh"),
184                                                        scriptdir, "$SMB_CONF_PATH", binpath('net'), configuration, 'rpc'])
185
186
187 plantestsuite("samba3.blackbox.testparm", "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_testparm_s3.sh"),
188                                                        "$LOCAL_PATH"])
189
190 plantestsuite(
191     "samba3.pthreadpool", "s3dc",
192     [os.path.join(samba3srcdir, "script/tests/test_pthreadpool.sh")])
193
194 #smbtorture4 tests
195
196 base = ["base.attr", "base.charset", "base.chkpath", "base.defer_open", "base.delaywrite", "base.delete",
197         "base.deny1", "base.deny2", "base.deny3", "base.denydos", "base.dir1", "base.dir2",
198         "base.disconnect", "base.fdpass", "base.lock",
199         "base.mangle", "base.negnowait", "base.ntdeny1",
200         "base.ntdeny2", "base.open", "base.openattr", "base.properties", "base.rename", "base.rw1",
201         "base.secleak", "base.tcon", "base.tcondev", "base.trans2", "base.unlink", "base.vuid",
202         "base.xcopy", "base.samba3error"]
203
204 raw = ["raw.acls", "raw.chkpath", "raw.close", "raw.composite", "raw.context", "raw.eas",
205        "raw.ioctl", "raw.lock", "raw.mkdir", "raw.mux", "raw.notify", "raw.open", "raw.oplock"
206        "raw.qfileinfo", "raw.qfsinfo", "raw.read", "raw.rename", "raw.search", "raw.seek",
207        "raw.sfileinfo.base", "raw.sfileinfo.bug", "raw.streams", "raw.unlink", "raw.write",
208        "raw.samba3hide", "raw.samba3badpath", "raw.sfileinfo.rename",
209        "raw.samba3caseinsensitive", "raw.samba3posixtimedlock",
210        "raw.samba3rootdirfid", "raw.sfileinfo.end-of-file",
211        "raw.bench-oplock", "raw.bench-lock", "raw.bench-open", "raw.bench-tcon",
212        "raw.samba3checkfsp", "raw.samba3closeerr", "raw.samba3oplocklogoff"]
213
214 smb2 = ["smb2.lock", "smb2.read", "smb2.compound", "smb2.connect", "smb2.scan", "smb2.scanfind",
215         "smb2.bench-oplock", "smb2.rename"]
216
217 rpc = ["rpc.authcontext", "rpc.samba3.bind", "rpc.samba3.srvsvc", "rpc.samba3.sharesec",
218        "rpc.samba3.spoolss", "rpc.samba3.wkssvc", "rpc.samba3.winreg",
219        "rpc.samba3.getaliasmembership-0",
220        "rpc.samba3.netlogon", "rpc.samba3.sessionkey", "rpc.samba3.getusername",
221        "rpc.svcctl", "rpc.ntsvcs", "rpc.winreg", "rpc.eventlog",
222        "rpc.spoolss.printserver", "rpc.spoolss.win", "rpc.spoolss.notify", "rpc.spoolss.printer",
223        "rpc.spoolss.driver",
224        "rpc.lsa", "rpc.lsa-getuser", "rpc.lsa.lookupsids", "rpc.lsa.lookupnames",
225        "rpc.lsa.privileges", "rpc.lsa.secrets",
226        "rpc.samr", "rpc.samr.users", "rpc.samr.users.privileges", "rpc.samr.passwords",
227        "rpc.samr.passwords.pwdlastset", "rpc.samr.large-dc", "rpc.samr.machine.auth",
228        "rpc.samr.priv",
229        "rpc.netlogon.admin",
230        "rpc.schannel", "rpc.schannel2", "rpc.bench-schannel1", "rpc.join", "rpc.bind"]
231
232 local = ["local.nss-wrapper", "local.ndr"]
233
234 winbind = ["winbind.struct", "winbind.wbclient"]
235
236 rap = ["rap.basic", "rap.rpc", "rap.printing", "rap.sam"]
237
238 unix = ["unix.info2", "unix.whoami"]
239
240 nbt = ["nbt.dgram" ]
241
242 libsmbclient = ["libsmbclient"]
243
244 tests= base + raw + smb2 + rpc + unix + local + winbind + rap + nbt + libsmbclient
245
246 for t in tests:
247     if t == "base.delaywrite":
248         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --maximum-runtime=900')
249     elif t == "rap.sam":
250         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --option=doscharset=ISO-8859-1')
251     elif t == "unix.whoami":
252         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD')
253     elif t == "raw.samba3posixtimedlock":
254         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpguest -U$USERNAME%$PASSWORD --option=torture:localdir=$SELFTEST_PREFIX/s3dc/share')
255     elif t == "raw.chkpath":
256         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmpcase -U$USERNAME%$PASSWORD')
257     else:
258         plansmbtorturetestsuite(t, "s3dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD')
259
260
261 test = 'rpc.lsa.lookupsids'
262 auth_options = ["", "ntlm", "spnego", "spnego,ntlm" ]
263 signseal_options = ["", ",connect", ",sign", ",seal"]
264 endianness_options = ["", ",bigendian"]
265 for s in signseal_options:
266     for e in endianness_options:
267         for a in auth_options:
268             binding_string = "ncacn_np:$SERVER[%s%s%s]" % (a, s, e)
269             options = binding_string + " -U$USERNAME%$PASSWORD"
270             plansmbtorturetestsuite(test, "s3dc", options, 'over ncacn_np with [%s%s%s] ' % (a, s, e))
271             plantestsuite("samba3.blackbox.rpcclient over ncacn_np with [%s%s%s] " % (a, s, e), "s3dc:local", [os.path.join(samba3srcdir, "script/tests/test_rpcclient.sh"),
272                                                              "none", options, configuration])
273
274     # We should try more combinations in future, but this is all
275     # the pre-calculated credentials cache supports at the moment
276     e = ""
277     a = ""
278     binding_string = "ncacn_np:$SERVER[%s%s%s]" % (a, s, e)
279     options = binding_string + " -k yes --krb5-ccache=$PREFIX/ktest/krb5_ccache-2"
280     plansmbtorturetestsuite(test, "ktest", options, 'krb5 with old ccache ncacn_np with [%s%s%s] ' % (a, s, e))
281
282     options = binding_string + " -k yes --krb5-ccache=$PREFIX/ktest/krb5_ccache-3"
283     plansmbtorturetestsuite(test, "ktest", options, 'krb5 ncacn_np with [%s%s%s] ' % (a, s, e))
284
285     auth_options2 = ["krb5", "spnego,krb5"]
286     for a in auth_options2:
287         binding_string = "ncacn_np:$SERVER[%s%s%s]" % (a, s, e)
288
289         plantestsuite("samba3.blackbox.rpcclient krb5 ncacn_np with [%s%s%s] " % (a, s, e), "ktest:local", [os.path.join(samba3srcdir, "script/tests/test_rpcclient.sh"),
290                                                                                                                               "$PREFIX/ktest/krb5_ccache-3", binding_string, "-k", configuration])
291
292
293 options_list = ["", "-e"]
294 for options in options_list:
295     plantestsuite("samba3.blackbox.smbclient_krb5 old ccache %s" % options, "ktest:local", 
296                   [os.path.join(samba3srcdir, "script/tests/test_smbclient_krb5.sh"),
297                    "$PREFIX/ktest/krb5_ccache-2", 
298                    binpath('smbclient3'), "$SERVER", options, configuration])
299
300     plantestsuite("samba3.blackbox.smbclient_krb5 old ccache %s" % options, "ktest:local",
301                   [os.path.join(samba3srcdir, "script/tests/test_smbclient_krb5.sh"),
302                    "$PREFIX/ktest/krb5_ccache-2",
303                    binpath('smbclient3'), "$SERVER", options, configuration])
304
305     plantestsuite("samba3.blackbox.smbclient_large_file %s" % options, "ktest:local",
306                   [os.path.join(samba3srcdir, "script/tests/test_smbclient_posix_large.sh"),
307                    "$PREFIX/ktest/krb5_ccache-3",
308                    binpath('smbclient3'), "$SERVER", "$PREFIX", options, "-k " + configuration])
309
310     plantestsuite("samba3.blackbox.smbclient_posix_large %s krb5" % options, "ktest:local",
311                   [os.path.join(samba3srcdir, "script/tests/test_smbclient_posix_large.sh"),
312                    "$PREFIX/ktest/krb5_ccache-3",
313                    binpath('smbclient3'), "$SERVER", "$PREFIX", options, "-k " + configuration])
314
315     plantestsuite("samba3.blackbox.smbclient_posix_large %s NTLM" % options, "s3dc:local",
316                   [os.path.join(samba3srcdir, "script/tests/test_smbclient_posix_large.sh"),
317                    "none",
318                    binpath('smbclient3'), "$SERVER", "$PREFIX", options, "-U$USERNAME%$PASSWORD " + configuration])
319
320 for e in endianness_options:
321     for a in auth_options:
322         for s in signseal_options:
323             binding_string = "ncacn_ip_tcp:$SERVER_IP[%s%s%s]" % (a, s, e)
324             options = binding_string + " -U$USERNAME%$PASSWORD"
325             plansmbtorturetestsuite(test, "s3dc", options, 'over ncacn_ip_tcp with [%s%s%s] ' % (a, s, e))
326
327 test = 'rpc.epmapper'
328 env = 's3dc:local'
329 binding_string = 'ncalrpc:'
330 options = binding_string + " -U$USERNAME%$PASSWORD"
331
332 plansmbtorturetestsuite(test, env, options, 'over ncalrpc')