autobuild: run "samba-ctdb" by default to catch dbwrap_ctdb regressions
[ddiss/samba.git] / script / autobuild.py
index 545e47ed489358d599d5972763a98af615868afb..c60d019a5c3a56df3aca933a2fe0241eeb0ef0c7 100755 (executable)
@@ -8,6 +8,7 @@ import os, tarfile, sys, time
 from optparse import OptionParser
 import smtplib
 from email.mime.text import MIMEText
+from distutils.sysconfig import get_python_lib
 
 samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
 samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
@@ -16,10 +17,13 @@ cleanup_list = []
 
 builddirs = {
     "samba3"  : "source3",
-    "samba3-waf": "source3",
-    "samba4"  : ".",
-    "ldb"     : "source4/lib/ldb",
+    "samba3-ctdb" : "source3",
+    "samba"  : ".",
+    "samba-ctdb" : ".",
+    "samba-libs"  : ".",
+    "ldb"     : "lib/ldb",
     "tdb"     : "lib/tdb",
+    "ntdb"    : "lib/ntdb",
     "talloc"  : "lib/talloc",
     "replace" : "lib/replace",
     "tevent"  : "lib/tevent",
@@ -29,74 +33,143 @@ builddirs = {
     "retry"   : "."
     }
 
-defaulttasks = [ "samba3", "samba3-waf", "samba4", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
+defaulttasks = [ "samba3", "samba", "samba-ctdb", "samba-libs", "ldb", "tdb", "ntdb", "talloc", "replace", "tevent", "pidl" ]
 
 tasks = {
     "samba3" : [ ("autogen", "./autogen.sh", "text/plain"),
                  ("configure", "./configure.developer ${PREFIX}", "text/plain"),
                  ("make basics", "make basics", "text/plain"),
-                 ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
+                 # we split 'make -j 4', 'make bin/smbtorture4' and 'make -j 4 everything'
+                 # because it makes it much easier to find errors.
+                 ("make", "make -j 4", "text/plain"), # don't use too many processes
+                 ("make bin/smbtorture4", "make bin/smbtorture4", "text/plain"),
+                 ("make everything", "make -j 4 everything", "text/plain"),
                  ("install", "make install", "text/plain"),
                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
                  ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
                  ("clean", "make clean", "text/plain") ],
 
-    "samba3-waf" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
-                 ("configure", "./configure.developer ${PREFIX}", "text/plain"),
-                 ("make", "make -j", "text/plain"),
-                 ("install", "make install", "text/plain"),
-                 ("clean", "make clean", "text/plain") ],
+    "samba3-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      ("autogen", "./autogen.sh", "text/plain"),
+                      ("configure", "./configure.developer ${PREFIX} --with-cluster-support --with-ctdb=../ctdb", "text/plain"),
+                      ("make basics", "make basics", "text/plain"),
+                      ("make", "make all", "text/plain"), # don't use too many processes
+                      ("check", "LD_LIBRARY_PATH=./bin ./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
+                      ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
+                      ("clean", "make clean", "text/plain") ],
 
     # We have 'test' before 'install' because, 'test' should work without 'install'
-    "samba4" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"),
-                 ("make", "make -j", "text/plain"),
-                 ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
-                 ("install", "make install", "text/plain"),
-                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
-                 ("clean", "make clean", "text/plain") ],
-
-    "ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
-              ("make", "make -j", "text/plain"),
+    "samba" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"),
+                ("make", "make -j", "text/plain"),
+                ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
+                ("install", "make install", "text/plain"),
+                ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                ("clean", "make clean", "text/plain") ],
+
+    "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+
+                     # make sure we have tdb around:
+                     ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                     ("tdb-make", "cd lib/tdb && make", "text/plain"),
+                     ("tdb-install", "cd lib/tdb && make install", "text/plain"),
+
+                     # install the ctdb headers under the prefix:
+                     ("ctdb-header-install", "cp ./ctdb/include/* ${PREFIX_DIR}/include", "text/plain"),
+                     ("ctdb-header-ls", "ls ${PREFIX_DIR}/include/ctdb.h", "text/plain"),
+
+                     ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --with-ctdb-dir=${PREFIX_DIR} --bundled-libraries=!tdb", "text/plain"),
+                     ("make", "make", "text/plain"),
+                     ("check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
+                     ("install", "make install", "text/plain"),
+                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                     ("clean", "make clean", "text/plain") ],
+
+    "samba-libs" : [
+                      ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      ("talloc-configure", "cd lib/talloc && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("talloc-make", "cd lib/talloc && make", "text/plain"),
+                      ("talloc-install", "cd lib/talloc && make install", "text/plain"),
+
+                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
+                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
+
+                      ("tevent-configure", "cd lib/tevent && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("tevent-make", "cd lib/tevent && make", "text/plain"),
+                      ("tevent-install", "cd lib/tevent && make install", "text/plain"),
+
+                      ("ldb-configure", "cd lib/ldb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("ldb-make", "cd lib/ldb && make", "text/plain"),
+                      ("ldb-install", "cd lib/ldb && make install", "text/plain"),
+
+                      ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=!talloc,!tdb,!pytdb,!ldb,!pyldb,!tevent,!pytevent --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("make", "make", "text/plain"),
+                      ("install", "make install", "text/plain")],
+
+    "ldb" : [
+              ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+              ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+              ("make", "make", "text/plain"),
               ("install", "make install", "text/plain"),
               ("test", "TDB_NO_FSYNC=1 make test", "text/plain"),
-              ("check-clean-tree", "../../../script/clean-source-tree.sh", "text/plain"),
+              ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
               ("distcheck", "make distcheck", "text/plain"),
               ("clean", "make clean", "text/plain") ],
 
     # We don't use TDB_NO_FSYNC=1 here, because we want to test the transaction code
-    "tdb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
-              ("make", "make -j", "text/plain"),
+    "tdb" : [
+              ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+              ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+              ("make", "make", "text/plain"),
               ("install", "make install", "text/plain"),
               ("test", "make test", "text/plain"),
               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
               ("distcheck", "make distcheck", "text/plain"),
               ("clean", "make clean", "text/plain") ],
 
-    "talloc" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
-                 ("make", "make -j", "text/plain"),
+    "ntdb" : [
+               ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+               ("make", "make", "text/plain"),
+               ("install", "make install", "text/plain"),
+               ("test", "make test", "text/plain"),
+               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
+               ("distcheck", "make distcheck", "text/plain"),
+               ("clean", "make clean", "text/plain") ],
+
+    "talloc" : [
+                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+                 ("make", "make", "text/plain"),
                  ("install", "make install", "text/plain"),
                  ("test", "make test", "text/plain"),
                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
                  ("distcheck", "make distcheck", "text/plain"),
                  ("clean", "make clean", "text/plain") ],
 
-    "replace" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
-                  ("make", "make -j", "text/plain"),
+    "replace" : [
+                  ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+                  ("make", "make", "text/plain"),
                   ("install", "make install", "text/plain"),
                   ("test", "make test", "text/plain"),
                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
                   ("distcheck", "make distcheck", "text/plain"),
                   ("clean", "make clean", "text/plain") ],
 
-    "tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
-                 ("make", "make -j", "text/plain"),
+    "tevent" : [
+                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+                 ("make", "make", "text/plain"),
                  ("install", "make install", "text/plain"),
                  ("test", "make test", "text/plain"),
                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
                  ("distcheck", "make distcheck", "text/plain"),
                  ("clean", "make clean", "text/plain") ],
 
-    "pidl" : [ ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
+    "pidl" : [
+               ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+               ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
                ("touch", "touch *.yp", "text/plain"),
                ("make", "make", "text/plain"),
                ("test", "make test", "text/plain"),
@@ -170,6 +243,7 @@ class builder(object):
             self.done = True
             return
         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
+        self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 #        if self.output_mime_type == "text/x-subunit":
@@ -496,7 +570,7 @@ while True:
         run_cmd("rm -rf %s" % test_master)
         cleanup_list.append(test_master)
         run_cmd("git clone --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
-    except:
+    except Exception:
         cleanup()
         raise
 
@@ -506,7 +580,9 @@ while True:
                 rebase_tree(options.rebase)
             elif options.rebase_master:
                 rebase_tree(samba_master)
-        except:
+        except Exception:
+            cleanup_list.append(gitroot + "/autobuild.pid")
+            cleanup()
             email_failure(-1, 'rebase', 'rebase', 'rebase', 'rebase on master failed')
             sys.exit(1)
         blist = buildlist(tasks, args)
@@ -516,7 +592,7 @@ while True:
         if status != 0 or errstr != "retry":
             break
         cleanup()
-    except:
+    except Exception:
         cleanup()
         raise