script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-xc": ".",
  42     "samba-o3": ".",
  43     "samba-ctdb": ".",
  44     "samba-libs": ".",
  45     "samba-static": ".",
  46     "samba-none-env": ".",
  47     "samba-ad-dc": ".",
  48     "samba-ad-dc-ntvfs": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-backup": ".",
  51     "samba-systemkrb5": ".",
  52     "samba-nopython": ".",
  53     "samba-nopython-py2": ".",
  54     "ldb": "lib/ldb",
  55     "tdb": "lib/tdb",
  56     "talloc": "lib/talloc",
  57     "replace": "lib/replace",
  58     "tevent": "lib/tevent",
  59     "pidl": "pidl"
  60 }
  61
  62 defaulttasks = builddirs.keys()
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  69
  70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  77
  78 tasks = {
  79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  81                ("make", "make all", "text/plain"),
  82                ("install", "make install", "text/plain"),
  83                ("test", "make autotest", "text/plain"),
  84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  85                ("clean", "make clean", "text/plain")],
  86
  87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  88     "samba": [
  89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  91                 ("make", "make -j", "text/plain"),
  92                 ("test", "make test FAIL_IMMEDIATELY=1 "
  93                  "TESTS='--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_member "
  96                  "--exclude-env=ad_dc "
  97                  "--exclude-env=ad_dc_backup "
  98                  "--exclude-env=ad_dc_ntvfs "
  99                  "--exclude-env=ad_dc_default "
 100                  "--exclude-env=ad_dc_slowtests "
 101                  "--exclude-env=ad_dc_no_nss "
 102                  "--exclude-env=fl2003dc "
 103                  "--exclude-env=fl2008dc "
 104                  "--exclude-env=fl2008r2dc "
 105                  "--exclude-env=ad_member "
 106                  "--exclude-env=ad_member_idmap_rid "
 107                  "--exclude-env=ad_member_idmap_ad "
 108                  "--exclude-env=chgdcpass "
 109                  "--exclude-env=vampire_2000_dc "
 110                  "--exclude-env=fl2000dc "
 111                  "--exclude-env=fileserver "
 112                  "--exclude-env=backupfromdc "
 113                  "--exclude-env=restoredc "
 114                  "--exclude-env=renamedc "
 115                  "--exclude-env=offlinebackupdc "
 116                  "--exclude-env=labdc "
 117                  "'",
 118                  "text/plain"),
 119                 ("install", "make install", "text/plain"),
 120                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 121                 ("clean", "make clean", "text/plain")],
 122
 123     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 124                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 125                     ("make", "make -j", "text/plain"),
 126                     ("test", "make test FAIL_IMMEDIATELY=1 "
 127                      "TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 128                     ("install", "make install", "text/plain"),
 129                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 130                     ("clean", "make clean", "text/plain")],
 131
 132     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 133                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 134                            ("make", "make -j", "text/plain"),
 135                            ("test", "make test FAIL_IMMEDIATELY=1 "
 136                             "TESTS='--include-env=fileserver'", "text/plain"),
 137                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 138
 139     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 "
 143                        "TESTS='--include-env=ad_dc "
 144                        "--include-env=ad_dc_backup "
 145                        "--include-env=fl2003dc "
 146                        "--include-env=fl2008r2dc "
 147                        "--include-env=ad_member "
 148                        "--include-env=ad_member_idmap_rid "
 149                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 150                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 151
 152     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 153                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 154                         ("make", "make -j", "text/plain"),
 155                         ("test", "make test FAIL_IMMEDIATELY=1 "
 156                          "TESTS='--include-env=chgdcpass "
 157                          "--include-env=vampire_2000_dc "
 158                          "--include-env=fl2000dc "
 159                          "--include-env=ad_dc_no_nss "
 160                          "'",
 161                          "text/plain"),
 162                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 163
 164     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 165     # This is currently the longest task, so we don't randomly delay it.
 166     "samba-ad-dc-ntvfs": [
 167                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 168                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 169                       ("make", "make -j", "text/plain"),
 170                       ("test", "make test FAIL_IMMEDIATELY=1 "
 171                        "TESTS='"
 172                        "--include-env=ad_dc_ntvfs "
 173                        "--include-env=fl2008dc "
 174                        "--include-env=ad_dc_default "
 175                        "--include-env=ad_dc_slowtests "
 176                        "'", "text/plain"),
 177                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 178
 179     # run the backup/restore testenvs separately as they're fairly standalone
 180     # (and CI seems to max out at ~8 different DCs running at once)
 181     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 182                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 183                         ("make", "make -j", "text/plain"),
 184                         ("test", "make test FAIL_IMMEDIATELY=1 "
 185                          "TESTS='--include-env=backupfromdc "
 186                          "--include-env=restoredc "
 187                          "--include-env=renamedc "
 188                          "--include-env=offlinebackupdc "
 189                          "--include-env=labdc "
 190                          "'",
 191                          "text/plain"),
 192                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 193
 194     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 195                           ("make", "make -j", "text/plain"),
 196                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 197
 198     # Test cross-compile infrastructure
 199     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 200                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 201                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 202                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 203                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 204                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 205                    ("compare-results", "script/compare_cc_results.py "
 206                     "./bin/c4che/default{} "
 207                     "./bin-xe/c4che/default{} "
 208                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 209
 210     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 211     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 212                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 213                    ("make", "make -j", "text/plain"),
 214                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 215                     "TESTS='--include-env=ad_dc'", "text/plain"),
 216                    ("install", "make install", "text/plain"),
 217                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 218                    ("clean", "make clean", "text/plain")],
 219
 220     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 221
 222                      # make sure we have tdb around:
 223                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 224                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 225                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 226
 227
 228                      # build samba with cluster support (also building ctdb):
 229                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 230                      ("samba-make", "make", "text/plain"),
 231                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 232                      ("samba-install", "make install", "text/plain"),
 233                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 234
 235                      # clean up:
 236                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 237                      ("clean", "make clean", "text/plain"),
 238                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 239
 240     "samba-libs": [
 241                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 242                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 243                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 244                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 245
 246                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 247                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 248                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 249
 250                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 251                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 252                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 253
 254                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 255                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 256                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 257
 258                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 259                       ("nondevel-make", "make -j", "text/plain"),
 260                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 261                       ("nondevel-install", "make install", "text/plain"),
 262                       ("nondevel-dist", "make dist", "text/plain"),
 263
 264                       # retry with all modules shared
 265                       ("allshared-distclean", "make distclean", "text/plain"),
 266                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 267                       ("allshared-make", "make -j", "text/plain")],
 268
 269     "samba-none-env": [
 270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 271                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 272                       ("make", "make -j", "text/plain"),
 273                       ("test", "make test "
 274                        "FAIL_IMMEDIATELY=1 "
 275                        "TESTS='--include-env=none'",
 276                        "text/plain")],
 277
 278     "samba-static": [
 279                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 280                       # build with all modules static
 281                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 282                       ("allstatic-make", "make -j", "text/plain"),
 283                       ("allstatic-test", "make test "
 284                        "FAIL_IMMEDIATELY=1 "
 285                        "TESTS='samba3.smb2.create.*nt4_dc'",
 286                        "text/plain"),
 287
 288                       # retry without any required modules
 289                       ("none-distclean", "make distclean", "text/plain"),
 290                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 291                       ("none-make", "make -j", "text/plain"),
 292
 293                       # retry with nonshared smbd and smbtorture
 294                       ("nonshared-distclean", "make distclean", "text/plain"),
 295                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 296                       ("nonshared-make", "make -j", "text/plain")],
 297
 298     "samba-systemkrb5": [
 299                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 300                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 301                       ("make", "make -j", "text/plain"),
 302                       # we currently cannot run a full make test, a limited list of tests could be run
 303                       # via "make test TESTS=sometests"
 304                       ("test", "make test FAIL_IMMEDIATELY=1 "
 305                        "TESTS='--include-env=ktest'", "text/plain"),
 306                       ("install", "make install", "text/plain"),
 307                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 308                       ("clean", "make clean", "text/plain")
 309                       ],
 310
 311     # Test Samba without python still builds.  When this test fails
 312     # due to more use of Python, the expectations is that the newly
 313     # failing part of the code should be disabled when
 314     # --disable-python is set (rather than major work being done to
 315     # support this environment).  The target here is for vendors
 316     # shipping a minimal smbd.
 317     "samba-nopython": [
 318                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 319                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 320                       ("make", "make -j", "text/plain"),
 321                       ("install", "make install", "text/plain"),
 322                       ("test", "make test-nopython", "text/plain"),
 323                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 324                       ("clean", "make clean", "text/plain"),
 325
 326                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 327                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 328                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 329
 330                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 331                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 332                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 333
 334                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 335                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 336                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 337
 338                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 339                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 340                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 341
 342                       # retry against installed library packages
 343                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 344                       ("libs-make", "make -j", "text/plain"),
 345                       ("libs-install", "make install", "text/plain"),
 346                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 347                       ("libs-clean", "make clean", "text/plain")
 348                       ],
 349
 350     # check we can do the same thing using python2
 351     "samba-nopython-py2": [
 352                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 353                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 354                       ("make", "PYTHON=python2 make -j", "text/plain"),
 355                       ("install", "PYTHON=python2 make install", "text/plain"),
 356                       ("test", "make test-nopython", "text/plain"),
 357                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 358                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 359
 360                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 361                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 362                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 363
 364                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 365                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 366                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 367
 368                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 369                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 370                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 371
 372                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 373                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 374                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 375
 376                       # retry against installed library packages
 377                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 378                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 379                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 380                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 381                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 382                       ],
 383
 384     "ldb": [
 385               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 386               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 387               ("make", "make", "text/plain"),
 388               ("install", "make install", "text/plain"),
 389               ("test", "make test", "text/plain"),
 390               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 391               ("make-no-lmdb", "make", "text/plain"),
 392               ("install-no-lmdb", "make install", "text/plain"),
 393               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 394               ("distcheck", "make distcheck", "text/plain"),
 395               ("clean", "make clean", "text/plain")],
 396
 397     "tdb": [
 398               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 399               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 400               ("make", "make", "text/plain"),
 401               ("install", "make install", "text/plain"),
 402               ("test", "make test", "text/plain"),
 403               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 404               ("distcheck", "make distcheck", "text/plain"),
 405               ("clean", "make clean", "text/plain")],
 406
 407     "talloc": [
 408                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 409                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 410                  ("make", "make", "text/plain"),
 411                  ("install", "make install", "text/plain"),
 412                  ("test", "make test", "text/plain"),
 413                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 414                  ("distcheck", "make distcheck", "text/plain"),
 415                  ("clean", "make clean", "text/plain")],
 416
 417     "replace": [
 418                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 419                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 420                   ("make", "make", "text/plain"),
 421                   ("install", "make install", "text/plain"),
 422                   ("test", "make test", "text/plain"),
 423                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 424                   ("distcheck", "make distcheck", "text/plain"),
 425                   ("clean", "make clean", "text/plain")],
 426
 427     "tevent": [
 428                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 429                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 430                  ("make", "make", "text/plain"),
 431                  ("install", "make install", "text/plain"),
 432                  ("test", "make test", "text/plain"),
 433                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 434                  ("distcheck", "make distcheck", "text/plain"),
 435                  ("clean", "make clean", "text/plain")],
 436
 437     "pidl": [
 438         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 439         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 440         ("touch", "touch *.yp", "text/plain"),
 441         ("make", "make", "text/plain"),
 442         ("test", "make test", "text/plain"),
 443         ("install", "make install", "text/plain"),
 444         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 445         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 446         ("clean", "make clean", "text/plain")],
 447
 448
 449     # these are useful for debugging autobuild
 450     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 451     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 452
 453
 454 }
 455
 456
 457 def do_print(msg):
 458     print("%s" % msg)
 459     sys.stdout.flush()
 460     sys.stderr.flush()
 461
 462
 463 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 464     if show is None:
 465         show = options.verbose
 466     if show:
 467         do_print("Running: '%s' in '%s'" % (cmd, dir))
 468     if output:
 469         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 470     elif checkfail:
 471         return check_call(cmd, shell=True, cwd=dir)
 472     else:
 473         return call(cmd, shell=True, cwd=dir)
 474
 475
 476 class builder(object):
 477     '''handle build of one directory'''
 478
 479     def __init__(self, name, sequence, cp=True):
 480         self.name = name
 481         if name in builddirs:
 482             self.dir = builddirs[name]
 483         else:
 484             self.dir = "."
 485
 486         self.tag = self.name.replace('/', '_')
 487         self.sequence = sequence
 488         self.next = 0
 489         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 490         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 491         if options.verbose:
 492             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 493             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 494         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 495         self.stdout = open(self.stdout_path, 'w')
 496         self.stderr = open(self.stderr_path, 'w')
 497         self.stdin  = open("/dev/null", 'r')
 498         self.sdir = "%s/%s" % (testbase, self.tag)
 499         self.prefix = "%s/%s" % (test_prefix, self.tag)
 500         run_cmd("rm -rf %s" % self.sdir)
 501         run_cmd("rm -rf %s" % self.prefix)
 502         if cp:
 503             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 504         else:
 505             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 506         self.start_next()
 507
 508     def start_next(self):
 509         if self.next == len(self.sequence):
 510             if not options.nocleanup:
 511                 run_cmd("rm -rf %s" % self.sdir)
 512                 run_cmd("rm -rf %s" % self.prefix)
 513             do_print('%s: Completed OK' % self.name)
 514             self.done = True
 515             return
 516         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 517         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 518         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 519         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 520         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 521 #        if self.output_mime_type == "text/x-subunit":
 522 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 523         cwd = os.getcwd()
 524         os.chdir("%s/%s" % (self.sdir, self.dir))
 525         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 526         self.proc = Popen(self.cmd, shell=True,
 527                           close_fds=True,
 528                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 529         os.chdir(cwd)
 530         self.next += 1
 531
 532
 533 class buildlist(object):
 534     '''handle build of multiple directories'''
 535
 536     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 537         global tasks
 538         self.tlist = []
 539         self.tail_proc = None
 540         self.retry = None
 541         if tasknames == []:
 542             if options.restrict_tests:
 543                 tasknames = ["samba-test-only"]
 544             else:
 545                 tasknames = defaulttasks
 546         else:
 547             # If we are only running one test,
 548             # do not sleep randomly to wait for it to start
 549             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 550
 551         for n in tasknames:
 552             b = builder(n, tasks[n], cp=n is not "pidl")
 553             self.tlist.append(b)
 554         if options.retry:
 555             rebase_remote = "rebaseon"
 556             retry_task = [("retry",
 557                             '''set -e
 558                             git remote add -t %s %s %s
 559                             git fetch %s
 560                             while :; do
 561                               sleep 60
 562                               git describe %s/%s > old_remote_branch.desc
 563                               git fetch %s
 564                               git describe %s/%s > remote_branch.desc
 565                               diff old_remote_branch.desc remote_branch.desc
 566                             done
 567                            ''' % (
 568                                 rebase_branch, rebase_remote, rebase_url,
 569                                rebase_remote,
 570                                rebase_remote, rebase_branch,
 571                                rebase_remote,
 572                                rebase_remote, rebase_branch
 573                             ),
 574                             "test/plain")]
 575
 576             self.retry = builder('retry', retry_task, cp=False)
 577             self.need_retry = False
 578
 579     def kill_kids(self):
 580         if self.tail_proc is not None:
 581             self.tail_proc.terminate()
 582             self.tail_proc.wait()
 583             self.tail_proc = None
 584         if self.retry is not None:
 585             self.retry.proc.terminate()
 586             self.retry.proc.wait()
 587             self.retry = None
 588         for b in self.tlist:
 589             if b.proc is not None:
 590                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 591                 b.proc.terminate()
 592                 b.proc.wait()
 593                 b.proc = None
 594
 595     def wait_one(self):
 596         while True:
 597             none_running = True
 598             for b in self.tlist:
 599                 if b.proc is None:
 600                     continue
 601                 none_running = False
 602                 b.status = b.proc.poll()
 603                 if b.status is None:
 604                     continue
 605                 b.proc = None
 606                 return b
 607             if options.retry:
 608                 ret = self.retry.proc.poll()
 609                 if ret is not None:
 610                     self.need_retry = True
 611                     self.retry = None
 612                     return None
 613             if none_running:
 614                 return None
 615             time.sleep(0.1)
 616
 617     def run(self):
 618         while True:
 619             b = self.wait_one()
 620             if options.retry and self.need_retry:
 621                 self.kill_kids()
 622                 do_print("retry needed")
 623                 return (0, None, None, None, "retry")
 624             if b is None:
 625                 break
 626             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 627                 self.kill_kids()
 628                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 629             b.start_next()
 630         self.kill_kids()
 631         return (0, None, None, None, "All OK")
 632
 633     def write_system_info(self):
 634         filename = 'system-info.txt'
 635         f = open(filename, 'w')
 636         for cmd in ['uname -a',
 637                     'lsb_release -a',
 638                     'free',
 639                     'mount',
 640                     'cat /proc/cpuinfo',
 641                     'cc --version',
 642                     'df -m .',
 643                     'df -m %s' % testbase]:
 644             out = run_cmd(cmd, output=True, checkfail=False)
 645             print('### %s' % cmd, file=f)
 646             print(out.decode('utf8', 'backslashreplace'), file=f)
 647             print(file=f)
 648         f.close()
 649         return filename
 650
 651     def tarlogs(self, fname):
 652         tar = tarfile.open(fname, "w:gz")
 653         for b in self.tlist:
 654             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 655             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 656         if os.path.exists("autobuild.log"):
 657             tar.add("autobuild.log")
 658         sys_info = self.write_system_info()
 659         tar.add(sys_info)
 660         tar.close()
 661
 662     def remove_logs(self):
 663         for b in self.tlist:
 664             os.unlink(b.stdout_path)
 665             os.unlink(b.stderr_path)
 666
 667     def start_tail(self):
 668         cmd = ["tail", "-f"]
 669         for b in self.tlist:
 670             cmd.append(b.stdout_path)
 671             cmd.append(b.stderr_path)
 672         self.tail_proc = Popen(cmd, close_fds=True)
 673
 674
 675 def cleanup():
 676     if options.nocleanup:
 677         return
 678     run_cmd("stat %s || true" % test_tmpdir, show=True)
 679     run_cmd("stat %s" % testbase, show=True)
 680     do_print("Cleaning up %r" % cleanup_list)
 681     for d in cleanup_list:
 682         run_cmd("rm -rf %s" % d)
 683
 684
 685 def find_git_root():
 686     '''get to the top of the git repo'''
 687     p = os.getcwd()
 688     while p != '/':
 689         if os.path.isdir(os.path.join(p, ".git")):
 690             return p
 691         p = os.path.abspath(os.path.join(p, '..'))
 692     return None
 693
 694
 695 def daemonize(logfile):
 696     pid = os.fork()
 697     if pid == 0:  # Parent
 698         os.setsid()
 699         pid = os.fork()
 700         if pid != 0:  # Actual daemon
 701             os._exit(0)
 702     else:  # Grandparent
 703         os._exit(0)
 704
 705     import resource      # Resource usage information.
 706     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 707     if maxfd == resource.RLIM_INFINITY:
 708         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 709     for fd in range(0, maxfd):
 710         try:
 711             os.close(fd)
 712         except OSError:
 713             pass
 714     os.open(logfile, os.O_RDWR | os.O_CREAT)
 715     os.dup2(0, 1)
 716     os.dup2(0, 2)
 717
 718
 719 def write_pidfile(fname):
 720     '''write a pid file, cleanup on exit'''
 721     f = open(fname, mode='w')
 722     f.write("%u\n" % os.getpid())
 723     f.close()
 724
 725
 726 def rebase_tree(rebase_url, rebase_branch="master"):
 727     rebase_remote = "rebaseon"
 728     do_print("Rebasing on %s" % rebase_url)
 729     run_cmd("git describe HEAD", show=True, dir=test_master)
 730     run_cmd("git remote add -t %s %s %s" %
 731             (rebase_branch, rebase_remote, rebase_url),
 732             show=True, dir=test_master)
 733     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 734     if options.fix_whitespace:
 735         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 736                 (rebase_remote, rebase_branch),
 737                 show=True, dir=test_master)
 738     else:
 739         run_cmd("git rebase --force-rebase %s/%s" %
 740                 (rebase_remote, rebase_branch),
 741                 show=True, dir=test_master)
 742     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 743                    (rebase_remote, rebase_branch),
 744                    dir=test_master, output=True)
 745     if diff == '':
 746         do_print("No differences between HEAD and %s/%s - exiting" %
 747                  (rebase_remote, rebase_branch))
 748         sys.exit(0)
 749     run_cmd("git describe %s/%s" %
 750             (rebase_remote, rebase_branch),
 751             show=True, dir=test_master)
 752     run_cmd("git describe HEAD", show=True, dir=test_master)
 753     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 754             (rebase_remote, rebase_branch),
 755             show=True, dir=test_master)
 756
 757
 758 def push_to(push_url, push_branch="master"):
 759     push_remote = "pushto"
 760     do_print("Pushing to %s" % push_url)
 761     if options.mark:
 762         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 763         run_cmd("git commit --amend -c HEAD", dir=test_master)
 764         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 765         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 766     run_cmd("git remote add -t %s %s %s" %
 767             (push_branch, push_remote, push_url),
 768             show=True, dir=test_master)
 769     run_cmd("git push %s +HEAD:%s" %
 770             (push_remote, push_branch),
 771             show=True, dir=test_master)
 772
 773
 774 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 775
 776 gitroot = find_git_root()
 777 if gitroot is None:
 778     raise Exception("Failed to find git root")
 779
 780 parser = OptionParser()
 781 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 782 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 783 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 784 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 785                   default=def_testbase)
 786 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 787 parser.add_option("", "--verbose", help="show all commands as they are run",
 788                   default=False, action="store_true")
 789 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 790                   default=None, type='str')
 791 parser.add_option("", "--pushto", help="push to a git url on success",
 792                   default=None, type='str')
 793 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 794                   default=False, action="store_true")
 795 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 796                   default=False, action="store_true")
 797 parser.add_option("", "--retry", help="automatically retry if master changes",
 798                   default=False, action="store_true")
 799 parser.add_option("", "--email", help="send email to the given address on failure",
 800                   type='str', default=None)
 801 parser.add_option("", "--email-from", help="send email from the given address",
 802                   type='str', default="autobuild@samba.org")
 803 parser.add_option("", "--email-server", help="send email via the given server",
 804                   type='str', default='localhost')
 805 parser.add_option("", "--always-email", help="always send email, even on success",
 806                   action="store_true")
 807 parser.add_option("", "--daemon", help="daemonize after initial setup",
 808                   action="store_true")
 809 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 810                   default="master", type='str')
 811 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 812                   default=gitroot, type='str')
 813 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 814                   default=False, action="store_true")
 815 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 816                   default='')
 817
 818
 819 def send_email(subject, text, log_tar):
 820     if options.email is None:
 821         do_print("not sending email because the recipient is not set")
 822         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 823                  (subject, text))
 824         return
 825     outer = MIMEMultipart()
 826     outer['Subject'] = subject
 827     outer['To'] = options.email
 828     outer['From'] = options.email_from
 829     outer['Date'] = email.utils.formatdate(localtime=True)
 830     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 831     outer.attach(MIMEText(text, 'plain'))
 832     if options.attach_logs:
 833         fp = open(log_tar, 'rb')
 834         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 835         fp.close()
 836         # Set the filename parameter
 837         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 838         outer.attach(msg)
 839     content = outer.as_string()
 840     s = smtplib.SMTP(options.email_server)
 841     s.sendmail(options.email_from, [options.email], content)
 842     s.set_debuglevel(1)
 843     s.quit()
 844
 845
 846 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 847                   elapsed_time, log_base=None, add_log_tail=True):
 848     '''send an email to options.email about the failure'''
 849     elapsed_minutes = elapsed_time / 60.0
 850     if log_base is None:
 851         log_base = gitroot
 852     text = '''
 853 Dear Developer,
 854
 855 Your autobuild on %s failed after %.1f minutes
 856 when trying to test %s with the following error:
 857
 858    %s
 859
 860 the autobuild has been abandoned. Please fix the error and resubmit.
 861
 862 A summary of the autobuild process is here:
 863
 864   %s/autobuild.log
 865 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 866
 867     if options.restrict_tests:
 868         text += """
 869 The build was restricted to tests matching %s\n""" % options.restrict_tests
 870
 871     if failed_task != 'rebase':
 872         text += '''
 873 You can see logs of the failed task here:
 874
 875   %s/%s.stdout
 876   %s/%s.stderr
 877
 878 or you can get full logs of all tasks in this job here:
 879
 880   %s/logs.tar.gz
 881
 882 The top commit for the tree that was built was:
 883
 884 %s
 885
 886 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 887
 888     if add_log_tail:
 889         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 890         lines = f.readlines()
 891         log_tail = "".join(lines[-50:])
 892         num_lines = len(lines)
 893         if num_lines < 50:
 894             # Also include stderr (compile failures) if < 50 lines of stdout
 895             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 896             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 897
 898         text += '''
 899 The last 50 lines of log messages:
 900
 901 %s
 902     ''' % log_tail
 903         f.close()
 904
 905     logs = os.path.join(gitroot, 'logs.tar.gz')
 906     send_email('autobuild[%s] failure on %s for task %s during %s'
 907                % (options.branch, platform.node(), failed_task, failed_stage),
 908                text, logs)
 909
 910
 911 def email_success(elapsed_time, log_base=None):
 912     '''send an email to options.email about a successful build'''
 913     if log_base is None:
 914         log_base = gitroot
 915     text = '''
 916 Dear Developer,
 917
 918 Your autobuild on %s has succeeded after %.1f minutes.
 919
 920 ''' % (platform.node(), elapsed_time / 60.)
 921
 922     if options.restrict_tests:
 923         text += """
 924 The build was restricted to tests matching %s\n""" % options.restrict_tests
 925
 926     if options.keeplogs:
 927         text += '''
 928
 929 you can get full logs of all tasks in this job here:
 930
 931   %s/logs.tar.gz
 932
 933 ''' % log_base
 934
 935     text += '''
 936 The top commit for the tree that was built was:
 937
 938 %s
 939 ''' % top_commit_msg
 940
 941     logs = os.path.join(gitroot, 'logs.tar.gz')
 942     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 943                text, logs)
 944
 945
 946 (options, args) = parser.parse_args()
 947
 948 if options.retry:
 949     if options.rebase is None:
 950         raise Exception('You can only use --retry if you also rebase')
 951
 952 testbase = "%s/b%u" % (options.testbase, os.getpid())
 953 test_master = "%s/master" % testbase
 954 test_prefix = "%s/prefix" % testbase
 955 test_tmpdir = "%s/tmp" % testbase
 956 os.environ['TMPDIR'] = test_tmpdir
 957
 958 # get the top commit message, for emails
 959 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 960 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 961
 962 try:
 963     os.makedirs(testbase)
 964 except Exception as reason:
 965     raise Exception("Unable to create %s : %s" % (testbase, reason))
 966 cleanup_list.append(testbase)
 967
 968 if options.daemon:
 969     logfile = os.path.join(testbase, "log")
 970     do_print("Forking into the background, writing progress to %s" % logfile)
 971     daemonize(logfile)
 972
 973 write_pidfile(gitroot + "/autobuild.pid")
 974
 975 start_time = time.time()
 976
 977 while True:
 978     try:
 979         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 980         os.makedirs(test_tmpdir)
 981         # The waf uninstall code removes empty directories all the way
 982         # up the tree.  Creating a file in test_tmpdir stops it from
 983         # being removed.
 984         run_cmd("touch %s" % os.path.join(test_tmpdir,
 985                                           ".directory-is-not-empty"), show=True)
 986         run_cmd("stat %s" % test_tmpdir, show=True)
 987         run_cmd("stat %s" % testbase, show=True)
 988         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 989     except Exception:
 990         cleanup()
 991         raise
 992
 993     try:
 994         try:
 995             if options.rebase is not None:
 996                 rebase_tree(options.rebase, rebase_branch=options.branch)
 997         except Exception:
 998             cleanup_list.append(gitroot + "/autobuild.pid")
 999             cleanup()
1000             elapsed_time = time.time() - start_time
1001             email_failure(-1, 'rebase', 'rebase', 'rebase',
1002                           'rebase on %s failed' % options.branch,
1003                           elapsed_time, log_base=options.log_base)
1004             sys.exit(1)
1005         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1006         if options.tail:
1007             blist.start_tail()
1008         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1009         if status != 0 or errstr != "retry":
1010             break
1011         cleanup()
1012     except Exception:
1013         cleanup()
1014         raise
1015
1016 cleanup_list.append(gitroot + "/autobuild.pid")
1017
1018 do_print(errstr)
1019
1020 blist.kill_kids()
1021 if options.tail:
1022     do_print("waiting for tail to flush")
1023     time.sleep(1)
1024
1025 elapsed_time = time.time() - start_time
1026 if status == 0:
1027     if options.passcmd is not None:
1028         do_print("Running passcmd: %s" % options.passcmd)
1029         run_cmd(options.passcmd, dir=test_master)
1030     if options.pushto is not None:
1031         push_to(options.pushto, push_branch=options.branch)
1032     if options.keeplogs or options.attach_logs:
1033         blist.tarlogs("logs.tar.gz")
1034         do_print("Logs in logs.tar.gz")
1035     if options.always_email:
1036         email_success(elapsed_time, log_base=options.log_base)
1037     blist.remove_logs()
1038     cleanup()
1039     do_print(errstr)
1040     sys.exit(0)
1041
1042 # something failed, gather a tar of the logs
1043 blist.tarlogs("logs.tar.gz")
1044
1045 if options.email is not None:
1046     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1047                   elapsed_time, log_base=options.log_base)
1048 else:
1049     elapsed_minutes = elapsed_time / 60.0
1050     print('''
1051
1052 ####################################################################
1053
1054 AUTOBUILD FAILURE
1055
1056 Your autobuild[%s] on %s failed after %.1f minutes
1057 when trying to test %s with the following error:
1058
1059    %s
1060
1061 the autobuild has been abandoned. Please fix the error and resubmit.
1062
1063 ####################################################################
1064
1065 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1066
1067 cleanup()
1068 do_print(errstr)
1069 do_print("Logs in logs.tar.gz")
1070 sys.exit(status)