script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py2": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py2": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py2": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py2": ".",
  52     "samba-ad-dc-ntvfs": ".",
  53     "samba-ad-dc-ntvfs-py2": ".",
  54     "samba-ad-dc-2": ".",
  55     "samba-ad-dc-2-py2": ".",
  56     "samba-ad-dc-backup": ".",
  57     "samba-systemkrb5": ".",
  58     "samba-nopython": ".",
  59     "samba-buildpy2-only": ".",
  60     "ldb": "lib/ldb",
  61     "tdb": "lib/tdb",
  62     "talloc": "lib/talloc",
  63     "replace": "lib/replace",
  64     "tevent": "lib/tevent",
  65     "pidl": "pidl"
  66 }
  67
  68 defaulttasks = builddirs.keys()
  69
  70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  71     defaulttasks.remove("samba-o3")
  72
  73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  74 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  75
  76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  83
  84 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  85     extra_python = ""
  86 else:
  87     extra_python = "--extra-python=/usr/bin/python2"
  88
  89 tasks = {
  90     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  91                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  92                ("make", "make all", "text/plain"),
  93                ("install", "make install", "text/plain"),
  94                ("test", "make autotest", "text/plain"),
  95                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  96                ("clean", "make clean", "text/plain")],
  97
  98     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  99     "samba": [
 100                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 101                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 102                 ("make", "make -j", "text/plain"),
 103                 ("test", "make test FAIL_IMMEDIATELY=1 "
 104                  "TESTS='${PY3_ONLY}"
 105                  "--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=ad_dc_ntvfs "
 110                  "--exclude-env=ad_dc_no_nss "
 111                  "--exclude-env=fl2003dc "
 112                  "--exclude-env=fl2008r2dc "
 113                  "--exclude-env=ad_member "
 114                  "--exclude-env=ad_member_idmap_rid "
 115                  "--exclude-env=ad_member_idmap_ad "
 116                  "--exclude-env=chgdcpass "
 117                  "--exclude-env=vampire_2000_dc "
 118                  "--exclude-env=fl2000dc "
 119                  "--exclude-env=fileserver "
 120                  "--exclude-env=backupfromdc "
 121                  "--exclude-env=restoredc "
 122                  "--exclude-env=renamedc "
 123                  "--exclude-env=offlinebackupdc "
 124                  "--exclude-env=labdc "
 125                  "'",
 126                  "text/plain"),
 127                 ("install", "make install", "text/plain"),
 128                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 129                 ("clean", "make clean", "text/plain")],
 130
 131     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 132                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 133                     ("make", "make -j", "text/plain"),
 134                     ("test", "make test FAIL_IMMEDIATELY=1 "
 135                      "TESTS='${PY3_ONLY}"
 136                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 137                     ("install", "make install", "text/plain"),
 138                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 139                     ("clean", "make clean", "text/plain")],
 140
 141     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                            ("make", "make -j", "text/plain"),
 144                            ("test", "make test FAIL_IMMEDIATELY=1 "
 145                             "TESTS='${PY3_ONLY}"
 146                             "--include-env=fileserver'", "text/plain"),
 147                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 148
 149     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 150                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 151                       ("make", "make -j", "text/plain"),
 152                       ("test", "make test FAIL_IMMEDIATELY=1 "
 153                        "TESTS='${PY3_ONLY}"
 154                        "--include-env=ad_dc "
 155                        "--include-env=fl2003dc "
 156                        "--include-env=fl2008r2dc "
 157                        "--include-env=ad_member "
 158                        "--include-env=ad_member_idmap_rid "
 159                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 160                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 161
 162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                         ("make", "make -j", "text/plain"),
 165                         ("test", "make test FAIL_IMMEDIATELY=1 "
 166                          "TESTS='${PY3_ONLY}"
 167                          "--include-env=chgdcpass "
 168                          "--include-env=vampire_2000_dc "
 169                          "--include-env=fl2000dc "
 170                          "--include-env=ad_dc_no_nss "
 171                          "'",
 172                          "text/plain"),
 173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 174
 175     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 176     # This is currently the longest task, so we don't randomly delay it.
 177     "samba-ad-dc-ntvfs": [
 178                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 179                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 180                       ("make", "make -j", "text/plain"),
 181                       ("test", "make test FAIL_IMMEDIATELY=1 "
 182                        "TESTS='${PY3_ONLY}"
 183                        "--include-env=ad_dc_ntvfs "
 184                        "'",
 185                        "text/plain"),
 186                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 187
 188     # run the backup/restore testenvs separately as they're fairly standalone
 189     # (and CI seems to max out at ~8 different DCs running at once)
 190     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 191                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 192                         ("make", "make -j", "text/plain"),
 193                         ("test", "make test FAIL_IMMEDIATELY=1 "
 194                          "TESTS='${PY3_ONLY}"
 195                          "--include-env=backupfromdc "
 196                          "--include-env=restoredc "
 197                          "--include-env=renamedc "
 198                          "--include-env=offlinebackupdc "
 199                          "--include-env=labdc "
 200                          "'",
 201                          "text/plain"),
 202                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 203
 204     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 205                           ("make", "make -j", "text/plain"),
 206                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 207
 208     # Test cross-compile infrastructure
 209     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 210                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 211                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 212                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 213                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 214                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 215                    ("compare-results", "script/compare_cc_results.py "
 216                     "./bin/c4che/default{} "
 217                     "./bin-xe/c4che/default{} "
 218                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 219
 220     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 221     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 222                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 223                    ("make", "make -j", "text/plain"),
 224                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 225                     "TESTS='${PY3_ONLY}"
 226                     "--include-env=ad_dc'", "text/plain"),
 227                    ("install", "make install", "text/plain"),
 228                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 229                    ("clean", "make clean", "text/plain")],
 230
 231     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 232
 233                      # make sure we have tdb around:
 234                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 235                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 236                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 237
 238
 239                      # build samba with cluster support (also building ctdb):
 240                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 241                      ("samba-make", "make", "text/plain"),
 242                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 243                      ("samba-install", "make install", "text/plain"),
 244                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 245
 246                      # clean up:
 247                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 248                      ("clean", "make clean", "text/plain"),
 249                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 250
 251     "samba-libs": [
 252                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 253                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 254                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 255                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 256
 257                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 258                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 259                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 260
 261                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 262                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 263                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 264
 265                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 266                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 267                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 268
 269                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 270                       ("nondevel-make", "make -j", "text/plain"),
 271                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 272                       ("nondevel-install", "make install", "text/plain"),
 273                       ("nondevel-dist", "make dist", "text/plain"),
 274
 275                       # retry with all modules shared
 276                       ("allshared-distclean", "make distclean", "text/plain"),
 277                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 278                       ("allshared-make", "make -j", "text/plain")],
 279
 280     "samba-none-env": [
 281                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 282                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 283                       ("make", "make -j", "text/plain"),
 284                       ("test", "make test "
 285                        "FAIL_IMMEDIATELY=1 "
 286                        "TESTS='${PY3_ONLY}"
 287                        "--include-env=none'",
 288                        "text/plain")],
 289
 290     "samba-static": [
 291                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 292                       # build with all modules static
 293                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 294                       ("allstatic-make", "make -j", "text/plain"),
 295                       ("allstatic-test", "make test "
 296                        "FAIL_IMMEDIATELY=1 "
 297                        "TESTS='samba3.smb2.create.*nt4_dc'",
 298                        "text/plain"),
 299
 300                       # retry without any required modules
 301                       ("none-distclean", "make distclean", "text/plain"),
 302                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 303                       ("none-make", "make -j", "text/plain"),
 304
 305                       # retry with nonshared smbd and smbtorture
 306                       ("nonshared-distclean", "make distclean", "text/plain"),
 307                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 308                       ("nonshared-make", "make -j", "text/plain")],
 309
 310     "samba-systemkrb5": [
 311                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 312                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 313                       ("make", "make -j", "text/plain"),
 314                       # we currently cannot run a full make test, a limited list of tests could be run
 315                       # via "make test TESTS=sometests"
 316                       ("test", "make test FAIL_IMMEDIATELY=1 "
 317                        "TESTS='${PY3_ONLY}"
 318                        "--include-env=ktest'", "text/plain"),
 319                       ("install", "make install", "text/plain"),
 320                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 321                       ("clean", "make clean", "text/plain")
 322                       ],
 323
 324     # Test Samba without python still builds.  When this test fails
 325     # due to more use of Python, the expectations is that the newly
 326     # failing part of the code should be disabled when
 327     # --disable-python is set (rather than major work being done to
 328     # support this environment).  The target here is for vendors
 329     # shipping a minimal smbd.
 330     "samba-nopython": [
 331                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 332                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 333                       ("make", "make -j", "text/plain"),
 334                       ("install", "make install", "text/plain"),
 335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 336                       ("clean", "make clean", "text/plain"),
 337
 338                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 339                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 340                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 341
 342                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 343                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 344                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 345
 346                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 347                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 348                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 349
 350                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 351                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 352                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 353
 354                       # retry against installed library packages
 355                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 356                       ("libs-make", "make -j", "text/plain"),
 357                       ("libs-install", "make install", "text/plain"),
 358                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 359                       ("libs-clean", "make clean", "text/plain")
 360                       ],
 361
 362
 363
 364     "ldb": [
 365               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 366               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 367               ("make", "make", "text/plain"),
 368               ("install", "make install", "text/plain"),
 369               ("test", "make test", "text/plain"),
 370               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 371               ("make-no-lmdb", "make", "text/plain"),
 372               ("install-no-lmdb", "make install", "text/plain"),
 373               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 374               ("distcheck", "make distcheck", "text/plain"),
 375               ("clean", "make clean", "text/plain")],
 376
 377     "tdb": [
 378               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 379               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 380               ("make", "make", "text/plain"),
 381               ("install", "make install", "text/plain"),
 382               ("test", "make test", "text/plain"),
 383               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 384               ("distcheck", "make distcheck", "text/plain"),
 385               ("clean", "make clean", "text/plain")],
 386
 387     "talloc": [
 388                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 389                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 390                  ("make", "make", "text/plain"),
 391                  ("install", "make install", "text/plain"),
 392                  ("test", "make test", "text/plain"),
 393                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 394                  ("distcheck", "make distcheck", "text/plain"),
 395                  ("clean", "make clean", "text/plain")],
 396
 397     "replace": [
 398                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 399                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 400                   ("make", "make", "text/plain"),
 401                   ("install", "make install", "text/plain"),
 402                   ("test", "make test", "text/plain"),
 403                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 404                   ("distcheck", "make distcheck", "text/plain"),
 405                   ("clean", "make clean", "text/plain")],
 406
 407     "tevent": [
 408                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 409                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 410                  ("make", "make", "text/plain"),
 411                  ("install", "make install", "text/plain"),
 412                  ("test", "make test", "text/plain"),
 413                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 414                  ("distcheck", "make distcheck", "text/plain"),
 415                  ("clean", "make clean", "text/plain")],
 416
 417     "pidl": [
 418         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 419         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 420         ("touch", "touch *.yp", "text/plain"),
 421         ("make", "make", "text/plain"),
 422         ("test", "make test", "text/plain"),
 423         ("install", "make install", "text/plain"),
 424         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 425         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 426         ("clean", "make clean", "text/plain")],
 427
 428     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 429                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 430                    ("make", "PYTHON='python' make -j", "text/plain"),
 431                    ("install", "PYTHON='python' make install", "text/plain"),
 432                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 433                    ("clean", "PYTHON='python' make clean", "text/plain")],
 434
 435
 436     # these are useful for debugging autobuild
 437     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 438     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 439
 440
 441 }
 442
 443
 444 def do_print(msg):
 445     print("%s" % msg)
 446     sys.stdout.flush()
 447     sys.stderr.flush()
 448
 449
 450 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 451     if show is None:
 452         show = options.verbose
 453     if show:
 454         do_print("Running: '%s' in '%s'" % (cmd, dir))
 455     if output:
 456         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 457     elif checkfail:
 458         return check_call(cmd, shell=True, cwd=dir)
 459     else:
 460         return call(cmd, shell=True, cwd=dir)
 461
 462
 463 class builder(object):
 464     '''handle build of one directory'''
 465
 466     def __init__(self, name, sequence, cp=True, py3=False):
 467         self.name = name
 468         self.py3 = py3
 469         if name in builddirs:
 470             self.dir = builddirs[name]
 471         else:
 472             self.dir = "."
 473
 474         self.tag = self.name.replace('/', '_')
 475         self.sequence = sequence
 476         self.next = 0
 477         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 478         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 479         if options.verbose:
 480             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 481             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 482         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 483         self.stdout = open(self.stdout_path, 'w')
 484         self.stderr = open(self.stderr_path, 'w')
 485         self.stdin  = open("/dev/null", 'r')
 486         self.sdir = "%s/%s" % (testbase, self.tag)
 487         self.prefix = "%s/%s" % (test_prefix, self.tag)
 488         run_cmd("rm -rf %s" % self.sdir)
 489         run_cmd("rm -rf %s" % self.prefix)
 490         if cp:
 491             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 492         else:
 493             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 494         self.start_next()
 495
 496     def start_next(self):
 497         if self.next == len(self.sequence):
 498             if not options.nocleanup:
 499                 run_cmd("rm -rf %s" % self.sdir)
 500                 run_cmd("rm -rf %s" % self.prefix)
 501             do_print('%s: Completed OK' % self.name)
 502             self.done = True
 503             return
 504         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 505         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 506         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 507         if self.py3:
 508             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 509             # The trailing space is important
 510             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
 511         else:
 512             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 513             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 514         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 515         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 516 #        if self.output_mime_type == "text/x-subunit":
 517 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 518         cwd = os.getcwd()
 519         os.chdir("%s/%s" % (self.sdir, self.dir))
 520         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 521         self.proc = Popen(self.cmd, shell=True,
 522                           close_fds=True,
 523                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 524         os.chdir(cwd)
 525         self.next += 1
 526
 527
 528 class buildlist(object):
 529     '''handle build of multiple directories'''
 530
 531     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 532         global tasks
 533         self.tlist = []
 534         self.tail_proc = None
 535         self.retry = None
 536         if tasknames == []:
 537             if options.restrict_tests:
 538                 tasknames = ["samba-test-only"]
 539             else:
 540                 tasknames = defaulttasks
 541         else:
 542             # If we are only running one test,
 543             # do not sleep randomly to wait for it to start
 544             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 545
 546         for n in tasknames:
 547             if n not in tasks and n.endswith("-py2"):
 548                 b = builder(n,
 549                             tasks[n[:-4]],
 550                             cp=n is not "pidl",
 551                             py3=True)
 552             else:
 553                 b = builder(n, tasks[n], cp=n is not "pidl")
 554             self.tlist.append(b)
 555         if options.retry:
 556             rebase_remote = "rebaseon"
 557             retry_task = [("retry",
 558                             '''set -e
 559                             git remote add -t %s %s %s
 560                             git fetch %s
 561                             while :; do
 562                               sleep 60
 563                               git describe %s/%s > old_remote_branch.desc
 564                               git fetch %s
 565                               git describe %s/%s > remote_branch.desc
 566                               diff old_remote_branch.desc remote_branch.desc
 567                             done
 568                            ''' % (
 569                                 rebase_branch, rebase_remote, rebase_url,
 570                                rebase_remote,
 571                                rebase_remote, rebase_branch,
 572                                rebase_remote,
 573                                rebase_remote, rebase_branch
 574                             ),
 575                             "test/plain")]
 576
 577             self.retry = builder('retry', retry_task, cp=False)
 578             self.need_retry = False
 579
 580     def kill_kids(self):
 581         if self.tail_proc is not None:
 582             self.tail_proc.terminate()
 583             self.tail_proc.wait()
 584             self.tail_proc = None
 585         if self.retry is not None:
 586             self.retry.proc.terminate()
 587             self.retry.proc.wait()
 588             self.retry = None
 589         for b in self.tlist:
 590             if b.proc is not None:
 591                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 592                 b.proc.terminate()
 593                 b.proc.wait()
 594                 b.proc = None
 595
 596     def wait_one(self):
 597         while True:
 598             none_running = True
 599             for b in self.tlist:
 600                 if b.proc is None:
 601                     continue
 602                 none_running = False
 603                 b.status = b.proc.poll()
 604                 if b.status is None:
 605                     continue
 606                 b.proc = None
 607                 return b
 608             if options.retry:
 609                 ret = self.retry.proc.poll()
 610                 if ret is not None:
 611                     self.need_retry = True
 612                     self.retry = None
 613                     return None
 614             if none_running:
 615                 return None
 616             time.sleep(0.1)
 617
 618     def run(self):
 619         while True:
 620             b = self.wait_one()
 621             if options.retry and self.need_retry:
 622                 self.kill_kids()
 623                 do_print("retry needed")
 624                 return (0, None, None, None, "retry")
 625             if b is None:
 626                 break
 627             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 628                 self.kill_kids()
 629                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 630             b.start_next()
 631         self.kill_kids()
 632         return (0, None, None, None, "All OK")
 633
 634     def write_system_info(self):
 635         filename = 'system-info.txt'
 636         f = open(filename, 'w')
 637         for cmd in ['uname -a',
 638                     'free',
 639                     'cat /proc/cpuinfo',
 640                     'cc --version',
 641                     'df -m .',
 642                     'df -m %s' % testbase]:
 643             out = run_cmd(cmd, output=True, checkfail=False)
 644             print('### %s' % cmd, file=f)
 645             print(out.decode('utf8', 'backslashreplace'), file=f)
 646             print(file=f)
 647         f.close()
 648         return filename
 649
 650     def tarlogs(self, fname):
 651         tar = tarfile.open(fname, "w:gz")
 652         for b in self.tlist:
 653             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 654             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 655         if os.path.exists("autobuild.log"):
 656             tar.add("autobuild.log")
 657         sys_info = self.write_system_info()
 658         tar.add(sys_info)
 659         tar.close()
 660
 661     def remove_logs(self):
 662         for b in self.tlist:
 663             os.unlink(b.stdout_path)
 664             os.unlink(b.stderr_path)
 665
 666     def start_tail(self):
 667         cmd = ["tail", "-f"]
 668         for b in self.tlist:
 669             cmd.append(b.stdout_path)
 670             cmd.append(b.stderr_path)
 671         self.tail_proc = Popen(cmd, close_fds=True)
 672
 673
 674 def cleanup():
 675     if options.nocleanup:
 676         return
 677     run_cmd("stat %s || true" % test_tmpdir, show=True)
 678     run_cmd("stat %s" % testbase, show=True)
 679     do_print("Cleaning up %r" % cleanup_list)
 680     for d in cleanup_list:
 681         run_cmd("rm -rf %s" % d)
 682
 683
 684 def find_git_root():
 685     '''get to the top of the git repo'''
 686     p = os.getcwd()
 687     while p != '/':
 688         if os.path.isdir(os.path.join(p, ".git")):
 689             return p
 690         p = os.path.abspath(os.path.join(p, '..'))
 691     return None
 692
 693
 694 def daemonize(logfile):
 695     pid = os.fork()
 696     if pid == 0:  # Parent
 697         os.setsid()
 698         pid = os.fork()
 699         if pid != 0:  # Actual daemon
 700             os._exit(0)
 701     else:  # Grandparent
 702         os._exit(0)
 703
 704     import resource      # Resource usage information.
 705     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 706     if maxfd == resource.RLIM_INFINITY:
 707         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 708     for fd in range(0, maxfd):
 709         try:
 710             os.close(fd)
 711         except OSError:
 712             pass
 713     os.open(logfile, os.O_RDWR | os.O_CREAT)
 714     os.dup2(0, 1)
 715     os.dup2(0, 2)
 716
 717
 718 def write_pidfile(fname):
 719     '''write a pid file, cleanup on exit'''
 720     f = open(fname, mode='w')
 721     f.write("%u\n" % os.getpid())
 722     f.close()
 723
 724
 725 def rebase_tree(rebase_url, rebase_branch="master"):
 726     rebase_remote = "rebaseon"
 727     do_print("Rebasing on %s" % rebase_url)
 728     run_cmd("git describe HEAD", show=True, dir=test_master)
 729     run_cmd("git remote add -t %s %s %s" %
 730             (rebase_branch, rebase_remote, rebase_url),
 731             show=True, dir=test_master)
 732     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 733     if options.fix_whitespace:
 734         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 735                 (rebase_remote, rebase_branch),
 736                 show=True, dir=test_master)
 737     else:
 738         run_cmd("git rebase --force-rebase %s/%s" %
 739                 (rebase_remote, rebase_branch),
 740                 show=True, dir=test_master)
 741     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 742                    (rebase_remote, rebase_branch),
 743                    dir=test_master, output=True)
 744     if diff == '':
 745         do_print("No differences between HEAD and %s/%s - exiting" %
 746                  (rebase_remote, rebase_branch))
 747         sys.exit(0)
 748     run_cmd("git describe %s/%s" %
 749             (rebase_remote, rebase_branch),
 750             show=True, dir=test_master)
 751     run_cmd("git describe HEAD", show=True, dir=test_master)
 752     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 753             (rebase_remote, rebase_branch),
 754             show=True, dir=test_master)
 755
 756
 757 def push_to(push_url, push_branch="master"):
 758     push_remote = "pushto"
 759     do_print("Pushing to %s" % push_url)
 760     if options.mark:
 761         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 762         run_cmd("git commit --amend -c HEAD", dir=test_master)
 763         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 764         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 765     run_cmd("git remote add -t %s %s %s" %
 766             (push_branch, push_remote, push_url),
 767             show=True, dir=test_master)
 768     run_cmd("git push %s +HEAD:%s" %
 769             (push_remote, push_branch),
 770             show=True, dir=test_master)
 771
 772
 773 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 774
 775 gitroot = find_git_root()
 776 if gitroot is None:
 777     raise Exception("Failed to find git root")
 778
 779 parser = OptionParser()
 780 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 781 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 782 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 783 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 784                   default=def_testbase)
 785 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 786 parser.add_option("", "--verbose", help="show all commands as they are run",
 787                   default=False, action="store_true")
 788 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 789                   default=None, type='str')
 790 parser.add_option("", "--pushto", help="push to a git url on success",
 791                   default=None, type='str')
 792 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 793                   default=False, action="store_true")
 794 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 795                   default=False, action="store_true")
 796 parser.add_option("", "--retry", help="automatically retry if master changes",
 797                   default=False, action="store_true")
 798 parser.add_option("", "--email", help="send email to the given address on failure",
 799                   type='str', default=None)
 800 parser.add_option("", "--email-from", help="send email from the given address",
 801                   type='str', default="autobuild@samba.org")
 802 parser.add_option("", "--email-server", help="send email via the given server",
 803                   type='str', default='localhost')
 804 parser.add_option("", "--always-email", help="always send email, even on success",
 805                   action="store_true")
 806 parser.add_option("", "--daemon", help="daemonize after initial setup",
 807                   action="store_true")
 808 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 809                   default="master", type='str')
 810 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 811                   default=gitroot, type='str')
 812 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 813                   default=False, action="store_true")
 814 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 815                   default='')
 816
 817
 818 def send_email(subject, text, log_tar):
 819     if options.email is None:
 820         do_print("not sending email because the recipient is not set")
 821         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 822                  (subject, text))
 823         return
 824     outer = MIMEMultipart()
 825     outer['Subject'] = subject
 826     outer['To'] = options.email
 827     outer['From'] = options.email_from
 828     outer['Date'] = email.utils.formatdate(localtime=True)
 829     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 830     outer.attach(MIMEText(text, 'plain'))
 831     if options.attach_logs:
 832         fp = open(log_tar, 'rb')
 833         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 834         fp.close()
 835         # Set the filename parameter
 836         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 837         outer.attach(msg)
 838     content = outer.as_string()
 839     s = smtplib.SMTP(options.email_server)
 840     s.sendmail(options.email_from, [options.email], content)
 841     s.set_debuglevel(1)
 842     s.quit()
 843
 844
 845 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 846                   elapsed_time, log_base=None, add_log_tail=True):
 847     '''send an email to options.email about the failure'''
 848     elapsed_minutes = elapsed_time / 60.0
 849     if log_base is None:
 850         log_base = gitroot
 851     text = '''
 852 Dear Developer,
 853
 854 Your autobuild on %s failed after %.1f minutes
 855 when trying to test %s with the following error:
 856
 857    %s
 858
 859 the autobuild has been abandoned. Please fix the error and resubmit.
 860
 861 A summary of the autobuild process is here:
 862
 863   %s/autobuild.log
 864 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 865
 866     if options.restrict_tests:
 867         text += """
 868 The build was restricted to tests matching %s\n""" % options.restrict_tests
 869
 870     if failed_task != 'rebase':
 871         text += '''
 872 You can see logs of the failed task here:
 873
 874   %s/%s.stdout
 875   %s/%s.stderr
 876
 877 or you can get full logs of all tasks in this job here:
 878
 879   %s/logs.tar.gz
 880
 881 The top commit for the tree that was built was:
 882
 883 %s
 884
 885 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 886
 887     if add_log_tail:
 888         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 889         lines = f.readlines()
 890         log_tail = "".join(lines[-50:])
 891         num_lines = len(lines)
 892         if num_lines < 50:
 893             # Also include stderr (compile failures) if < 50 lines of stdout
 894             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 895             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 896
 897         text += '''
 898 The last 50 lines of log messages:
 899
 900 %s
 901     ''' % log_tail
 902         f.close()
 903
 904     logs = os.path.join(gitroot, 'logs.tar.gz')
 905     send_email('autobuild[%s] failure on %s for task %s during %s'
 906                % (options.branch, platform.node(), failed_task, failed_stage),
 907                text, logs)
 908
 909
 910 def email_success(elapsed_time, log_base=None):
 911     '''send an email to options.email about a successful build'''
 912     if log_base is None:
 913         log_base = gitroot
 914     text = '''
 915 Dear Developer,
 916
 917 Your autobuild on %s has succeeded after %.1f minutes.
 918
 919 ''' % (platform.node(), elapsed_time / 60.)
 920
 921     if options.restrict_tests:
 922         text += """
 923 The build was restricted to tests matching %s\n""" % options.restrict_tests
 924
 925     if options.keeplogs:
 926         text += '''
 927
 928 you can get full logs of all tasks in this job here:
 929
 930   %s/logs.tar.gz
 931
 932 ''' % log_base
 933
 934     text += '''
 935 The top commit for the tree that was built was:
 936
 937 %s
 938 ''' % top_commit_msg
 939
 940     logs = os.path.join(gitroot, 'logs.tar.gz')
 941     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 942                text, logs)
 943
 944
 945 (options, args) = parser.parse_args()
 946
 947 if options.retry:
 948     if options.rebase is None:
 949         raise Exception('You can only use --retry if you also rebase')
 950
 951 testbase = "%s/b%u" % (options.testbase, os.getpid())
 952 test_master = "%s/master" % testbase
 953 test_prefix = "%s/prefix" % testbase
 954 test_tmpdir = "%s/tmp" % testbase
 955 os.environ['TMPDIR'] = test_tmpdir
 956
 957 # get the top commit message, for emails
 958 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 959 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 960
 961 try:
 962     os.makedirs(testbase)
 963 except Exception as reason:
 964     raise Exception("Unable to create %s : %s" % (testbase, reason))
 965 cleanup_list.append(testbase)
 966
 967 if options.daemon:
 968     logfile = os.path.join(testbase, "log")
 969     do_print("Forking into the background, writing progress to %s" % logfile)
 970     daemonize(logfile)
 971
 972 write_pidfile(gitroot + "/autobuild.pid")
 973
 974 start_time = time.time()
 975
 976 while True:
 977     try:
 978         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 979         os.makedirs(test_tmpdir)
 980         # The waf uninstall code removes empty directories all the way
 981         # up the tree.  Creating a file in test_tmpdir stops it from
 982         # being removed.
 983         run_cmd("touch %s" % os.path.join(test_tmpdir,
 984                                           ".directory-is-not-empty"), show=True)
 985         run_cmd("stat %s" % test_tmpdir, show=True)
 986         run_cmd("stat %s" % testbase, show=True)
 987         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 988     except Exception:
 989         cleanup()
 990         raise
 991
 992     try:
 993         try:
 994             if options.rebase is not None:
 995                 rebase_tree(options.rebase, rebase_branch=options.branch)
 996         except Exception:
 997             cleanup_list.append(gitroot + "/autobuild.pid")
 998             cleanup()
 999             elapsed_time = time.time() - start_time
1000             email_failure(-1, 'rebase', 'rebase', 'rebase',
1001                           'rebase on %s failed' % options.branch,
1002                           elapsed_time, log_base=options.log_base)
1003             sys.exit(1)
1004         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1005         if options.tail:
1006             blist.start_tail()
1007         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1008         if status != 0 or errstr != "retry":
1009             break
1010         cleanup()
1011     except Exception:
1012         cleanup()
1013         raise
1014
1015 cleanup_list.append(gitroot + "/autobuild.pid")
1016
1017 do_print(errstr)
1018
1019 blist.kill_kids()
1020 if options.tail:
1021     do_print("waiting for tail to flush")
1022     time.sleep(1)
1023
1024 elapsed_time = time.time() - start_time
1025 if status == 0:
1026     if options.passcmd is not None:
1027         do_print("Running passcmd: %s" % options.passcmd)
1028         run_cmd(options.passcmd, dir=test_master)
1029     if options.pushto is not None:
1030         push_to(options.pushto, push_branch=options.branch)
1031     if options.keeplogs or options.attach_logs:
1032         blist.tarlogs("logs.tar.gz")
1033         do_print("Logs in logs.tar.gz")
1034     if options.always_email:
1035         email_success(elapsed_time, log_base=options.log_base)
1036     blist.remove_logs()
1037     cleanup()
1038     do_print(errstr)
1039     sys.exit(0)
1040
1041 # something failed, gather a tar of the logs
1042 blist.tarlogs("logs.tar.gz")
1043
1044 if options.email is not None:
1045     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1046                   elapsed_time, log_base=options.log_base)
1047 else:
1048     elapsed_minutes = elapsed_time / 60.0
1049     print('''
1050
1051 ####################################################################
1052
1053 AUTOBUILD FAILURE
1054
1055 Your autobuild[%s] on %s failed after %.1f minutes
1056 when trying to test %s with the following error:
1057
1058    %s
1059
1060 the autobuild has been abandoned. Please fix the error and resubmit.
1061
1062 ####################################################################
1063
1064 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1065
1066 cleanup()
1067 do_print(errstr)
1068 do_print("Logs in logs.tar.gz")
1069 sys.exit(status)