script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-xc": ".",
  42     "samba-o3": ".",
  43     "samba-ctdb": ".",
  44     "samba-libs": ".",
  45     "samba-static": ".",
  46     "samba-none-env": ".",
  47     "samba-ad-dc": ".",
  48     "samba-ad-dc-ntvfs": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-backup": ".",
  51     "samba-systemkrb5": ".",
  52     "samba-nopython": ".",
  53     "samba-buildpy2-only": ".",
  54     "ldb": "lib/ldb",
  55     "tdb": "lib/tdb",
  56     "talloc": "lib/talloc",
  57     "replace": "lib/replace",
  58     "tevent": "lib/tevent",
  59     "pidl": "pidl"
  60 }
  61
  62 defaulttasks = builddirs.keys()
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  69
  70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  77
  78 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  79     extra_python = ""
  80 else:
  81     extra_python = "--extra-python=/usr/bin/python2"
  82
  83 tasks = {
  84     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  85                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  86                ("make", "make all", "text/plain"),
  87                ("install", "make install", "text/plain"),
  88                ("test", "make autotest", "text/plain"),
  89                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  90                ("clean", "make clean", "text/plain")],
  91
  92     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  93     "samba": [
  94                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  95                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  96                 ("make", "make -j", "text/plain"),
  97                 ("test", "make test FAIL_IMMEDIATELY=1 "
  98                  "TESTS='"
  99                  "--exclude-env=none "
 100                  "--exclude-env=nt4_dc "
 101                  "--exclude-env=nt4_member "
 102                  "--exclude-env=ad_dc "
 103                  "--exclude-env=ad_dc_ntvfs "
 104                  "--exclude-env=ad_dc_no_nss "
 105                  "--exclude-env=fl2003dc "
 106                  "--exclude-env=fl2008r2dc "
 107                  "--exclude-env=ad_member "
 108                  "--exclude-env=ad_member_idmap_rid "
 109                  "--exclude-env=ad_member_idmap_ad "
 110                  "--exclude-env=chgdcpass "
 111                  "--exclude-env=vampire_2000_dc "
 112                  "--exclude-env=fl2000dc "
 113                  "--exclude-env=fileserver "
 114                  "--exclude-env=backupfromdc "
 115                  "--exclude-env=restoredc "
 116                  "--exclude-env=renamedc "
 117                  "--exclude-env=offlinebackupdc "
 118                  "--exclude-env=labdc "
 119                  "'",
 120                  "text/plain"),
 121                 ("install", "make install", "text/plain"),
 122                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 123                 ("clean", "make clean", "text/plain")],
 124
 125     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 127                     ("make", "make -j", "text/plain"),
 128                     ("test", "make test FAIL_IMMEDIATELY=1 "
 129                      "TESTS='"
 130                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 131                     ("install", "make install", "text/plain"),
 132                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 133                     ("clean", "make clean", "text/plain")],
 134
 135     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 136                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 137                            ("make", "make -j", "text/plain"),
 138                            ("test", "make test FAIL_IMMEDIATELY=1 "
 139                             "TESTS='"
 140                             "--include-env=fileserver'", "text/plain"),
 141                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 142
 143     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 144                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 145                       ("make", "make -j", "text/plain"),
 146                       ("test", "make test FAIL_IMMEDIATELY=1 "
 147                        "TESTS='"
 148                        "--include-env=ad_dc "
 149                        "--include-env=fl2003dc "
 150                        "--include-env=fl2008r2dc "
 151                        "--include-env=ad_member "
 152                        "--include-env=ad_member_idmap_rid "
 153                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 154                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 155
 156     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 157                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 158                         ("make", "make -j", "text/plain"),
 159                         ("test", "make test FAIL_IMMEDIATELY=1 "
 160                          "TESTS='"
 161                          "--include-env=chgdcpass "
 162                          "--include-env=vampire_2000_dc "
 163                          "--include-env=fl2000dc "
 164                          "--include-env=ad_dc_no_nss "
 165                          "'",
 166                          "text/plain"),
 167                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 168
 169     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 170     # This is currently the longest task, so we don't randomly delay it.
 171     "samba-ad-dc-ntvfs": [
 172                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 173                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 174                       ("make", "make -j", "text/plain"),
 175                       ("test", "make test FAIL_IMMEDIATELY=1 "
 176                        "TESTS='"
 177                        "--include-env=ad_dc_ntvfs "
 178                        "'",
 179                        "text/plain"),
 180                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 181
 182     # run the backup/restore testenvs separately as they're fairly standalone
 183     # (and CI seems to max out at ~8 different DCs running at once)
 184     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 185                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 186                         ("make", "make -j", "text/plain"),
 187                         ("test", "make test FAIL_IMMEDIATELY=1 "
 188                          "TESTS='"
 189                          "--include-env=backupfromdc "
 190                          "--include-env=restoredc "
 191                          "--include-env=renamedc "
 192                          "--include-env=offlinebackupdc "
 193                          "--include-env=labdc "
 194                          "'",
 195                          "text/plain"),
 196                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 197
 198     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 199                           ("make", "make -j", "text/plain"),
 200                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 201
 202     # Test cross-compile infrastructure
 203     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 204                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 205                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 206                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 207                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 208                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 209                    ("compare-results", "script/compare_cc_results.py "
 210                     "./bin/c4che/default{} "
 211                     "./bin-xe/c4che/default{} "
 212                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 213
 214     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 215     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 216                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 217                    ("make", "make -j", "text/plain"),
 218                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 219                     "TESTS='"
 220                     "--include-env=ad_dc'", "text/plain"),
 221                    ("install", "make install", "text/plain"),
 222                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 223                    ("clean", "make clean", "text/plain")],
 224
 225     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 226
 227                      # make sure we have tdb around:
 228                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 229                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 230                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 231
 232
 233                      # build samba with cluster support (also building ctdb):
 234                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 235                      ("samba-make", "make", "text/plain"),
 236                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 237                      ("samba-install", "make install", "text/plain"),
 238                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 239
 240                      # clean up:
 241                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 242                      ("clean", "make clean", "text/plain"),
 243                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 244
 245     "samba-libs": [
 246                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 247                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 248                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 249                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 250
 251                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 252                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 253                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 254
 255                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 256                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 257                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 258
 259                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 260                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 261                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 262
 263                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 264                       ("nondevel-make", "make -j", "text/plain"),
 265                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 266                       ("nondevel-install", "make install", "text/plain"),
 267                       ("nondevel-dist", "make dist", "text/plain"),
 268
 269                       # retry with all modules shared
 270                       ("allshared-distclean", "make distclean", "text/plain"),
 271                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 272                       ("allshared-make", "make -j", "text/plain")],
 273
 274     "samba-none-env": [
 275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 276                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 277                       ("make", "make -j", "text/plain"),
 278                       ("test", "make test "
 279                        "FAIL_IMMEDIATELY=1 "
 280                        "TESTS='"
 281                        "--include-env=none'",
 282                        "text/plain")],
 283
 284     "samba-static": [
 285                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 286                       # build with all modules static
 287                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 288                       ("allstatic-make", "make -j", "text/plain"),
 289                       ("allstatic-test", "make test "
 290                        "FAIL_IMMEDIATELY=1 "
 291                        "TESTS='samba3.smb2.create.*nt4_dc'",
 292                        "text/plain"),
 293
 294                       # retry without any required modules
 295                       ("none-distclean", "make distclean", "text/plain"),
 296                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 297                       ("none-make", "make -j", "text/plain"),
 298
 299                       # retry with nonshared smbd and smbtorture
 300                       ("nonshared-distclean", "make distclean", "text/plain"),
 301                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 302                       ("nonshared-make", "make -j", "text/plain")],
 303
 304     "samba-systemkrb5": [
 305                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 306                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 307                       ("make", "make -j", "text/plain"),
 308                       # we currently cannot run a full make test, a limited list of tests could be run
 309                       # via "make test TESTS=sometests"
 310                       ("test", "make test FAIL_IMMEDIATELY=1 "
 311                        "TESTS='"
 312                        "--include-env=ktest'", "text/plain"),
 313                       ("install", "make install", "text/plain"),
 314                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 315                       ("clean", "make clean", "text/plain")
 316                       ],
 317
 318     # Test Samba without python still builds.  When this test fails
 319     # due to more use of Python, the expectations is that the newly
 320     # failing part of the code should be disabled when
 321     # --disable-python is set (rather than major work being done to
 322     # support this environment).  The target here is for vendors
 323     # shipping a minimal smbd.
 324     "samba-nopython": [
 325                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 326                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 327                       ("make", "make -j", "text/plain"),
 328                       ("install", "make install", "text/plain"),
 329                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 330                       ("clean", "make clean", "text/plain"),
 331
 332                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 333                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 334                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 335
 336                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 337                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 338                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 339
 340                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 341                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 342                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 343
 344                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 345                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 346                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 347
 348                       # retry against installed library packages
 349                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 350                       ("libs-make", "make -j", "text/plain"),
 351                       ("libs-install", "make install", "text/plain"),
 352                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 353                       ("libs-clean", "make clean", "text/plain")
 354                       ],
 355
 356
 357
 358     "ldb": [
 359               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 360               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 361               ("make", "make", "text/plain"),
 362               ("install", "make install", "text/plain"),
 363               ("test", "make test", "text/plain"),
 364               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 365               ("make-no-lmdb", "make", "text/plain"),
 366               ("install-no-lmdb", "make install", "text/plain"),
 367               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368               ("distcheck", "make distcheck", "text/plain"),
 369               ("clean", "make clean", "text/plain")],
 370
 371     "tdb": [
 372               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 374               ("make", "make", "text/plain"),
 375               ("install", "make install", "text/plain"),
 376               ("test", "make test", "text/plain"),
 377               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378               ("distcheck", "make distcheck", "text/plain"),
 379               ("clean", "make clean", "text/plain")],
 380
 381     "talloc": [
 382                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 383                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 384                  ("make", "make", "text/plain"),
 385                  ("install", "make install", "text/plain"),
 386                  ("test", "make test", "text/plain"),
 387                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 388                  ("distcheck", "make distcheck", "text/plain"),
 389                  ("clean", "make clean", "text/plain")],
 390
 391     "replace": [
 392                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 393                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 394                   ("make", "make", "text/plain"),
 395                   ("install", "make install", "text/plain"),
 396                   ("test", "make test", "text/plain"),
 397                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 398                   ("distcheck", "make distcheck", "text/plain"),
 399                   ("clean", "make clean", "text/plain")],
 400
 401     "tevent": [
 402                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 403                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 404                  ("make", "make", "text/plain"),
 405                  ("install", "make install", "text/plain"),
 406                  ("test", "make test", "text/plain"),
 407                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 408                  ("distcheck", "make distcheck", "text/plain"),
 409                  ("clean", "make clean", "text/plain")],
 410
 411     "pidl": [
 412         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 413         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 414         ("touch", "touch *.yp", "text/plain"),
 415         ("make", "make", "text/plain"),
 416         ("test", "make test", "text/plain"),
 417         ("install", "make install", "text/plain"),
 418         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 419         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 420         ("clean", "make clean", "text/plain")],
 421
 422     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 423                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 424                    ("make", "PYTHON='python' make -j", "text/plain"),
 425                    ("install", "PYTHON='python' make install", "text/plain"),
 426                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 427                    ("clean", "PYTHON='python' make clean", "text/plain")],
 428
 429
 430     # these are useful for debugging autobuild
 431     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 432     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 433
 434
 435 }
 436
 437
 438 def do_print(msg):
 439     print("%s" % msg)
 440     sys.stdout.flush()
 441     sys.stderr.flush()
 442
 443
 444 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 445     if show is None:
 446         show = options.verbose
 447     if show:
 448         do_print("Running: '%s' in '%s'" % (cmd, dir))
 449     if output:
 450         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 451     elif checkfail:
 452         return check_call(cmd, shell=True, cwd=dir)
 453     else:
 454         return call(cmd, shell=True, cwd=dir)
 455
 456
 457 class builder(object):
 458     '''handle build of one directory'''
 459
 460     def __init__(self, name, sequence, cp=True, py2=False):
 461         self.name = name
 462         self.py2 = py2
 463         if name in builddirs:
 464             self.dir = builddirs[name]
 465         else:
 466             self.dir = "."
 467
 468         self.tag = self.name.replace('/', '_')
 469         self.sequence = sequence
 470         self.next = 0
 471         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 472         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 473         if options.verbose:
 474             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 475             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 476         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 477         self.stdout = open(self.stdout_path, 'w')
 478         self.stderr = open(self.stderr_path, 'w')
 479         self.stdin  = open("/dev/null", 'r')
 480         self.sdir = "%s/%s" % (testbase, self.tag)
 481         self.prefix = "%s/%s" % (test_prefix, self.tag)
 482         run_cmd("rm -rf %s" % self.sdir)
 483         run_cmd("rm -rf %s" % self.prefix)
 484         if cp:
 485             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 486         else:
 487             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 488         self.start_next()
 489
 490     def start_next(self):
 491         if self.next == len(self.sequence):
 492             if not options.nocleanup:
 493                 run_cmd("rm -rf %s" % self.sdir)
 494                 run_cmd("rm -rf %s" % self.prefix)
 495             do_print('%s: Completed OK' % self.name)
 496             self.done = True
 497             return
 498         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 499         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 500         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 501         if self.py2:
 502             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 503         else:
 504             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 505         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 506         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 507 #        if self.output_mime_type == "text/x-subunit":
 508 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 509         cwd = os.getcwd()
 510         os.chdir("%s/%s" % (self.sdir, self.dir))
 511         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 512         self.proc = Popen(self.cmd, shell=True,
 513                           close_fds=True,
 514                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 515         os.chdir(cwd)
 516         self.next += 1
 517
 518
 519 class buildlist(object):
 520     '''handle build of multiple directories'''
 521
 522     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 523         global tasks
 524         self.tlist = []
 525         self.tail_proc = None
 526         self.retry = None
 527         if tasknames == []:
 528             if options.restrict_tests:
 529                 tasknames = ["samba-test-only"]
 530             else:
 531                 tasknames = defaulttasks
 532         else:
 533             # If we are only running one test,
 534             # do not sleep randomly to wait for it to start
 535             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 536
 537         for n in tasknames:
 538             if n not in tasks and n.endswith("-py2"):
 539                 b = builder(n,
 540                             tasks[n[:-4]],
 541                             cp=n is not "pidl",
 542                             py2=True)
 543             else:
 544                 b = builder(n, tasks[n], cp=n is not "pidl")
 545             self.tlist.append(b)
 546         if options.retry:
 547             rebase_remote = "rebaseon"
 548             retry_task = [("retry",
 549                             '''set -e
 550                             git remote add -t %s %s %s
 551                             git fetch %s
 552                             while :; do
 553                               sleep 60
 554                               git describe %s/%s > old_remote_branch.desc
 555                               git fetch %s
 556                               git describe %s/%s > remote_branch.desc
 557                               diff old_remote_branch.desc remote_branch.desc
 558                             done
 559                            ''' % (
 560                                 rebase_branch, rebase_remote, rebase_url,
 561                                rebase_remote,
 562                                rebase_remote, rebase_branch,
 563                                rebase_remote,
 564                                rebase_remote, rebase_branch
 565                             ),
 566                             "test/plain")]
 567
 568             self.retry = builder('retry', retry_task, cp=False)
 569             self.need_retry = False
 570
 571     def kill_kids(self):
 572         if self.tail_proc is not None:
 573             self.tail_proc.terminate()
 574             self.tail_proc.wait()
 575             self.tail_proc = None
 576         if self.retry is not None:
 577             self.retry.proc.terminate()
 578             self.retry.proc.wait()
 579             self.retry = None
 580         for b in self.tlist:
 581             if b.proc is not None:
 582                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 583                 b.proc.terminate()
 584                 b.proc.wait()
 585                 b.proc = None
 586
 587     def wait_one(self):
 588         while True:
 589             none_running = True
 590             for b in self.tlist:
 591                 if b.proc is None:
 592                     continue
 593                 none_running = False
 594                 b.status = b.proc.poll()
 595                 if b.status is None:
 596                     continue
 597                 b.proc = None
 598                 return b
 599             if options.retry:
 600                 ret = self.retry.proc.poll()
 601                 if ret is not None:
 602                     self.need_retry = True
 603                     self.retry = None
 604                     return None
 605             if none_running:
 606                 return None
 607             time.sleep(0.1)
 608
 609     def run(self):
 610         while True:
 611             b = self.wait_one()
 612             if options.retry and self.need_retry:
 613                 self.kill_kids()
 614                 do_print("retry needed")
 615                 return (0, None, None, None, "retry")
 616             if b is None:
 617                 break
 618             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 619                 self.kill_kids()
 620                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 621             b.start_next()
 622         self.kill_kids()
 623         return (0, None, None, None, "All OK")
 624
 625     def write_system_info(self):
 626         filename = 'system-info.txt'
 627         f = open(filename, 'w')
 628         for cmd in ['uname -a',
 629                     'lsb_release -a',
 630                     'free',
 631                     'mount',
 632                     'cat /proc/cpuinfo',
 633                     'cc --version',
 634                     'df -m .',
 635                     'df -m %s' % testbase]:
 636             out = run_cmd(cmd, output=True, checkfail=False)
 637             print('### %s' % cmd, file=f)
 638             print(out.decode('utf8', 'backslashreplace'), file=f)
 639             print(file=f)
 640         f.close()
 641         return filename
 642
 643     def tarlogs(self, fname):
 644         tar = tarfile.open(fname, "w:gz")
 645         for b in self.tlist:
 646             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 647             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 648         if os.path.exists("autobuild.log"):
 649             tar.add("autobuild.log")
 650         sys_info = self.write_system_info()
 651         tar.add(sys_info)
 652         tar.close()
 653
 654     def remove_logs(self):
 655         for b in self.tlist:
 656             os.unlink(b.stdout_path)
 657             os.unlink(b.stderr_path)
 658
 659     def start_tail(self):
 660         cmd = ["tail", "-f"]
 661         for b in self.tlist:
 662             cmd.append(b.stdout_path)
 663             cmd.append(b.stderr_path)
 664         self.tail_proc = Popen(cmd, close_fds=True)
 665
 666
 667 def cleanup():
 668     if options.nocleanup:
 669         return
 670     run_cmd("stat %s || true" % test_tmpdir, show=True)
 671     run_cmd("stat %s" % testbase, show=True)
 672     do_print("Cleaning up %r" % cleanup_list)
 673     for d in cleanup_list:
 674         run_cmd("rm -rf %s" % d)
 675
 676
 677 def find_git_root():
 678     '''get to the top of the git repo'''
 679     p = os.getcwd()
 680     while p != '/':
 681         if os.path.isdir(os.path.join(p, ".git")):
 682             return p
 683         p = os.path.abspath(os.path.join(p, '..'))
 684     return None
 685
 686
 687 def daemonize(logfile):
 688     pid = os.fork()
 689     if pid == 0:  # Parent
 690         os.setsid()
 691         pid = os.fork()
 692         if pid != 0:  # Actual daemon
 693             os._exit(0)
 694     else:  # Grandparent
 695         os._exit(0)
 696
 697     import resource      # Resource usage information.
 698     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 699     if maxfd == resource.RLIM_INFINITY:
 700         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 701     for fd in range(0, maxfd):
 702         try:
 703             os.close(fd)
 704         except OSError:
 705             pass
 706     os.open(logfile, os.O_RDWR | os.O_CREAT)
 707     os.dup2(0, 1)
 708     os.dup2(0, 2)
 709
 710
 711 def write_pidfile(fname):
 712     '''write a pid file, cleanup on exit'''
 713     f = open(fname, mode='w')
 714     f.write("%u\n" % os.getpid())
 715     f.close()
 716
 717
 718 def rebase_tree(rebase_url, rebase_branch="master"):
 719     rebase_remote = "rebaseon"
 720     do_print("Rebasing on %s" % rebase_url)
 721     run_cmd("git describe HEAD", show=True, dir=test_master)
 722     run_cmd("git remote add -t %s %s %s" %
 723             (rebase_branch, rebase_remote, rebase_url),
 724             show=True, dir=test_master)
 725     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 726     if options.fix_whitespace:
 727         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 728                 (rebase_remote, rebase_branch),
 729                 show=True, dir=test_master)
 730     else:
 731         run_cmd("git rebase --force-rebase %s/%s" %
 732                 (rebase_remote, rebase_branch),
 733                 show=True, dir=test_master)
 734     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 735                    (rebase_remote, rebase_branch),
 736                    dir=test_master, output=True)
 737     if diff == '':
 738         do_print("No differences between HEAD and %s/%s - exiting" %
 739                  (rebase_remote, rebase_branch))
 740         sys.exit(0)
 741     run_cmd("git describe %s/%s" %
 742             (rebase_remote, rebase_branch),
 743             show=True, dir=test_master)
 744     run_cmd("git describe HEAD", show=True, dir=test_master)
 745     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 746             (rebase_remote, rebase_branch),
 747             show=True, dir=test_master)
 748
 749
 750 def push_to(push_url, push_branch="master"):
 751     push_remote = "pushto"
 752     do_print("Pushing to %s" % push_url)
 753     if options.mark:
 754         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 755         run_cmd("git commit --amend -c HEAD", dir=test_master)
 756         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 757         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 758     run_cmd("git remote add -t %s %s %s" %
 759             (push_branch, push_remote, push_url),
 760             show=True, dir=test_master)
 761     run_cmd("git push %s +HEAD:%s" %
 762             (push_remote, push_branch),
 763             show=True, dir=test_master)
 764
 765
 766 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 767
 768 gitroot = find_git_root()
 769 if gitroot is None:
 770     raise Exception("Failed to find git root")
 771
 772 parser = OptionParser()
 773 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 774 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 775 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 776 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 777                   default=def_testbase)
 778 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 779 parser.add_option("", "--verbose", help="show all commands as they are run",
 780                   default=False, action="store_true")
 781 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 782                   default=None, type='str')
 783 parser.add_option("", "--pushto", help="push to a git url on success",
 784                   default=None, type='str')
 785 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 786                   default=False, action="store_true")
 787 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 788                   default=False, action="store_true")
 789 parser.add_option("", "--retry", help="automatically retry if master changes",
 790                   default=False, action="store_true")
 791 parser.add_option("", "--email", help="send email to the given address on failure",
 792                   type='str', default=None)
 793 parser.add_option("", "--email-from", help="send email from the given address",
 794                   type='str', default="autobuild@samba.org")
 795 parser.add_option("", "--email-server", help="send email via the given server",
 796                   type='str', default='localhost')
 797 parser.add_option("", "--always-email", help="always send email, even on success",
 798                   action="store_true")
 799 parser.add_option("", "--daemon", help="daemonize after initial setup",
 800                   action="store_true")
 801 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 802                   default="master", type='str')
 803 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 804                   default=gitroot, type='str')
 805 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 806                   default=False, action="store_true")
 807 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 808                   default='')
 809
 810
 811 def send_email(subject, text, log_tar):
 812     if options.email is None:
 813         do_print("not sending email because the recipient is not set")
 814         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 815                  (subject, text))
 816         return
 817     outer = MIMEMultipart()
 818     outer['Subject'] = subject
 819     outer['To'] = options.email
 820     outer['From'] = options.email_from
 821     outer['Date'] = email.utils.formatdate(localtime=True)
 822     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 823     outer.attach(MIMEText(text, 'plain'))
 824     if options.attach_logs:
 825         fp = open(log_tar, 'rb')
 826         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 827         fp.close()
 828         # Set the filename parameter
 829         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 830         outer.attach(msg)
 831     content = outer.as_string()
 832     s = smtplib.SMTP(options.email_server)
 833     s.sendmail(options.email_from, [options.email], content)
 834     s.set_debuglevel(1)
 835     s.quit()
 836
 837
 838 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 839                   elapsed_time, log_base=None, add_log_tail=True):
 840     '''send an email to options.email about the failure'''
 841     elapsed_minutes = elapsed_time / 60.0
 842     if log_base is None:
 843         log_base = gitroot
 844     text = '''
 845 Dear Developer,
 846
 847 Your autobuild on %s failed after %.1f minutes
 848 when trying to test %s with the following error:
 849
 850    %s
 851
 852 the autobuild has been abandoned. Please fix the error and resubmit.
 853
 854 A summary of the autobuild process is here:
 855
 856   %s/autobuild.log
 857 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 858
 859     if options.restrict_tests:
 860         text += """
 861 The build was restricted to tests matching %s\n""" % options.restrict_tests
 862
 863     if failed_task != 'rebase':
 864         text += '''
 865 You can see logs of the failed task here:
 866
 867   %s/%s.stdout
 868   %s/%s.stderr
 869
 870 or you can get full logs of all tasks in this job here:
 871
 872   %s/logs.tar.gz
 873
 874 The top commit for the tree that was built was:
 875
 876 %s
 877
 878 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 879
 880     if add_log_tail:
 881         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 882         lines = f.readlines()
 883         log_tail = "".join(lines[-50:])
 884         num_lines = len(lines)
 885         if num_lines < 50:
 886             # Also include stderr (compile failures) if < 50 lines of stdout
 887             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 888             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 889
 890         text += '''
 891 The last 50 lines of log messages:
 892
 893 %s
 894     ''' % log_tail
 895         f.close()
 896
 897     logs = os.path.join(gitroot, 'logs.tar.gz')
 898     send_email('autobuild[%s] failure on %s for task %s during %s'
 899                % (options.branch, platform.node(), failed_task, failed_stage),
 900                text, logs)
 901
 902
 903 def email_success(elapsed_time, log_base=None):
 904     '''send an email to options.email about a successful build'''
 905     if log_base is None:
 906         log_base = gitroot
 907     text = '''
 908 Dear Developer,
 909
 910 Your autobuild on %s has succeeded after %.1f minutes.
 911
 912 ''' % (platform.node(), elapsed_time / 60.)
 913
 914     if options.restrict_tests:
 915         text += """
 916 The build was restricted to tests matching %s\n""" % options.restrict_tests
 917
 918     if options.keeplogs:
 919         text += '''
 920
 921 you can get full logs of all tasks in this job here:
 922
 923   %s/logs.tar.gz
 924
 925 ''' % log_base
 926
 927     text += '''
 928 The top commit for the tree that was built was:
 929
 930 %s
 931 ''' % top_commit_msg
 932
 933     logs = os.path.join(gitroot, 'logs.tar.gz')
 934     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 935                text, logs)
 936
 937
 938 (options, args) = parser.parse_args()
 939
 940 if options.retry:
 941     if options.rebase is None:
 942         raise Exception('You can only use --retry if you also rebase')
 943
 944 testbase = "%s/b%u" % (options.testbase, os.getpid())
 945 test_master = "%s/master" % testbase
 946 test_prefix = "%s/prefix" % testbase
 947 test_tmpdir = "%s/tmp" % testbase
 948 os.environ['TMPDIR'] = test_tmpdir
 949
 950 # get the top commit message, for emails
 951 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 952 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 953
 954 try:
 955     os.makedirs(testbase)
 956 except Exception as reason:
 957     raise Exception("Unable to create %s : %s" % (testbase, reason))
 958 cleanup_list.append(testbase)
 959
 960 if options.daemon:
 961     logfile = os.path.join(testbase, "log")
 962     do_print("Forking into the background, writing progress to %s" % logfile)
 963     daemonize(logfile)
 964
 965 write_pidfile(gitroot + "/autobuild.pid")
 966
 967 start_time = time.time()
 968
 969 while True:
 970     try:
 971         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 972         os.makedirs(test_tmpdir)
 973         # The waf uninstall code removes empty directories all the way
 974         # up the tree.  Creating a file in test_tmpdir stops it from
 975         # being removed.
 976         run_cmd("touch %s" % os.path.join(test_tmpdir,
 977                                           ".directory-is-not-empty"), show=True)
 978         run_cmd("stat %s" % test_tmpdir, show=True)
 979         run_cmd("stat %s" % testbase, show=True)
 980         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 981     except Exception:
 982         cleanup()
 983         raise
 984
 985     try:
 986         try:
 987             if options.rebase is not None:
 988                 rebase_tree(options.rebase, rebase_branch=options.branch)
 989         except Exception:
 990             cleanup_list.append(gitroot + "/autobuild.pid")
 991             cleanup()
 992             elapsed_time = time.time() - start_time
 993             email_failure(-1, 'rebase', 'rebase', 'rebase',
 994                           'rebase on %s failed' % options.branch,
 995                           elapsed_time, log_base=options.log_base)
 996             sys.exit(1)
 997         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 998         if options.tail:
 999             blist.start_tail()
1000         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1001         if status != 0 or errstr != "retry":
1002             break
1003         cleanup()
1004     except Exception:
1005         cleanup()
1006         raise
1007
1008 cleanup_list.append(gitroot + "/autobuild.pid")
1009
1010 do_print(errstr)
1011
1012 blist.kill_kids()
1013 if options.tail:
1014     do_print("waiting for tail to flush")
1015     time.sleep(1)
1016
1017 elapsed_time = time.time() - start_time
1018 if status == 0:
1019     if options.passcmd is not None:
1020         do_print("Running passcmd: %s" % options.passcmd)
1021         run_cmd(options.passcmd, dir=test_master)
1022     if options.pushto is not None:
1023         push_to(options.pushto, push_branch=options.branch)
1024     if options.keeplogs or options.attach_logs:
1025         blist.tarlogs("logs.tar.gz")
1026         do_print("Logs in logs.tar.gz")
1027     if options.always_email:
1028         email_success(elapsed_time, log_base=options.log_base)
1029     blist.remove_logs()
1030     cleanup()
1031     do_print(errstr)
1032     sys.exit(0)
1033
1034 # something failed, gather a tar of the logs
1035 blist.tarlogs("logs.tar.gz")
1036
1037 if options.email is not None:
1038     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1039                   elapsed_time, log_base=options.log_base)
1040 else:
1041     elapsed_minutes = elapsed_time / 60.0
1042     print('''
1043
1044 ####################################################################
1045
1046 AUTOBUILD FAILURE
1047
1048 Your autobuild[%s] on %s failed after %.1f minutes
1049 when trying to test %s with the following error:
1050
1051    %s
1052
1053 the autobuild has been abandoned. Please fix the error and resubmit.
1054
1055 ####################################################################
1056
1057 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1058
1059 cleanup()
1060 do_print(errstr)
1061 do_print("Logs in logs.tar.gz")
1062 sys.exit(status)