script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-xc" : ".",
  30     "samba-o3" : ".",
  31     "samba-ctdb" : ".",
  32     "samba-libs"  : ".",
  33     "samba-static"  : ".",
  34     "samba-test-only"  : ".",
  35     "samba-none-env"  : ".",
  36     "samba-systemkrb5"  : ".",
  37     "samba-nopython"  : ".",
  38     "ldb"     : "lib/ldb",
  39     "tdb"     : "lib/tdb",
  40     "talloc"  : "lib/talloc",
  41     "replace" : "lib/replace",
  42     "tevent"  : "lib/tevent",
  43     "pidl"    : "pidl",
  44     "pass"    : ".",
  45     "fail"    : ".",
  46     "retry"   : "."
  47     }
  48
  49 defaulttasks = [ "ctdb",
  50                  "samba",
  51                  "samba-xc",
  52                  "samba-o3",
  53                  "samba-ctdb",
  54                  "samba-libs",
  55                  "samba-static",
  56                  "samba-none-env",
  57                  "samba-systemkrb5",
  58                  "samba-nopython",
  59                  "ldb",
  60                  "tdb",
  61                  "talloc",
  62                  "replace",
  63                  "tevent",
  64                  "pidl" ]
  65
  66 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  67     defaulttasks.remove("samba-o3")
  68
  69 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  70 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  71
  72 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  73 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  74 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  75 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  76 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  77 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  78
  79 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  80     extra_python = ""
  81 else:
  82     extra_python = "--extra-python=/usr/bin/python3"
  83
  84 tasks = {
  85     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  86                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  87                ("make", "make all", "text/plain"),
  88                ("install", "make install", "text/plain"),
  89                ("test", "make autotest", "text/plain"),
  90                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  91                ("clean", "make clean", "text/plain") ],
  92
  93     # We have 'test' before 'install' because, 'test' should work without 'install'
  94     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  95                 ("make", "make -j", "text/plain"),
  96                 ("test", "make test FAIL_IMMEDIATELY=1 "
  97                  "TESTS='--exclude-env=none'",
  98                  "text/plain"),
  99                 ("install", "make install", "text/plain"),
 100                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 101                 ("clean", "make clean", "text/plain") ],
 102
 103     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 104                           ("make", "make -j", "text/plain"),
 105                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 106
 107     # Test cross-compile infrastructure
 108     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 109                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 110                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 111                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 112                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 113                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 114
 115     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 116     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 117                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 118                    ("make", "make -j", "text/plain"),
 119                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 120                    ("install", "make install", "text/plain"),
 121                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                    ("clean", "make clean", "text/plain") ],
 123
 124     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 125
 126                      # make sure we have tdb around:
 127                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 128                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 129                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 130
 131
 132                      # build samba with cluster support (also building ctdb):
 133                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 134                      ("samba-make", "make", "text/plain"),
 135                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 136                      ("samba-install", "make install", "text/plain"),
 137                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 138
 139                      # clean up:
 140                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 141                      ("clean", "make clean", "text/plain"),
 142                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 143
 144     "samba-libs" : [
 145                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 146                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 147                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 148                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 149
 150                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 151                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 152                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 153
 154                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 155                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 156                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 157
 158                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 159                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 160                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 161
 162                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 163                       ("nondevel-make", "make -j", "text/plain"),
 164                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 165                       ("nondevel-install", "make install", "text/plain"),
 166                       ("nondevel-dist", "make dist", "text/plain"),
 167
 168                       # retry with all modules shared
 169                       ("allshared-distclean", "make distclean", "text/plain"),
 170                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 171                       ("allshared-make", "make -j", "text/plain")],
 172
 173     "samba-none-env" : [
 174                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 175                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 176                       ("make", "make -j", "text/plain"),
 177                       ("test", "make test "
 178                        "FAIL_IMMEDIATELY=1 "
 179                        "TESTS='--include-env=none'",
 180                        "text/plain")],
 181
 182     "samba-static" : [
 183                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 184                       # build with all modules static
 185                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 186                       ("allstatic-make", "make -j", "text/plain"),
 187
 188                       # retry without any required modules
 189                       ("none-distclean", "make distclean", "text/plain"),
 190                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 191                       ("none-make", "make -j", "text/plain"),
 192
 193                       # retry with nonshared smbd and smbtorture
 194                       ("nonshared-distclean", "make distclean", "text/plain"),
 195                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 196                       ("nonshared-make", "make -j", "text/plain")],
 197
 198     "samba-systemkrb5" : [
 199                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 200                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 201                       ("make", "make -j", "text/plain"),
 202                       # we currently cannot run a full make test, a limited list of tests could be run
 203                       # via "make test TESTS=sometests"
 204                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 205                       ("install", "make install", "text/plain"),
 206                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 207                       ("clean", "make clean", "text/plain")
 208                       ],
 209
 210     # Test Samba without python still builds.  When this test fails
 211     # due to more use of Python, the expectations is that the newly
 212     # failing part of the code should be disabled when
 213     # --disable-python is set (rather than major work being done to
 214     # support this environment).  The target here is for vendors
 215     # shipping a minimal smbd.
 216     "samba-nopython" : [
 217                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 218                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 219                       ("make", "make -j", "text/plain"),
 220                       ("install", "make install", "text/plain"),
 221                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 222                       ("clean", "make clean", "text/plain")
 223                       ],
 224
 225
 226
 227     "ldb" : [
 228               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 229               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 230               ("make", "make", "text/plain"),
 231               ("install", "make install", "text/plain"),
 232               ("test", "make test", "text/plain"),
 233               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 234               ("distcheck", "make distcheck", "text/plain"),
 235               ("clean", "make clean", "text/plain") ],
 236
 237     "tdb" : [
 238               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 239               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 240               ("make", "make", "text/plain"),
 241               ("install", "make install", "text/plain"),
 242               ("test", "make test", "text/plain"),
 243               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 244               ("distcheck", "make distcheck", "text/plain"),
 245               ("clean", "make clean", "text/plain") ],
 246
 247     "talloc" : [
 248                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 249                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 250                  ("make", "make", "text/plain"),
 251                  ("install", "make install", "text/plain"),
 252                  ("test", "make test", "text/plain"),
 253                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 254                  ("distcheck", "make distcheck", "text/plain"),
 255                  ("clean", "make clean", "text/plain") ],
 256
 257     "replace" : [
 258                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 259                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 260                   ("make", "make", "text/plain"),
 261                   ("install", "make install", "text/plain"),
 262                   ("test", "make test", "text/plain"),
 263                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 264                   ("distcheck", "make distcheck", "text/plain"),
 265                   ("clean", "make clean", "text/plain") ],
 266
 267     "tevent" : [
 268                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 269                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 270                  ("make", "make", "text/plain"),
 271                  ("install", "make install", "text/plain"),
 272                  ("test", "make test", "text/plain"),
 273                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 274                  ("distcheck", "make distcheck", "text/plain"),
 275                  ("clean", "make clean", "text/plain") ],
 276
 277     "pidl" : [
 278                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 279                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 280                ("touch", "touch *.yp", "text/plain"),
 281                ("make", "make", "text/plain"),
 282                ("test", "make test", "text/plain"),
 283                ("install", "make install", "text/plain"),
 284                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 285                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 286                ("clean", "make clean", "text/plain") ],
 287
 288     # these are useful for debugging autobuild
 289     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 290     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 291 }
 292
 293 def do_print(msg):
 294     print("%s" % msg)
 295     sys.stdout.flush()
 296     sys.stderr.flush()
 297
 298 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 299     if show is None:
 300         show = options.verbose
 301     if show:
 302         do_print("Running: '%s' in '%s'" % (cmd, dir))
 303     if output:
 304         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 305     elif checkfail:
 306         return check_call(cmd, shell=True, cwd=dir)
 307     else:
 308         return call(cmd, shell=True, cwd=dir)
 309
 310
 311 class builder(object):
 312     '''handle build of one directory'''
 313
 314     def __init__(self, name, sequence, cp=True):
 315         self.name = name
 316         self.dir = builddirs[name]
 317
 318         self.tag = self.name.replace('/', '_')
 319         self.sequence = sequence
 320         self.next = 0
 321         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 322         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 323         if options.verbose:
 324             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 325             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 326         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 327         self.stdout = open(self.stdout_path, 'w')
 328         self.stderr = open(self.stderr_path, 'w')
 329         self.stdin  = open("/dev/null", 'r')
 330         self.sdir = "%s/%s" % (testbase, self.tag)
 331         self.prefix = "%s/%s" % (test_prefix, self.tag)
 332         run_cmd("rm -rf %s" % self.sdir)
 333         run_cmd("rm -rf %s" % self.prefix)
 334         if cp:
 335             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 336         else:
 337             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 338         self.start_next()
 339
 340     def start_next(self):
 341         if self.next == len(self.sequence):
 342             if not options.nocleanup:
 343                 run_cmd("rm -rf %s" % self.sdir)
 344                 run_cmd("rm -rf %s" % self.prefix)
 345             do_print('%s: Completed OK' % self.name)
 346             self.done = True
 347             return
 348         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 349         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 350         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 351         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 352         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 353         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 354 #        if self.output_mime_type == "text/x-subunit":
 355 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 356         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 357         cwd = os.getcwd()
 358         os.chdir("%s/%s" % (self.sdir, self.dir))
 359         self.proc = Popen(self.cmd, shell=True,
 360                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 361         os.chdir(cwd)
 362         self.next += 1
 363
 364
 365 class buildlist(object):
 366     '''handle build of multiple directories'''
 367
 368     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 369         global tasks
 370         self.tlist = []
 371         self.tail_proc = None
 372         self.retry = None
 373         if tasknames == []:
 374             if options.restrict_tests:
 375                 tasknames = ["samba-test-only"]
 376             else:
 377                 tasknames = defaulttasks
 378         else:
 379             # If we are only running one test,
 380             # do not sleep randomly to wait for it to start
 381             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 382
 383         for n in tasknames:
 384             b = builder(n, tasks[n], cp=n is not "pidl")
 385             self.tlist.append(b)
 386         if options.retry:
 387             rebase_remote = "rebaseon"
 388             retry_task = [ ("retry",
 389                             '''set -e
 390                             git remote add -t %s %s %s
 391                             git fetch %s
 392                             while :; do
 393                               sleep 60
 394                               git describe %s/%s > old_remote_branch.desc
 395                               git fetch %s
 396                               git describe %s/%s > remote_branch.desc
 397                               diff old_remote_branch.desc remote_branch.desc
 398                             done
 399                            ''' % (
 400                                rebase_branch, rebase_remote, rebase_url,
 401                                rebase_remote,
 402                                rebase_remote, rebase_branch,
 403                                rebase_remote,
 404                                rebase_remote, rebase_branch
 405                            ),
 406                            "test/plain" ) ]
 407
 408             self.retry = builder('retry', retry_task, cp=False)
 409             self.need_retry = False
 410
 411     def kill_kids(self):
 412         if self.tail_proc is not None:
 413             self.tail_proc.terminate()
 414             self.tail_proc.wait()
 415             self.tail_proc = None
 416         if self.retry is not None:
 417             self.retry.proc.terminate()
 418             self.retry.proc.wait()
 419             self.retry = None
 420         for b in self.tlist:
 421             if b.proc is not None:
 422                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 423                 b.proc.terminate()
 424                 b.proc.wait()
 425                 b.proc = None
 426
 427     def wait_one(self):
 428         while True:
 429             none_running = True
 430             for b in self.tlist:
 431                 if b.proc is None:
 432                     continue
 433                 none_running = False
 434                 b.status = b.proc.poll()
 435                 if b.status is None:
 436                     continue
 437                 b.proc = None
 438                 return b
 439             if options.retry:
 440                 ret = self.retry.proc.poll()
 441                 if ret is not None:
 442                     self.need_retry = True
 443                     self.retry = None
 444                     return None
 445             if none_running:
 446                 return None
 447             time.sleep(0.1)
 448
 449     def run(self):
 450         while True:
 451             b = self.wait_one()
 452             if options.retry and self.need_retry:
 453                 self.kill_kids()
 454                 do_print("retry needed")
 455                 return (0, None, None, None, "retry")
 456             if b is None:
 457                 break
 458             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 459                 self.kill_kids()
 460                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 461             b.start_next()
 462         self.kill_kids()
 463         return (0, None, None, None, "All OK")
 464
 465     def write_system_info(self):
 466         filename = 'system-info.txt'
 467         f = open(filename, 'w')
 468         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
 469             print('### %s' % cmd, file=f)
 470             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 471             print(file=f)
 472         f.close()
 473         return filename
 474
 475     def tarlogs(self, fname):
 476         tar = tarfile.open(fname, "w:gz")
 477         for b in self.tlist:
 478             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 479             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 480         if os.path.exists("autobuild.log"):
 481             tar.add("autobuild.log")
 482         sys_info = self.write_system_info()
 483         tar.add(sys_info)
 484         tar.close()
 485
 486     def remove_logs(self):
 487         for b in self.tlist:
 488             os.unlink(b.stdout_path)
 489             os.unlink(b.stderr_path)
 490
 491     def start_tail(self):
 492         cwd = os.getcwd()
 493         cmd = "tail -f *.stdout *.stderr"
 494         os.chdir(gitroot)
 495         self.tail_proc = Popen(cmd, shell=True)
 496         os.chdir(cwd)
 497
 498
 499 def cleanup():
 500     if options.nocleanup:
 501         return
 502     run_cmd("stat %s || true" % test_tmpdir, show=True)
 503     run_cmd("stat %s" % testbase, show=True)
 504     do_print("Cleaning up ....")
 505     for d in cleanup_list:
 506         run_cmd("rm -rf %s" % d)
 507
 508
 509 def find_git_root():
 510     '''get to the top of the git repo'''
 511     p=os.getcwd()
 512     while p != '/':
 513         if os.path.isdir(os.path.join(p, ".git")):
 514             return p
 515         p = os.path.abspath(os.path.join(p, '..'))
 516     return None
 517
 518
 519 def daemonize(logfile):
 520     pid = os.fork()
 521     if pid == 0: # Parent
 522         os.setsid()
 523         pid = os.fork()
 524         if pid != 0: # Actual daemon
 525             os._exit(0)
 526     else: # Grandparent
 527         os._exit(0)
 528
 529     import resource      # Resource usage information.
 530     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 531     if maxfd == resource.RLIM_INFINITY:
 532         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 533     for fd in range(0, maxfd):
 534         try:
 535             os.close(fd)
 536         except OSError:
 537             pass
 538     os.open(logfile, os.O_RDWR | os.O_CREAT)
 539     os.dup2(0, 1)
 540     os.dup2(0, 2)
 541
 542 def write_pidfile(fname):
 543     '''write a pid file, cleanup on exit'''
 544     f = open(fname, mode='w')
 545     f.write("%u\n" % os.getpid())
 546     f.close()
 547
 548
 549 def rebase_tree(rebase_url, rebase_branch = "master"):
 550     rebase_remote = "rebaseon"
 551     do_print("Rebasing on %s" % rebase_url)
 552     run_cmd("git describe HEAD", show=True, dir=test_master)
 553     run_cmd("git remote add -t %s %s %s" %
 554             (rebase_branch, rebase_remote, rebase_url),
 555             show=True, dir=test_master)
 556     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 557     if options.fix_whitespace:
 558         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 559                 (rebase_remote, rebase_branch),
 560                 show=True, dir=test_master)
 561     else:
 562         run_cmd("git rebase --force-rebase %s/%s" %
 563                 (rebase_remote, rebase_branch),
 564                 show=True, dir=test_master)
 565     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 566                    (rebase_remote, rebase_branch),
 567                    dir=test_master, output=True)
 568     if diff == '':
 569         do_print("No differences between HEAD and %s/%s - exiting" %
 570               (rebase_remote, rebase_branch))
 571         sys.exit(0)
 572     run_cmd("git describe %s/%s" %
 573             (rebase_remote, rebase_branch),
 574             show=True, dir=test_master)
 575     run_cmd("git describe HEAD", show=True, dir=test_master)
 576     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 577             (rebase_remote, rebase_branch),
 578             show=True, dir=test_master)
 579
 580 def push_to(push_url, push_branch = "master"):
 581     push_remote = "pushto"
 582     do_print("Pushing to %s" % push_url)
 583     if options.mark:
 584         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 585         run_cmd("git commit --amend -c HEAD", dir=test_master)
 586         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 587         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 588     run_cmd("git remote add -t %s %s %s" %
 589             (push_branch, push_remote, push_url),
 590             show=True, dir=test_master)
 591     run_cmd("git push %s +HEAD:%s" %
 592             (push_remote, push_branch),
 593             show=True, dir=test_master)
 594
 595 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 596
 597 gitroot = find_git_root()
 598 if gitroot is None:
 599     raise Exception("Failed to find git root")
 600
 601 parser = OptionParser()
 602 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 603 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 604 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 605 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 606                   default=def_testbase)
 607 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 608 parser.add_option("", "--verbose", help="show all commands as they are run",
 609                   default=False, action="store_true")
 610 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 611                   default=None, type='str')
 612 parser.add_option("", "--pushto", help="push to a git url on success",
 613                   default=None, type='str')
 614 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 615                   default=False, action="store_true")
 616 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 617                   default=False, action="store_true")
 618 parser.add_option("", "--retry", help="automatically retry if master changes",
 619                   default=False, action="store_true")
 620 parser.add_option("", "--email", help="send email to the given address on failure",
 621                   type='str', default=None)
 622 parser.add_option("", "--email-from", help="send email from the given address",
 623                   type='str', default="autobuild@samba.org")
 624 parser.add_option("", "--email-server", help="send email via the given server",
 625                   type='str', default='localhost')
 626 parser.add_option("", "--always-email", help="always send email, even on success",
 627                   action="store_true")
 628 parser.add_option("", "--daemon", help="daemonize after initial setup",
 629                   action="store_true")
 630 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 631                   default="master", type='str')
 632 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 633                   default=gitroot, type='str')
 634 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 635                   default=False, action="store_true")
 636 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 637                   default='')
 638
 639 def send_email(subject, text, log_tar):
 640     outer = MIMEMultipart()
 641     outer['Subject'] = subject
 642     outer['To'] = options.email
 643     outer['From'] = options.email_from
 644     outer['Date'] = email.utils.formatdate(localtime = True)
 645     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 646     outer.attach(MIMEText(text, 'plain'))
 647     if options.attach_logs:
 648         fp = open(log_tar, 'rb')
 649         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 650         fp.close()
 651         # Set the filename parameter
 652         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 653         outer.attach(msg)
 654     content = outer.as_string()
 655     s = smtplib.SMTP(options.email_server)
 656     s.sendmail(options.email_from, [options.email], content)
 657     s.set_debuglevel(1)
 658     s.quit()
 659
 660 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 661                   elapsed_time, log_base=None, add_log_tail=True):
 662     '''send an email to options.email about the failure'''
 663     elapsed_minutes = elapsed_time / 60.0
 664     user = os.getenv("USER")
 665     if log_base is None:
 666         log_base = gitroot
 667     text = '''
 668 Dear Developer,
 669
 670 Your autobuild on %s failed after %.1f minutes
 671 when trying to test %s with the following error:
 672
 673    %s
 674
 675 the autobuild has been abandoned. Please fix the error and resubmit.
 676
 677 A summary of the autobuild process is here:
 678
 679   %s/autobuild.log
 680 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 681
 682     if options.restrict_tests:
 683         text += """
 684 The build was restricted to tests matching %s\n""" % options.restrict_tests
 685
 686     if failed_task != 'rebase':
 687         text += '''
 688 You can see logs of the failed task here:
 689
 690   %s/%s.stdout
 691   %s/%s.stderr
 692
 693 or you can get full logs of all tasks in this job here:
 694
 695   %s/logs.tar.gz
 696
 697 The top commit for the tree that was built was:
 698
 699 %s
 700
 701 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 702
 703     if add_log_tail:
 704         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 705         lines = f.readlines()
 706         log_tail = "".join(lines[-50:])
 707         num_lines = len(lines)
 708         if num_lines < 50:
 709             # Also include stderr (compile failures) if < 50 lines of stdout
 710             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 711             log_tail += "".join(f.readlines()[-(50-num_lines):])
 712
 713         text += '''
 714 The last 50 lines of log messages:
 715
 716 %s
 717     ''' % log_tail
 718         f.close()
 719
 720     logs = os.path.join(gitroot, 'logs.tar.gz')
 721     send_email('autobuild[%s] failure on %s for task %s during %s'
 722                % (options.branch, platform.node(), failed_task, failed_stage),
 723                text, logs)
 724
 725 def email_success(elapsed_time, log_base=None):
 726     '''send an email to options.email about a successful build'''
 727     user = os.getenv("USER")
 728     if log_base is None:
 729         log_base = gitroot
 730     text = '''
 731 Dear Developer,
 732
 733 Your autobuild on %s has succeeded after %.1f minutes.
 734
 735 ''' % (platform.node(), elapsed_time / 60.)
 736
 737     if options.restrict_tests:
 738         text += """
 739 The build was restricted to tests matching %s\n""" % options.restrict_tests
 740
 741     if options.keeplogs:
 742         text += '''
 743
 744 you can get full logs of all tasks in this job here:
 745
 746   %s/logs.tar.gz
 747
 748 ''' % log_base
 749
 750     text += '''
 751 The top commit for the tree that was built was:
 752
 753 %s
 754 ''' % top_commit_msg
 755
 756     logs = os.path.join(gitroot, 'logs.tar.gz')
 757     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 758                text, logs)
 759
 760
 761 (options, args) = parser.parse_args()
 762
 763 if options.retry:
 764     if options.rebase is None:
 765         raise Exception('You can only use --retry if you also rebase')
 766
 767 testbase = "%s/b%u" % (options.testbase, os.getpid())
 768 test_master = "%s/master" % testbase
 769 test_prefix = "%s/prefix" % testbase
 770 test_tmpdir = "%s/tmp" % testbase
 771 os.environ['TMPDIR'] = test_tmpdir
 772
 773 # get the top commit message, for emails
 774 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 775
 776 try:
 777     os.makedirs(testbase)
 778 except Exception as reason:
 779     raise Exception("Unable to create %s : %s" % (testbase, reason))
 780 cleanup_list.append(testbase)
 781
 782 if options.daemon:
 783     logfile = os.path.join(testbase, "log")
 784     do_print("Forking into the background, writing progress to %s" % logfile)
 785     daemonize(logfile)
 786
 787 write_pidfile(gitroot + "/autobuild.pid")
 788
 789 start_time = time.time()
 790
 791 while True:
 792     try:
 793         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 794         os.makedirs(test_tmpdir)
 795         # The waf uninstall code removes empty directories all the way
 796         # up the tree.  Creating a file in test_tmpdir stops it from
 797         # being removed.
 798         run_cmd("touch %s" % os.path.join(test_tmpdir,
 799                                           ".directory-is-not-empty"), show=True)
 800         run_cmd("stat %s" % test_tmpdir, show=True)
 801         run_cmd("stat %s" % testbase, show=True)
 802         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 803     except Exception:
 804         cleanup()
 805         raise
 806
 807     try:
 808         try:
 809             if options.rebase is not None:
 810                 rebase_tree(options.rebase, rebase_branch=options.branch)
 811         except Exception:
 812             cleanup_list.append(gitroot + "/autobuild.pid")
 813             cleanup()
 814             elapsed_time = time.time() - start_time
 815             email_failure(-1, 'rebase', 'rebase', 'rebase',
 816                           'rebase on %s failed' % options.branch,
 817                           elapsed_time, log_base=options.log_base)
 818             sys.exit(1)
 819         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 820         if options.tail:
 821             blist.start_tail()
 822         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 823         if status != 0 or errstr != "retry":
 824             break
 825         cleanup()
 826     except Exception:
 827         cleanup()
 828         raise
 829
 830 cleanup_list.append(gitroot + "/autobuild.pid")
 831
 832 do_print(errstr)
 833
 834 blist.kill_kids()
 835 if options.tail:
 836     do_print("waiting for tail to flush")
 837     time.sleep(1)
 838
 839 elapsed_time = time.time() - start_time
 840 if status == 0:
 841     if options.passcmd is not None:
 842         do_print("Running passcmd: %s" % options.passcmd)
 843         run_cmd(options.passcmd, dir=test_master)
 844     if options.pushto is not None:
 845         push_to(options.pushto, push_branch=options.branch)
 846     if options.keeplogs or options.attach_logs:
 847         blist.tarlogs("logs.tar.gz")
 848         do_print("Logs in logs.tar.gz")
 849     if options.always_email:
 850         email_success(elapsed_time, log_base=options.log_base)
 851     blist.remove_logs()
 852     cleanup()
 853     do_print(errstr)
 854     sys.exit(0)
 855
 856 # something failed, gather a tar of the logs
 857 blist.tarlogs("logs.tar.gz")
 858
 859 if options.email is not None:
 860     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 861                   elapsed_time, log_base=options.log_base)
 862 else:
 863     elapsed_minutes = elapsed_time / 60.0
 864     print('''
 865
 866 ####################################################################
 867
 868 AUTOBUILD FAILURE
 869
 870 Your autobuild[%s] on %s failed after %.1f minutes
 871 when trying to test %s with the following error:
 872
 873    %s
 874
 875 the autobuild has been abandoned. Please fix the error and resubmit.
 876
 877 ####################################################################
 878
 879 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 880
 881 cleanup()
 882 do_print(errstr)
 883 do_print("Logs in logs.tar.gz")
 884 sys.exit(status)