script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-systemkrb5"  : ".",
  40     "samba-nopython"  : ".",
  41     "ldb"     : "lib/ldb",
  42     "tdb"     : "lib/tdb",
  43     "talloc"  : "lib/talloc",
  44     "replace" : "lib/replace",
  45     "tevent"  : "lib/tevent",
  46     "pidl"    : "pidl",
  47     "pass"    : ".",
  48     "fail"    : ".",
  49     "retry"   : "."
  50     }
  51
  52 defaulttasks = [ "ctdb",
  53                  "samba",
  54                  "samba-nt4",
  55                  "samba-fileserver",
  56                  "samba-xc",
  57                  "samba-o3",
  58                  "samba-ctdb",
  59                  "samba-libs",
  60                  "samba-static",
  61                  "samba-none-env",
  62                  "samba-ad-dc",
  63                  "samba-systemkrb5",
  64                  "samba-nopython",
  65                  "ldb",
  66                  "tdb",
  67                  "talloc",
  68                  "replace",
  69                  "tevent",
  70                  "pidl" ]
  71
  72 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  73     defaulttasks.remove("samba-o3")
  74
  75 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  76 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  77
  78 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  79 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  80 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  81 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  82 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  83 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  84
  85 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  86     extra_python = ""
  87 else:
  88     extra_python = "--extra-python=/usr/bin/python3"
  89
  90 tasks = {
  91     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  92                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  93                ("make", "make all", "text/plain"),
  94                ("install", "make install", "text/plain"),
  95                ("test", "make autotest", "text/plain"),
  96                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  97                ("clean", "make clean", "text/plain") ],
  98
  99     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 100     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 101                 ("make", "make -j", "text/plain"),
 102                 ("test", "make test FAIL_IMMEDIATELY=1 "
 103                  "TESTS='--exclude-env=none "
 104                  "--exclude-env=nt4_dc "
 105                  "--exclude-env=nt4_member "
 106                  "--exclude-env=ad_dc "
 107                  "--exclude-env=fileserver'",
 108                  "text/plain"),
 109                 ("install", "make install", "text/plain"),
 110                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 111                 ("clean", "make clean", "text/plain") ],
 112
 113     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 114     "samba-nt4" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 115                        ("make", "make -j", "text/plain"),
 116                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 117                        ("install", "make install", "text/plain"),
 118                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 119                        ("clean", "make clean", "text/plain") ],
 120
 121     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 122     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 123                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 124                       ("make", "make -j", "text/plain"),
 125                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 126                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 127
 128     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 129     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 130                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 131                       ("make", "make -j", "text/plain"),
 132                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 133                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 134
 135     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 136                           ("make", "make -j", "text/plain"),
 137                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 138
 139     # Test cross-compile infrastructure
 140     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 142                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 143                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 144                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 145                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 146
 147     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 148     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 149                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 150                    ("make", "make -j", "text/plain"),
 151                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 152                    ("install", "make install", "text/plain"),
 153                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 154                    ("clean", "make clean", "text/plain") ],
 155
 156     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 157
 158                      # make sure we have tdb around:
 159                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 160                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 161                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 162
 163
 164                      # build samba with cluster support (also building ctdb):
 165                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 166                      ("samba-make", "make", "text/plain"),
 167                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 168                      ("samba-install", "make install", "text/plain"),
 169                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 170
 171                      # clean up:
 172                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 173                      ("clean", "make clean", "text/plain"),
 174                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 175
 176     "samba-libs" : [
 177                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 178                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 179                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 180                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 181
 182                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 183                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 184                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 185
 186                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 187                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 188                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 189
 190                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 191                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 192                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 193
 194                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 195                       ("nondevel-make", "make -j", "text/plain"),
 196                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 197                       ("nondevel-install", "make install", "text/plain"),
 198                       ("nondevel-dist", "make dist", "text/plain"),
 199
 200                       # retry with all modules shared
 201                       ("allshared-distclean", "make distclean", "text/plain"),
 202                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 203                       ("allshared-make", "make -j", "text/plain")],
 204
 205     "samba-none-env" : [
 206                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 207                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 208                       ("make", "make -j", "text/plain"),
 209                       ("test", "make test "
 210                        "FAIL_IMMEDIATELY=1 "
 211                        "TESTS='--include-env=none'",
 212                        "text/plain")],
 213
 214     "samba-static" : [
 215                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 216                       # build with all modules static
 217                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 218                       ("allstatic-make", "make -j", "text/plain"),
 219
 220                       # retry without any required modules
 221                       ("none-distclean", "make distclean", "text/plain"),
 222                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 223                       ("none-make", "make -j", "text/plain"),
 224
 225                       # retry with nonshared smbd and smbtorture
 226                       ("nonshared-distclean", "make distclean", "text/plain"),
 227                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 228                       ("nonshared-make", "make -j", "text/plain")],
 229
 230     "samba-systemkrb5" : [
 231                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 232                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 233                       ("make", "make -j", "text/plain"),
 234                       # we currently cannot run a full make test, a limited list of tests could be run
 235                       # via "make test TESTS=sometests"
 236                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 237                       ("install", "make install", "text/plain"),
 238                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 239                       ("clean", "make clean", "text/plain")
 240                       ],
 241
 242     # Test Samba without python still builds.  When this test fails
 243     # due to more use of Python, the expectations is that the newly
 244     # failing part of the code should be disabled when
 245     # --disable-python is set (rather than major work being done to
 246     # support this environment).  The target here is for vendors
 247     # shipping a minimal smbd.
 248     "samba-nopython" : [
 249                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 250                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 251                       ("make", "make -j", "text/plain"),
 252                       ("install", "make install", "text/plain"),
 253                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 254                       ("clean", "make clean", "text/plain")
 255                       ],
 256
 257
 258
 259     "ldb" : [
 260               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 261               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 262               ("make", "make", "text/plain"),
 263               ("install", "make install", "text/plain"),
 264               ("test", "make test", "text/plain"),
 265               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 266               ("distcheck", "make distcheck", "text/plain"),
 267               ("clean", "make clean", "text/plain") ],
 268
 269     "tdb" : [
 270               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 271               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 272               ("make", "make", "text/plain"),
 273               ("install", "make install", "text/plain"),
 274               ("test", "make test", "text/plain"),
 275               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 276               ("distcheck", "make distcheck", "text/plain"),
 277               ("clean", "make clean", "text/plain") ],
 278
 279     "talloc" : [
 280                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 281                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 282                  ("make", "make", "text/plain"),
 283                  ("install", "make install", "text/plain"),
 284                  ("test", "make test", "text/plain"),
 285                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 286                  ("distcheck", "make distcheck", "text/plain"),
 287                  ("clean", "make clean", "text/plain") ],
 288
 289     "replace" : [
 290                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 291                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 292                   ("make", "make", "text/plain"),
 293                   ("install", "make install", "text/plain"),
 294                   ("test", "make test", "text/plain"),
 295                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 296                   ("distcheck", "make distcheck", "text/plain"),
 297                   ("clean", "make clean", "text/plain") ],
 298
 299     "tevent" : [
 300                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 301                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 302                  ("make", "make", "text/plain"),
 303                  ("install", "make install", "text/plain"),
 304                  ("test", "make test", "text/plain"),
 305                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 306                  ("distcheck", "make distcheck", "text/plain"),
 307                  ("clean", "make clean", "text/plain") ],
 308
 309     "pidl" : [
 310                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 311                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 312                ("touch", "touch *.yp", "text/plain"),
 313                ("make", "make", "text/plain"),
 314                ("test", "make test", "text/plain"),
 315                ("install", "make install", "text/plain"),
 316                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 317                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 318                ("clean", "make clean", "text/plain") ],
 319
 320     # these are useful for debugging autobuild
 321     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 322     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 323 }
 324
 325 def do_print(msg):
 326     print("%s" % msg)
 327     sys.stdout.flush()
 328     sys.stderr.flush()
 329
 330 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 331     if show is None:
 332         show = options.verbose
 333     if show:
 334         do_print("Running: '%s' in '%s'" % (cmd, dir))
 335     if output:
 336         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 337     elif checkfail:
 338         return check_call(cmd, shell=True, cwd=dir)
 339     else:
 340         return call(cmd, shell=True, cwd=dir)
 341
 342
 343 class builder(object):
 344     '''handle build of one directory'''
 345
 346     def __init__(self, name, sequence, cp=True):
 347         self.name = name
 348         self.dir = builddirs[name]
 349
 350         self.tag = self.name.replace('/', '_')
 351         self.sequence = sequence
 352         self.next = 0
 353         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 354         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 355         if options.verbose:
 356             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 357             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 358         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 359         self.stdout = open(self.stdout_path, 'w')
 360         self.stderr = open(self.stderr_path, 'w')
 361         self.stdin  = open("/dev/null", 'r')
 362         self.sdir = "%s/%s" % (testbase, self.tag)
 363         self.prefix = "%s/%s" % (test_prefix, self.tag)
 364         run_cmd("rm -rf %s" % self.sdir)
 365         run_cmd("rm -rf %s" % self.prefix)
 366         if cp:
 367             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 368         else:
 369             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 370         self.start_next()
 371
 372     def start_next(self):
 373         if self.next == len(self.sequence):
 374             if not options.nocleanup:
 375                 run_cmd("rm -rf %s" % self.sdir)
 376                 run_cmd("rm -rf %s" % self.prefix)
 377             do_print('%s: Completed OK' % self.name)
 378             self.done = True
 379             return
 380         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 381         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 382         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 383         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 384         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 385         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 386 #        if self.output_mime_type == "text/x-subunit":
 387 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 388         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 389         cwd = os.getcwd()
 390         os.chdir("%s/%s" % (self.sdir, self.dir))
 391         self.proc = Popen(self.cmd, shell=True,
 392                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 393         os.chdir(cwd)
 394         self.next += 1
 395
 396
 397 class buildlist(object):
 398     '''handle build of multiple directories'''
 399
 400     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 401         global tasks
 402         self.tlist = []
 403         self.tail_proc = None
 404         self.retry = None
 405         if tasknames == []:
 406             if options.restrict_tests:
 407                 tasknames = ["samba-test-only"]
 408             else:
 409                 tasknames = defaulttasks
 410         else:
 411             # If we are only running one test,
 412             # do not sleep randomly to wait for it to start
 413             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 414
 415         for n in tasknames:
 416             b = builder(n, tasks[n], cp=n is not "pidl")
 417             self.tlist.append(b)
 418         if options.retry:
 419             rebase_remote = "rebaseon"
 420             retry_task = [ ("retry",
 421                             '''set -e
 422                             git remote add -t %s %s %s
 423                             git fetch %s
 424                             while :; do
 425                               sleep 60
 426                               git describe %s/%s > old_remote_branch.desc
 427                               git fetch %s
 428                               git describe %s/%s > remote_branch.desc
 429                               diff old_remote_branch.desc remote_branch.desc
 430                             done
 431                            ''' % (
 432                                rebase_branch, rebase_remote, rebase_url,
 433                                rebase_remote,
 434                                rebase_remote, rebase_branch,
 435                                rebase_remote,
 436                                rebase_remote, rebase_branch
 437                            ),
 438                            "test/plain" ) ]
 439
 440             self.retry = builder('retry', retry_task, cp=False)
 441             self.need_retry = False
 442
 443     def kill_kids(self):
 444         if self.tail_proc is not None:
 445             self.tail_proc.terminate()
 446             self.tail_proc.wait()
 447             self.tail_proc = None
 448         if self.retry is not None:
 449             self.retry.proc.terminate()
 450             self.retry.proc.wait()
 451             self.retry = None
 452         for b in self.tlist:
 453             if b.proc is not None:
 454                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 455                 b.proc.terminate()
 456                 b.proc.wait()
 457                 b.proc = None
 458
 459     def wait_one(self):
 460         while True:
 461             none_running = True
 462             for b in self.tlist:
 463                 if b.proc is None:
 464                     continue
 465                 none_running = False
 466                 b.status = b.proc.poll()
 467                 if b.status is None:
 468                     continue
 469                 b.proc = None
 470                 return b
 471             if options.retry:
 472                 ret = self.retry.proc.poll()
 473                 if ret is not None:
 474                     self.need_retry = True
 475                     self.retry = None
 476                     return None
 477             if none_running:
 478                 return None
 479             time.sleep(0.1)
 480
 481     def run(self):
 482         while True:
 483             b = self.wait_one()
 484             if options.retry and self.need_retry:
 485                 self.kill_kids()
 486                 do_print("retry needed")
 487                 return (0, None, None, None, "retry")
 488             if b is None:
 489                 break
 490             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 491                 self.kill_kids()
 492                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 493             b.start_next()
 494         self.kill_kids()
 495         return (0, None, None, None, "All OK")
 496
 497     def write_system_info(self):
 498         filename = 'system-info.txt'
 499         f = open(filename, 'w')
 500         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
 501             print('### %s' % cmd, file=f)
 502             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 503             print(file=f)
 504         f.close()
 505         return filename
 506
 507     def tarlogs(self, fname):
 508         tar = tarfile.open(fname, "w:gz")
 509         for b in self.tlist:
 510             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 511             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 512         if os.path.exists("autobuild.log"):
 513             tar.add("autobuild.log")
 514         sys_info = self.write_system_info()
 515         tar.add(sys_info)
 516         tar.close()
 517
 518     def remove_logs(self):
 519         for b in self.tlist:
 520             os.unlink(b.stdout_path)
 521             os.unlink(b.stderr_path)
 522
 523     def start_tail(self):
 524         cwd = os.getcwd()
 525         cmd = "tail -f *.stdout *.stderr"
 526         os.chdir(gitroot)
 527         self.tail_proc = Popen(cmd, shell=True)
 528         os.chdir(cwd)
 529
 530
 531 def cleanup():
 532     if options.nocleanup:
 533         return
 534     run_cmd("stat %s || true" % test_tmpdir, show=True)
 535     run_cmd("stat %s" % testbase, show=True)
 536     do_print("Cleaning up ....")
 537     for d in cleanup_list:
 538         run_cmd("rm -rf %s" % d)
 539
 540
 541 def find_git_root():
 542     '''get to the top of the git repo'''
 543     p=os.getcwd()
 544     while p != '/':
 545         if os.path.isdir(os.path.join(p, ".git")):
 546             return p
 547         p = os.path.abspath(os.path.join(p, '..'))
 548     return None
 549
 550
 551 def daemonize(logfile):
 552     pid = os.fork()
 553     if pid == 0: # Parent
 554         os.setsid()
 555         pid = os.fork()
 556         if pid != 0: # Actual daemon
 557             os._exit(0)
 558     else: # Grandparent
 559         os._exit(0)
 560
 561     import resource      # Resource usage information.
 562     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 563     if maxfd == resource.RLIM_INFINITY:
 564         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 565     for fd in range(0, maxfd):
 566         try:
 567             os.close(fd)
 568         except OSError:
 569             pass
 570     os.open(logfile, os.O_RDWR | os.O_CREAT)
 571     os.dup2(0, 1)
 572     os.dup2(0, 2)
 573
 574 def write_pidfile(fname):
 575     '''write a pid file, cleanup on exit'''
 576     f = open(fname, mode='w')
 577     f.write("%u\n" % os.getpid())
 578     f.close()
 579
 580
 581 def rebase_tree(rebase_url, rebase_branch = "master"):
 582     rebase_remote = "rebaseon"
 583     do_print("Rebasing on %s" % rebase_url)
 584     run_cmd("git describe HEAD", show=True, dir=test_master)
 585     run_cmd("git remote add -t %s %s %s" %
 586             (rebase_branch, rebase_remote, rebase_url),
 587             show=True, dir=test_master)
 588     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 589     if options.fix_whitespace:
 590         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 591                 (rebase_remote, rebase_branch),
 592                 show=True, dir=test_master)
 593     else:
 594         run_cmd("git rebase --force-rebase %s/%s" %
 595                 (rebase_remote, rebase_branch),
 596                 show=True, dir=test_master)
 597     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 598                    (rebase_remote, rebase_branch),
 599                    dir=test_master, output=True)
 600     if diff == '':
 601         do_print("No differences between HEAD and %s/%s - exiting" %
 602               (rebase_remote, rebase_branch))
 603         sys.exit(0)
 604     run_cmd("git describe %s/%s" %
 605             (rebase_remote, rebase_branch),
 606             show=True, dir=test_master)
 607     run_cmd("git describe HEAD", show=True, dir=test_master)
 608     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 609             (rebase_remote, rebase_branch),
 610             show=True, dir=test_master)
 611
 612 def push_to(push_url, push_branch = "master"):
 613     push_remote = "pushto"
 614     do_print("Pushing to %s" % push_url)
 615     if options.mark:
 616         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 617         run_cmd("git commit --amend -c HEAD", dir=test_master)
 618         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 619         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 620     run_cmd("git remote add -t %s %s %s" %
 621             (push_branch, push_remote, push_url),
 622             show=True, dir=test_master)
 623     run_cmd("git push %s +HEAD:%s" %
 624             (push_remote, push_branch),
 625             show=True, dir=test_master)
 626
 627 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 628
 629 gitroot = find_git_root()
 630 if gitroot is None:
 631     raise Exception("Failed to find git root")
 632
 633 parser = OptionParser()
 634 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 635 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 636 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 637 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 638                   default=def_testbase)
 639 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 640 parser.add_option("", "--verbose", help="show all commands as they are run",
 641                   default=False, action="store_true")
 642 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 643                   default=None, type='str')
 644 parser.add_option("", "--pushto", help="push to a git url on success",
 645                   default=None, type='str')
 646 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 647                   default=False, action="store_true")
 648 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 649                   default=False, action="store_true")
 650 parser.add_option("", "--retry", help="automatically retry if master changes",
 651                   default=False, action="store_true")
 652 parser.add_option("", "--email", help="send email to the given address on failure",
 653                   type='str', default=None)
 654 parser.add_option("", "--email-from", help="send email from the given address",
 655                   type='str', default="autobuild@samba.org")
 656 parser.add_option("", "--email-server", help="send email via the given server",
 657                   type='str', default='localhost')
 658 parser.add_option("", "--always-email", help="always send email, even on success",
 659                   action="store_true")
 660 parser.add_option("", "--daemon", help="daemonize after initial setup",
 661                   action="store_true")
 662 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 663                   default="master", type='str')
 664 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 665                   default=gitroot, type='str')
 666 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 667                   default=False, action="store_true")
 668 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 669                   default='')
 670
 671 def send_email(subject, text, log_tar):
 672     outer = MIMEMultipart()
 673     outer['Subject'] = subject
 674     outer['To'] = options.email
 675     outer['From'] = options.email_from
 676     outer['Date'] = email.utils.formatdate(localtime = True)
 677     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 678     outer.attach(MIMEText(text, 'plain'))
 679     if options.attach_logs:
 680         fp = open(log_tar, 'rb')
 681         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 682         fp.close()
 683         # Set the filename parameter
 684         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 685         outer.attach(msg)
 686     content = outer.as_string()
 687     s = smtplib.SMTP(options.email_server)
 688     s.sendmail(options.email_from, [options.email], content)
 689     s.set_debuglevel(1)
 690     s.quit()
 691
 692 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 693                   elapsed_time, log_base=None, add_log_tail=True):
 694     '''send an email to options.email about the failure'''
 695     elapsed_minutes = elapsed_time / 60.0
 696     user = os.getenv("USER")
 697     if log_base is None:
 698         log_base = gitroot
 699     text = '''
 700 Dear Developer,
 701
 702 Your autobuild on %s failed after %.1f minutes
 703 when trying to test %s with the following error:
 704
 705    %s
 706
 707 the autobuild has been abandoned. Please fix the error and resubmit.
 708
 709 A summary of the autobuild process is here:
 710
 711   %s/autobuild.log
 712 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 713
 714     if options.restrict_tests:
 715         text += """
 716 The build was restricted to tests matching %s\n""" % options.restrict_tests
 717
 718     if failed_task != 'rebase':
 719         text += '''
 720 You can see logs of the failed task here:
 721
 722   %s/%s.stdout
 723   %s/%s.stderr
 724
 725 or you can get full logs of all tasks in this job here:
 726
 727   %s/logs.tar.gz
 728
 729 The top commit for the tree that was built was:
 730
 731 %s
 732
 733 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 734
 735     if add_log_tail:
 736         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 737         lines = f.readlines()
 738         log_tail = "".join(lines[-50:])
 739         num_lines = len(lines)
 740         if num_lines < 50:
 741             # Also include stderr (compile failures) if < 50 lines of stdout
 742             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 743             log_tail += "".join(f.readlines()[-(50-num_lines):])
 744
 745         text += '''
 746 The last 50 lines of log messages:
 747
 748 %s
 749     ''' % log_tail
 750         f.close()
 751
 752     logs = os.path.join(gitroot, 'logs.tar.gz')
 753     send_email('autobuild[%s] failure on %s for task %s during %s'
 754                % (options.branch, platform.node(), failed_task, failed_stage),
 755                text, logs)
 756
 757 def email_success(elapsed_time, log_base=None):
 758     '''send an email to options.email about a successful build'''
 759     user = os.getenv("USER")
 760     if log_base is None:
 761         log_base = gitroot
 762     text = '''
 763 Dear Developer,
 764
 765 Your autobuild on %s has succeeded after %.1f minutes.
 766
 767 ''' % (platform.node(), elapsed_time / 60.)
 768
 769     if options.restrict_tests:
 770         text += """
 771 The build was restricted to tests matching %s\n""" % options.restrict_tests
 772
 773     if options.keeplogs:
 774         text += '''
 775
 776 you can get full logs of all tasks in this job here:
 777
 778   %s/logs.tar.gz
 779
 780 ''' % log_base
 781
 782     text += '''
 783 The top commit for the tree that was built was:
 784
 785 %s
 786 ''' % top_commit_msg
 787
 788     logs = os.path.join(gitroot, 'logs.tar.gz')
 789     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 790                text, logs)
 791
 792
 793 (options, args) = parser.parse_args()
 794
 795 if options.retry:
 796     if options.rebase is None:
 797         raise Exception('You can only use --retry if you also rebase')
 798
 799 testbase = "%s/b%u" % (options.testbase, os.getpid())
 800 test_master = "%s/master" % testbase
 801 test_prefix = "%s/prefix" % testbase
 802 test_tmpdir = "%s/tmp" % testbase
 803 os.environ['TMPDIR'] = test_tmpdir
 804
 805 # get the top commit message, for emails
 806 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 807
 808 try:
 809     os.makedirs(testbase)
 810 except Exception as reason:
 811     raise Exception("Unable to create %s : %s" % (testbase, reason))
 812 cleanup_list.append(testbase)
 813
 814 if options.daemon:
 815     logfile = os.path.join(testbase, "log")
 816     do_print("Forking into the background, writing progress to %s" % logfile)
 817     daemonize(logfile)
 818
 819 write_pidfile(gitroot + "/autobuild.pid")
 820
 821 start_time = time.time()
 822
 823 while True:
 824     try:
 825         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 826         os.makedirs(test_tmpdir)
 827         # The waf uninstall code removes empty directories all the way
 828         # up the tree.  Creating a file in test_tmpdir stops it from
 829         # being removed.
 830         run_cmd("touch %s" % os.path.join(test_tmpdir,
 831                                           ".directory-is-not-empty"), show=True)
 832         run_cmd("stat %s" % test_tmpdir, show=True)
 833         run_cmd("stat %s" % testbase, show=True)
 834         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 835     except Exception:
 836         cleanup()
 837         raise
 838
 839     try:
 840         try:
 841             if options.rebase is not None:
 842                 rebase_tree(options.rebase, rebase_branch=options.branch)
 843         except Exception:
 844             cleanup_list.append(gitroot + "/autobuild.pid")
 845             cleanup()
 846             elapsed_time = time.time() - start_time
 847             email_failure(-1, 'rebase', 'rebase', 'rebase',
 848                           'rebase on %s failed' % options.branch,
 849                           elapsed_time, log_base=options.log_base)
 850             sys.exit(1)
 851         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 852         if options.tail:
 853             blist.start_tail()
 854         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 855         if status != 0 or errstr != "retry":
 856             break
 857         cleanup()
 858     except Exception:
 859         cleanup()
 860         raise
 861
 862 cleanup_list.append(gitroot + "/autobuild.pid")
 863
 864 do_print(errstr)
 865
 866 blist.kill_kids()
 867 if options.tail:
 868     do_print("waiting for tail to flush")
 869     time.sleep(1)
 870
 871 elapsed_time = time.time() - start_time
 872 if status == 0:
 873     if options.passcmd is not None:
 874         do_print("Running passcmd: %s" % options.passcmd)
 875         run_cmd(options.passcmd, dir=test_master)
 876     if options.pushto is not None:
 877         push_to(options.pushto, push_branch=options.branch)
 878     if options.keeplogs or options.attach_logs:
 879         blist.tarlogs("logs.tar.gz")
 880         do_print("Logs in logs.tar.gz")
 881     if options.always_email:
 882         email_success(elapsed_time, log_base=options.log_base)
 883     blist.remove_logs()
 884     cleanup()
 885     do_print(errstr)
 886     sys.exit(0)
 887
 888 # something failed, gather a tar of the logs
 889 blist.tarlogs("logs.tar.gz")
 890
 891 if options.email is not None:
 892     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 893                   elapsed_time, log_base=options.log_base)
 894 else:
 895     elapsed_minutes = elapsed_time / 60.0
 896     print('''
 897
 898 ####################################################################
 899
 900 AUTOBUILD FAILURE
 901
 902 Your autobuild[%s] on %s failed after %.1f minutes
 903 when trying to test %s with the following error:
 904
 905    %s
 906
 907 the autobuild has been abandoned. Please fix the error and resubmit.
 908
 909 ####################################################################
 910
 911 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 912
 913 cleanup()
 914 do_print(errstr)
 915 do_print("Logs in logs.tar.gz")
 916 sys.exit(status)