buildfarm/build.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2007-2010
   9 #
  10 #   This program is free software; you can redistribute it and/or modify
  11 #   it under the terms of the GNU General Public License as published by
  12 #   the Free Software Foundation; either version 2 of the License, or
  13 #   (at your option) any later version.
  14 #
  15 #   This program is distributed in the hope that it will be useful,
  16 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 #   GNU General Public License for more details.
  19 #
  20 #   You should have received a copy of the GNU General Public License
  21 #   along with this program; if not, write to the Free Software
  22 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 import bz2
  25 from cStringIO import StringIO
  26 import collections
  27 import hashlib
  28 import os
  29 import re
  30 from storm.locals import Int, RawStr
  31 from storm.store import Store
  32 from storm.expr import Desc
  33 import time
  34
  35
  36 def open_opt_compressed_file(path):
  37     try:
  38         return bz2.BZ2File(path+".bz2", 'r')
  39     except IOError:
  40         return open(path, 'r')
  41
  42
  43 class Test(object):
  44
  45     def __init__(self, name):
  46         self.name = name
  47
  48
  49
  50 class TestResult(object):
  51
  52     def __init__(self, build, test, result):
  53         self.build = build
  54         self.test = test
  55         self.result = result
  56
  57
  58 class BuildSummary(object):
  59
  60     def __init__(self, host, tree, compiler, revision, status):
  61         self.host = host
  62         self.tree = tree
  63         self.compiler = compiler
  64         self.revision = revision
  65         self.status = status
  66
  67
  68 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
  69
  70
  71 class MissingRevisionInfo(Exception):
  72     """Revision info could not be found in the build log."""
  73
  74     def __init__(self, build=None):
  75         self.build = build
  76
  77
  78 class LogFileMissing(Exception):
  79     """Log file missing."""
  80
  81
  82 class BuildStatus(object):
  83
  84     def __init__(self, stages=None, other_failures=None):
  85         if stages is not None:
  86             self.stages = [BuildStageResult(n, r) for (n, r) in stages]
  87         else:
  88             self.stages = []
  89         if other_failures is not None:
  90             self.other_failures = other_failures
  91         else:
  92             self.other_failures = set()
  93
  94     @property
  95     def failed(self):
  96         if self.other_failures:
  97             return True
  98         return not all([x.result == 0 for x in self.stages])
  99
 100     def __serialize__(self):
 101         return repr(self)
 102
 103     @classmethod
 104     def __deserialize__(cls, text):
 105         return eval(text)
 106
 107     def __str__(self):
 108         if self.other_failures:
 109             return ",".join(self.other_failures)
 110         return "/".join([str(x.result) for x in self.stages])
 111
 112     def broken_host(self):
 113         if "disk full" in self.other_failures:
 114             return True
 115         return False
 116
 117     def regressed_since(self, older):
 118         """Check if this build has regressed since another build."""
 119         if "disk full" in self.other_failures:
 120             return False
 121         if ("timeout" in self.other_failures and
 122             "timeout" in older.other_failures):
 123             # When the timeout happens exactly can differ slightly, so it's
 124             # okay if the numbers are a bit different..
 125             return False
 126         if ("panic" in self.other_failures and
 127             not "panic" in older.other_failures):
 128             # If this build introduced panics, then that's always worse.
 129             return True
 130         if len(self.stages) < len(older.stages):
 131             # Less stages completed
 132             return True
 133         old_stages = dict(older.stages)
 134         new_stages = dict(self.stages)
 135         for name, new_result in new_stages.iteritems():
 136             try:
 137                 old_result = old_stages[name]
 138             except KeyError:
 139                 continue
 140             if new_result == old_result:
 141                 continue
 142             if new_result < 0 and old_result >= 0:
 143                 return True
 144             elif new_result >= 0 and old_result < 0:
 145                 return False
 146             else:
 147                 return (abs(new_result) > abs(old_result))
 148         return False
 149
 150     def __cmp__(self, other):
 151         other_extra = other.other_failures - self.other_failures
 152         self_extra = self.other_failures - other.other_failures
 153         # Give more importance to other failures
 154         if other_extra:
 155             return 1
 156         if self_extra:
 157             return -1
 158
 159         la = len(self.stages)
 160         lb = len(other.stages)
 161         if la > lb:
 162             return 1
 163         elif lb > la:
 164             return -1
 165         else:
 166             return cmp(other.stages, self.stages)
 167
 168     def __repr__(self):
 169         return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
 170
 171
 172 def check_dir_exists(kind, path):
 173     if not os.path.isdir(path):
 174         raise Exception("%s directory %s does not exist" % (kind, path))
 175
 176
 177 def extract_phase_output(f):
 178     name = None
 179     output = None
 180     re_action = re.compile("^ACTION (PASSED|FAILED):\s+(.*)$")
 181     for l in f:
 182         if l.startswith("Running action "):
 183             name = l[len("Running action "):].strip()
 184             output = []
 185             continue
 186         m = re_action.match(l)
 187         if m:
 188             assert name == m.group(2).strip(), "%r != %r" % (name, m.group(2))
 189             yield name, output
 190             name = None
 191             output = []
 192         elif output is not None:
 193             output.append(l)
 194
 195
 196 def extract_test_output(f):
 197     for name, output in extract_phase_output(f):
 198         if name == "test":
 199             return output
 200     raise NoTestOutput()
 201
 202
 203 def build_status_from_logs(log, err):
 204     """get status of build"""
 205     # FIXME: Perhaps also extract revision here?
 206
 207     test_failures = 0
 208     test_successes = 0
 209     test_seen = 0
 210     ret = BuildStatus()
 211
 212     stages = []
 213     re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
 214     re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
 215
 216     for l in log:
 217         if l.startswith("No space left on device"):
 218             ret.other_failures.add("disk full")
 219             continue
 220         if "Maximum time expired in timelimit" in l: # Ugh.
 221             ret.other_failures.add("timeout")
 222             continue
 223         if "maximum runtime exceeded" in l: # Ugh.
 224             ret.other_failures.add("timeout")
 225             continue
 226         if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
 227             ret.other_failures.add("panic")
 228             continue
 229         if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
 230             test_failures += 1
 231             continue
 232         if l.startswith("testsuite-success: "):
 233             test_successes += 1
 234             continue
 235         m = re_status.match(l)
 236         if m:
 237             stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
 238             if m.group(1) == "TEST":
 239                 test_seen = 1
 240             continue
 241         m = re_action.match(l)
 242         if m and not test_seen:
 243             if m.group(1) == "PASSED":
 244                 stages.append(BuildStageResult("TEST", 0))
 245             else:
 246                 stages.append(BuildStageResult("TEST", 1))
 247             continue
 248
 249     # Scan err file for specific errors
 250     for l in err:
 251         if "No space left on device" in l:
 252             ret.other_failures.add("disk full")
 253
 254     def map_stage(sr):
 255         if sr.name != "TEST":
 256             return sr
 257         # TEST is special
 258         if test_successes + test_failures == 0:
 259             # No granular test output
 260             return BuildStageResult("TEST", sr.result)
 261         if sr.result == 1 and test_failures == 0:
 262             ret.other_failures.add("inconsistent test result")
 263             return BuildStageResult("TEST", -1)
 264         return BuildStageResult("TEST", test_failures)
 265
 266     ret.stages = map(map_stage, stages)
 267     return ret
 268
 269
 270 def revision_from_log(log):
 271     revid = None
 272     for l in log:
 273         if l.startswith("BUILD COMMIT REVISION: "):
 274             revid = l.split(":", 1)[1].strip()
 275     if revid is None:
 276         raise MissingRevisionInfo()
 277     return revid
 278
 279
 280 class NoSuchBuildError(Exception):
 281     """The build with the specified name does not exist."""
 282
 283     def __init__(self, tree, host, compiler, rev=None):
 284         self.tree = tree
 285         self.host = host
 286         self.compiler = compiler
 287         self.rev = rev
 288
 289
 290 class NoTestOutput(Exception):
 291     """The build did not have any associated test output."""
 292
 293
 294 class Build(object):
 295     """A single build of a tree on a particular host using a particular compiler.
 296     """
 297
 298     def __init__(self, basename, tree, host, compiler, rev=None):
 299         self.basename = basename
 300         self.tree = tree
 301         self.host = host
 302         self.compiler = compiler
 303         self.revision = rev
 304
 305     def __cmp__(self, other):
 306         return cmp(
 307             (self.upload_time, self.revision, self.host, self.tree, self.compiler),
 308             (other.upload_time, other.revision, other.host, other.tree, other.compiler))
 309
 310     def __eq__(self, other):
 311         return (isinstance(other, Build) and
 312                 self.log_checksum() == other.log_checksum())
 313
 314     def __repr__(self):
 315         if self.revision is not None:
 316             return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
 317         else:
 318             return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
 319
 320     def remove_logs(self):
 321         # In general, basename.log should *always* exist.
 322         if os.path.exists(self.basename+".log"):
 323             os.unlink(self.basename + ".log")
 324         if os.path.exists(self.basename+".err"):
 325             os.unlink(self.basename+".err")
 326
 327     def remove(self):
 328         self.remove_logs()
 329
 330     @property
 331     def upload_time(self):
 332         """get timestamp of build"""
 333         st = os.stat("%s.log" % self.basename)
 334         return st.st_mtime
 335
 336     @property
 337     def age(self):
 338         """get the age of build"""
 339         return time.time() - self.upload_time
 340
 341     def read_subunit(self):
 342         """read the test output as subunit"""
 343         return StringIO("".join(extract_test_output(self.read_log())))
 344
 345     def read_log(self):
 346         """read full log file"""
 347         try:
 348             return open_opt_compressed_file(self.basename+".log")
 349         except IOError:
 350             raise LogFileMissing()
 351
 352     def read_err(self):
 353         """read full err file"""
 354         try:
 355             return open_opt_compressed_file(self.basename+".err")
 356         except IOError:
 357             # No such file
 358             return StringIO()
 359
 360     def log_checksum(self):
 361         f = self.read_log()
 362         try:
 363             return hashlib.sha1(f.read()).hexdigest()
 364         finally:
 365             f.close()
 366
 367     def summary(self):
 368         revid = self.revision_details()
 369         status = self.status()
 370         return BuildSummary(self.host, self.tree, self.compiler, revid, status)
 371
 372     def revision_details(self):
 373         """get the revision of build
 374
 375         :return: revision id
 376         """
 377         f = self.read_log()
 378         try:
 379             return revision_from_log(f)
 380         finally:
 381             f.close()
 382
 383     def status(self):
 384         """get status of build
 385
 386         :return: tuple with build status
 387         """
 388         log = self.read_log()
 389         try:
 390             err = self.read_err()
 391             try:
 392                 return build_status_from_logs(log, err)
 393             finally:
 394                 err.close()
 395         finally:
 396             log.close()
 397
 398     def err_count(self):
 399         """get status of build"""
 400         file = self.read_err()
 401         return len(file.readlines())
 402
 403
 404 class UploadBuildResultStore(object):
 405
 406     def __init__(self, path):
 407         """Open the database.
 408
 409         :param path: Build result base directory
 410         """
 411         self.path = path
 412
 413     def get_all_builds(self):
 414         for name in os.listdir(self.path):
 415             try:
 416                 (build, tree, host, compiler, extension) = name.split(".")
 417             except ValueError:
 418                 continue
 419             if build != "build" or extension != "log":
 420                 continue
 421             yield self.get_build(tree, host, compiler)
 422
 423     def build_fname(self, tree, host, compiler):
 424         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 425
 426     def has_host(self, host):
 427         for name in os.listdir(self.path):
 428             try:
 429                 if name.split(".")[2] == host:
 430                     return True
 431             except IndexError:
 432                 pass
 433         return False
 434
 435     def get_build(self, tree, host, compiler):
 436         basename = self.build_fname(tree, host, compiler)
 437         logf = "%s.log" % basename
 438         if not os.path.exists(logf):
 439             raise NoSuchBuildError(tree, host, compiler)
 440         return Build(basename, tree, host, compiler)
 441
 442
 443 class StormBuild(Build):
 444     __storm_table__ = "build"
 445
 446     id = Int(primary=True)
 447     tree = RawStr()
 448     revision = RawStr()
 449     host = RawStr()
 450     compiler = RawStr()
 451     checksum = RawStr()
 452     upload_time = Int(name="age")
 453     status_str = RawStr(name="status")
 454     basename = RawStr()
 455     host_id = Int()
 456     tree_id = Int()
 457     compiler_id = Int()
 458
 459     def status(self):
 460         return BuildStatus.__deserialize__(self.status_str)
 461
 462     def revision_details(self):
 463         return self.revision
 464
 465     def log_checksum(self):
 466         return self.checksum
 467
 468     def remove(self):
 469         super(StormBuild, self).remove()
 470         Store.of(self).remove(self)
 471
 472     def remove_logs(self):
 473         super(StormBuild, self).remove_logs()
 474         self.basename = None
 475
 476
 477 class BuildResultStore(object):
 478     """The build farm build result database."""
 479
 480     def __init__(self, basedir, store=None):
 481         from buildfarm.sqldb import memory_store
 482         if store is None:
 483             store = memory_store()
 484
 485         self.store = store
 486         self.path = basedir
 487
 488     def __contains__(self, build):
 489         try:
 490             self.get_by_checksum(build.log_checksum())
 491             return True
 492         except NoSuchBuildError:
 493             return False
 494
 495     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 496         from buildfarm.sqldb import Cast
 497         expr = [
 498             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 499             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 500             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 501             ]
 502         if revision is not None:
 503             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 504         if checksum is not None:
 505             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 506         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 507         ret = result.first()
 508         if ret is None:
 509             raise NoSuchBuildError(tree, host, compiler, revision)
 510         return ret
 511
 512     def build_fname(self, tree, host, compiler, rev):
 513         """get the name of the build file"""
 514         return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
 515
 516     def get_all_builds(self):
 517         for l in os.listdir(self.path):
 518             m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
 519             if not m:
 520                 continue
 521             tree = m.group(1)
 522             host = m.group(2)
 523             compiler = m.group(3)
 524             rev = m.group(4)
 525             stat = os.stat(os.path.join(self.path, l))
 526             # skip the current build
 527             if stat.st_nlink == 2:
 528                 continue
 529             yield self.get_build(tree, host, compiler, rev)
 530
 531     def get_old_builds(self, tree, host, compiler):
 532         result = self.store.find(StormBuild,
 533             StormBuild.tree == tree,
 534             StormBuild.host == host,
 535             StormBuild.compiler == compiler)
 536         return result.order_by(Desc(StormBuild.upload_time))
 537
 538     def upload_build(self, build):
 539         from buildfarm.sqldb import Cast, StormHost
 540         try:
 541             existing_build = self.get_by_checksum(build.log_checksum())
 542         except NoSuchBuildError:
 543             pass
 544         else:
 545             # Already present
 546             assert build.tree == existing_build.tree
 547             assert build.host == existing_build.host
 548             assert build.compiler == existing_build.compiler
 549             return existing_build
 550         rev = build.revision_details()
 551
 552         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 553         for name in os.listdir(self.path):
 554             p = os.path.join(self.path, name)
 555             if p.startswith(new_basename+"."):
 556                 os.remove(p)
 557         os.link(build.basename+".log", new_basename+".log")
 558         if os.path.exists(build.basename+".err"):
 559             os.link(build.basename+".err", new_basename+".err")
 560         new_build = StormBuild(new_basename, build.tree, build.host,
 561             build.compiler, rev)
 562         new_build.checksum = build.log_checksum()
 563         new_build.upload_time = build.upload_time
 564         new_build.status_str = build.status().__serialize__()
 565         new_build.basename = new_basename
 566         host = self.store.find(StormHost,
 567             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 568         assert host is not None, "Unable to find host %r" % build.host
 569         new_build.host_id = host.id
 570         self.store.add(new_build)
 571         return new_build
 572
 573     def get_by_checksum(self, checksum):
 574         from buildfarm.sqldb import Cast
 575         result = self.store.find(StormBuild,
 576             Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT")).order_by(Desc(StormBuild.upload_time))
 577         ret = result.first()
 578         if ret is None:
 579             raise NoSuchBuildError(None, None, None, None)
 580         return ret
 581
 582     def get_previous_build(self, tree, host, compiler, revision):
 583         from buildfarm.sqldb import Cast
 584         cur_build = self.get_build(tree, host, compiler, revision)
 585
 586         result = self.store.find(StormBuild,
 587             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 588             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 589             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 590             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 591             StormBuild.id < cur_build.id)
 592         result = result.order_by(Desc(StormBuild.id))
 593         prev_build = result.first()
 594         if prev_build is None:
 595             raise NoSuchBuildError(tree, host, compiler, revision)
 596         return prev_build
 597
 598     def get_latest_build(self, tree, host, compiler):
 599         result = self.store.find(StormBuild,
 600             StormBuild.tree == tree,
 601             StormBuild.host == host,
 602             StormBuild.compiler == compiler)
 603         result = result.order_by(Desc(StormBuild.id))
 604         build = result.first()
 605         if build is None:
 606             raise NoSuchBuildError(tree, host, compiler)
 607         return build