buildfarm/build.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2007-2010
   9 #
  10 #   This program is free software; you can redistribute it and/or modify
  11 #   it under the terms of the GNU General Public License as published by
  12 #   the Free Software Foundation; either version 2 of the License, or
  13 #   (at your option) any later version.
  14 #
  15 #   This program is distributed in the hope that it will be useful,
  16 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 #   GNU General Public License for more details.
  19 #
  20 #   You should have received a copy of the GNU General Public License
  21 #   along with this program; if not, write to the Free Software
  22 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 import bz2
  25 from cStringIO import StringIO
  26 import collections
  27 import hashlib
  28 import os
  29 import re
  30 from storm.locals import Int, RawStr
  31 from storm.store import Store
  32 from storm.expr import Desc
  33 import time
  34
  35
  36 def open_opt_compressed_file(path):
  37     try:
  38         return bz2.BZ2File(path+".bz2", 'r')
  39     except IOError:
  40         return open(path, 'r')
  41
  42
  43 class Test(object):
  44
  45     def __init__(self, name):
  46         self.name = name
  47
  48
  49
  50 class TestResult(object):
  51
  52     def __init__(self, build, test, result):
  53         self.build = build
  54         self.test = test
  55         self.result = result
  56
  57
  58 class BuildSummary(object):
  59
  60     def __init__(self, host, tree, compiler, revision, status):
  61         self.host = host
  62         self.tree = tree
  63         self.compiler = compiler
  64         self.revision = revision
  65         self.status = status
  66
  67
  68 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
  69
  70
  71 class MissingRevisionInfo(Exception):
  72     """Revision info could not be found in the build log."""
  73
  74     def __init__(self, build=None):
  75         self.build = build
  76
  77
  78 class LogFileMissing(Exception):
  79     """Log file missing."""
  80
  81
  82 class BuildStatus(object):
  83
  84     def __init__(self, stages=None, other_failures=None):
  85         if stages is not None:
  86             self.stages = [BuildStageResult(n, r) for (n, r) in stages]
  87         else:
  88             self.stages = []
  89         if other_failures is not None:
  90             self.other_failures = other_failures
  91         else:
  92             self.other_failures = set()
  93
  94     @property
  95     def failed(self):
  96         if self.other_failures:
  97             return True
  98         return not all([x.result == 0 for x in self.stages])
  99
 100     def __serialize__(self):
 101         return repr(self)
 102
 103     @classmethod
 104     def __deserialize__(cls, text):
 105         return eval(text)
 106
 107     def __str__(self):
 108         if self.other_failures:
 109             return ",".join(self.other_failures)
 110         return "/".join([str(x.result) for x in self.stages])
 111
 112     def broken_host(self):
 113         if "disk full" in self.other_failures:
 114             return True
 115         return False
 116
 117     def regressed_since(self, older):
 118         """Check if this build has regressed since another build."""
 119         if "disk full" in self.other_failures:
 120             return False
 121         if ("timeout" in self.other_failures and
 122             "timeout" in older.other_failures):
 123             # When the timeout happens exactly can differ slightly, so it's
 124             # okay if the numbers are a bit different..
 125             return False
 126         if ("panic" in self.other_failures and
 127             not "panic" in older.other_failures):
 128             return True
 129         if len(self.stages) < len(older.stages):
 130             # Less stages completed
 131             return True
 132         for ((old_name, old_result), (new_name, new_result)) in zip(
 133             older.stages, self.stages):
 134             assert old_name == new_name
 135             if new_result > old_result:
 136                 return True
 137         return False
 138
 139     def __cmp__(self, other):
 140         other_extra = other.other_failures - self.other_failures
 141         self_extra = self.other_failures - other.other_failures
 142         # Give more importance to other failures
 143         if other_extra:
 144             return 1
 145         if self_extra:
 146             return -1
 147
 148         la = len(self.stages)
 149         lb = len(other.stages)
 150         if la > lb:
 151             return 1
 152         elif lb > la:
 153             return -1
 154         else:
 155             return cmp(other.stages, self.stages)
 156
 157     def __repr__(self):
 158         return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
 159
 160
 161 def check_dir_exists(kind, path):
 162     if not os.path.isdir(path):
 163         raise Exception("%s directory %s does not exist" % (kind, path))
 164
 165
 166 def extract_phase_output(f):
 167     name = None
 168     output = None
 169     re_action = re.compile("^ACTION (PASSED|FAILED):\s+(.*)$")
 170     for l in f:
 171         if l.startswith("Running action "):
 172             name = l[len("Running action "):].strip()
 173             output = []
 174             continue
 175         m = re_action.match(l)
 176         if m:
 177             assert name == m.group(2).strip(), "%r != %r" % (name, m.group(2))
 178             yield name, output
 179             name = None
 180             output = []
 181         elif output is not None:
 182             output.append(l)
 183
 184
 185 def extract_test_output(f):
 186     for name, output in extract_phase_output(f):
 187         if name == "test":
 188             return output
 189     raise NoTestOutput()
 190
 191
 192 def build_status_from_logs(log, err):
 193     """get status of build"""
 194     # FIXME: Perhaps also extract revision here?
 195
 196     test_failures = 0
 197     test_successes = 0
 198     test_seen = 0
 199     ret = BuildStatus()
 200
 201     stages = []
 202     re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
 203     re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
 204
 205     for l in log:
 206         if l.startswith("No space left on device"):
 207             ret.other_failures.add("disk full")
 208             continue
 209         if "Maximum time expired in timelimit" in l: # Ugh.
 210             ret.other_failures.add("timeout")
 211             continue
 212         if "maximum runtime exceeded" in l: # Ugh.
 213             ret.other_failures.add("timeout")
 214             continue
 215         if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
 216             ret.other_failures.add("panic")
 217             continue
 218         if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
 219             test_failures += 1
 220             continue
 221         if l.startswith("testsuite-success: "):
 222             test_successes += 1
 223             continue
 224         m = re_status.match(l)
 225         if m:
 226             stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
 227             if m.group(1) == "TEST":
 228                 test_seen = 1
 229             continue
 230         m = re_action.match(l)
 231         if m and not test_seen:
 232             if m.group(1) == "PASSED":
 233                 stages.append(BuildStageResult("TEST", 0))
 234             else:
 235                 stages.append(BuildStageResult("TEST", 1))
 236             continue
 237
 238     # Scan err file for specific errors
 239     for l in err:
 240         if "No space left on device" in l:
 241             ret.other_failures.add("disk full")
 242
 243     def map_stage(sr):
 244         if sr.name != "TEST":
 245             return sr
 246         # TEST is special
 247         if test_successes + test_failures == 0:
 248             # No granular test output
 249             return BuildStageResult("TEST", sr.result)
 250         if sr.result == 1 and test_failures == 0:
 251             ret.other_failures.add("inconsistent test result")
 252             return BuildStageResult("TEST", -1)
 253         return BuildStageResult("TEST", test_failures)
 254
 255     ret.stages = map(map_stage, stages)
 256     return ret
 257
 258
 259 def revision_from_log(log):
 260     revid = None
 261     for l in log:
 262         if l.startswith("BUILD COMMIT REVISION: "):
 263             revid = l.split(":", 1)[1].strip()
 264     if revid is None:
 265         raise MissingRevisionInfo()
 266     return revid
 267
 268
 269 class NoSuchBuildError(Exception):
 270     """The build with the specified name does not exist."""
 271
 272     def __init__(self, tree, host, compiler, rev=None):
 273         self.tree = tree
 274         self.host = host
 275         self.compiler = compiler
 276         self.rev = rev
 277
 278
 279 class NoTestOutput(Exception):
 280     """The build did not have any associated test output."""
 281
 282
 283 class Build(object):
 284     """A single build of a tree on a particular host using a particular compiler.
 285     """
 286
 287     def __init__(self, basename, tree, host, compiler, rev=None):
 288         self.basename = basename
 289         self.tree = tree
 290         self.host = host
 291         self.compiler = compiler
 292         self.revision = rev
 293
 294     def __cmp__(self, other):
 295         return cmp(
 296             (self.upload_time, self.revision, self.host, self.tree, self.compiler),
 297             (other.upload_time, other.revision, other.host, other.tree, other.compiler))
 298
 299     def __eq__(self, other):
 300         return (isinstance(other, Build) and
 301                 self.log_checksum() == other.log_checksum())
 302
 303     def __repr__(self):
 304         if self.revision is not None:
 305             return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
 306         else:
 307             return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
 308
 309     def remove_logs(self):
 310         # In general, basename.log should *always* exist.
 311         if os.path.exists(self.basename+".log"):
 312             os.unlink(self.basename + ".log")
 313         if os.path.exists(self.basename+".err"):
 314             os.unlink(self.basename+".err")
 315
 316     def remove(self):
 317         self.remove_logs()
 318
 319     @property
 320     def upload_time(self):
 321         """get timestamp of build"""
 322         st = os.stat("%s.log" % self.basename)
 323         return st.st_mtime
 324
 325     @property
 326     def age(self):
 327         """get the age of build"""
 328         return time.time() - self.upload_time
 329
 330     def read_subunit(self):
 331         """read the test output as subunit"""
 332         return StringIO("".join(extract_test_output(self.read_log())))
 333
 334     def read_log(self):
 335         """read full log file"""
 336         try:
 337             return open_opt_compressed_file(self.basename+".log")
 338         except IOError:
 339             raise LogFileMissing()
 340
 341     def read_err(self):
 342         """read full err file"""
 343         try:
 344             return open_opt_compressed_file(self.basename+".err")
 345         except IOError:
 346             # No such file
 347             return StringIO()
 348
 349     def log_checksum(self):
 350         f = self.read_log()
 351         try:
 352             return hashlib.sha1(f.read()).hexdigest()
 353         finally:
 354             f.close()
 355
 356     def summary(self):
 357         revid = self.revision_details()
 358         status = self.status()
 359         return BuildSummary(self.host, self.tree, self.compiler, revid, status)
 360
 361     def revision_details(self):
 362         """get the revision of build
 363
 364         :return: revision id
 365         """
 366         f = self.read_log()
 367         try:
 368             return revision_from_log(f)
 369         finally:
 370             f.close()
 371
 372     def status(self):
 373         """get status of build
 374
 375         :return: tuple with build status
 376         """
 377         log = self.read_log()
 378         try:
 379             err = self.read_err()
 380             try:
 381                 return build_status_from_logs(log, err)
 382             finally:
 383                 err.close()
 384         finally:
 385             log.close()
 386
 387     def err_count(self):
 388         """get status of build"""
 389         file = self.read_err()
 390         return len(file.readlines())
 391
 392
 393 class UploadBuildResultStore(object):
 394
 395     def __init__(self, path):
 396         """Open the database.
 397
 398         :param path: Build result base directory
 399         """
 400         self.path = path
 401
 402     def get_all_builds(self):
 403         for name in os.listdir(self.path):
 404             try:
 405                 (build, tree, host, compiler, extension) = name.split(".")
 406             except ValueError:
 407                 continue
 408             if build != "build" or extension != "log":
 409                 continue
 410             yield self.get_build(tree, host, compiler)
 411
 412     def build_fname(self, tree, host, compiler):
 413         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 414
 415     def has_host(self, host):
 416         for name in os.listdir(self.path):
 417             try:
 418                 if name.split(".")[2] == host:
 419                     return True
 420             except IndexError:
 421                 pass
 422         return False
 423
 424     def get_build(self, tree, host, compiler):
 425         basename = self.build_fname(tree, host, compiler)
 426         logf = "%s.log" % basename
 427         if not os.path.exists(logf):
 428             raise NoSuchBuildError(tree, host, compiler)
 429         return Build(basename, tree, host, compiler)
 430
 431
 432 class StormBuild(Build):
 433     __storm_table__ = "build"
 434
 435     id = Int(primary=True)
 436     tree = RawStr()
 437     revision = RawStr()
 438     host = RawStr()
 439     compiler = RawStr()
 440     checksum = RawStr()
 441     upload_time = Int(name="age")
 442     status_str = RawStr(name="status")
 443     basename = RawStr()
 444     host_id = Int()
 445     tree_id = Int()
 446     compiler_id = Int()
 447
 448     def status(self):
 449         return BuildStatus.__deserialize__(self.status_str)
 450
 451     def revision_details(self):
 452         return self.revision
 453
 454     def log_checksum(self):
 455         return self.checksum
 456
 457     def remove(self):
 458         super(StormBuild, self).remove()
 459         Store.of(self).remove(self)
 460
 461     def remove_logs(self):
 462         super(StormBuild, self).remove_logs()
 463         self.basename = None
 464
 465
 466 class BuildResultStore(object):
 467     """The build farm build result database."""
 468
 469     def __init__(self, basedir, store=None):
 470         from buildfarm.sqldb import memory_store
 471         if store is None:
 472             store = memory_store()
 473
 474         self.store = store
 475         self.path = basedir
 476
 477     def __contains__(self, build):
 478         try:
 479             self.get_by_checksum(build.log_checksum())
 480             return True
 481         except NoSuchBuildError:
 482             return False
 483
 484     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 485         from buildfarm.sqldb import Cast
 486         expr = [
 487             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 488             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 489             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 490             ]
 491         if revision is not None:
 492             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 493         if checksum is not None:
 494             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 495         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 496         ret = result.first()
 497         if ret is None:
 498             raise NoSuchBuildError(tree, host, compiler, revision)
 499         return ret
 500
 501     def build_fname(self, tree, host, compiler, rev):
 502         """get the name of the build file"""
 503         return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
 504
 505     def get_all_builds(self):
 506         for l in os.listdir(self.path):
 507             m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
 508             if not m:
 509                 continue
 510             tree = m.group(1)
 511             host = m.group(2)
 512             compiler = m.group(3)
 513             rev = m.group(4)
 514             stat = os.stat(os.path.join(self.path, l))
 515             # skip the current build
 516             if stat.st_nlink == 2:
 517                 continue
 518             yield self.get_build(tree, host, compiler, rev)
 519
 520     def get_old_builds(self, tree, host, compiler):
 521         result = self.store.find(StormBuild,
 522             StormBuild.tree == tree,
 523             StormBuild.host == host,
 524             StormBuild.compiler == compiler)
 525         return result.order_by(Desc(StormBuild.upload_time))
 526
 527     def upload_build(self, build):
 528         from buildfarm.sqldb import Cast, StormHost
 529         try:
 530             existing_build = self.get_by_checksum(build.log_checksum())
 531         except NoSuchBuildError:
 532             pass
 533         else:
 534             # Already present
 535             assert build.tree == existing_build.tree
 536             assert build.host == existing_build.host
 537             assert build.compiler == existing_build.compiler
 538             return existing_build
 539         rev = build.revision_details()
 540
 541         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 542         for name in os.listdir(self.path):
 543             p = os.path.join(self.path, name)
 544             if p.startswith(new_basename+"."):
 545                 os.remove(p)
 546         os.link(build.basename+".log", new_basename+".log")
 547         if os.path.exists(build.basename+".err"):
 548             os.link(build.basename+".err", new_basename+".err")
 549         new_build = StormBuild(new_basename, build.tree, build.host,
 550             build.compiler, rev)
 551         new_build.checksum = build.log_checksum()
 552         new_build.upload_time = build.upload_time
 553         new_build.status_str = build.status().__serialize__()
 554         new_build.basename = new_basename
 555         host = self.store.find(StormHost,
 556             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 557         assert host is not None, "Unable to find host %r" % build.host
 558         new_build.host_id = host.id
 559         self.store.add(new_build)
 560         return new_build
 561
 562     def get_by_checksum(self, checksum):
 563         from buildfarm.sqldb import Cast
 564         result = self.store.find(StormBuild,
 565             Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT")).order_by(Desc(StormBuild.upload_time))
 566         ret = result.first()
 567         if ret is None:
 568             raise NoSuchBuildError(None, None, None, None)
 569         return ret
 570
 571     def get_previous_build(self, tree, host, compiler, revision):
 572         from buildfarm.sqldb import Cast
 573         cur_build = self.get_build(tree, host, compiler, revision)
 574
 575         result = self.store.find(StormBuild,
 576             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 577             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 578             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 579             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 580             StormBuild.id < cur_build.id)
 581         result = result.order_by(Desc(StormBuild.id))
 582         prev_build = result.first()
 583         if prev_build is None:
 584             raise NoSuchBuildError(tree, host, compiler, revision)
 585         return prev_build
 586
 587     def get_latest_build(self, tree, host, compiler):
 588         result = self.store.find(StormBuild,
 589             StormBuild.tree == tree,
 590             StormBuild.host == host,
 591             StormBuild.compiler == compiler)
 592         result = result.order_by(Desc(StormBuild.id))
 593         build = result.first()
 594         if build is None:
 595             raise NoSuchBuildError(tree, host, compiler)
 596         return build