buildfarm/build.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2007-2010
   9 #
  10 #   This program is free software; you can redistribute it and/or modify
  11 #   it under the terms of the GNU General Public License as published by
  12 #   the Free Software Foundation; either version 2 of the License, or
  13 #   (at your option) any later version.
  14 #
  15 #   This program is distributed in the hope that it will be useful,
  16 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 #   GNU General Public License for more details.
  19 #
  20 #   You should have received a copy of the GNU General Public License
  21 #   along with this program; if not, write to the Free Software
  22 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 from cStringIO import StringIO
  25 import collections
  26 import hashlib
  27 import os
  28 import re
  29 from storm.locals import Int, RawStr
  30 from storm.store import Store
  31 from storm.expr import Desc
  32 import time
  33
  34
  35 class Test(object):
  36
  37     def __init__(self, name):
  38         self.name = name
  39
  40
  41
  42 class TestResult(object):
  43
  44     def __init__(self, build, test, result):
  45         self.build = build
  46         self.test = test
  47         self.result = result
  48
  49
  50 class BuildSummary(object):
  51
  52     def __init__(self, host, tree, compiler, revision, status):
  53         self.host = host
  54         self.tree = tree
  55         self.compiler = compiler
  56         self.revision = revision
  57         self.status = status
  58
  59
  60 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
  61
  62
  63 class MissingRevisionInfo(Exception):
  64     """Revision info could not be found in the build log."""
  65
  66     def __init__(self, build=None):
  67         self.build = build
  68
  69
  70 class LogFileMissing(Exception):
  71     """Log file missing."""
  72
  73
  74 class BuildStatus(object):
  75
  76     def __init__(self, stages=None, other_failures=None):
  77         if stages is not None:
  78             self.stages = [BuildStageResult(n, r) for (n, r) in stages]
  79         else:
  80             self.stages = []
  81         if other_failures is not None:
  82             self.other_failures = other_failures
  83         else:
  84             self.other_failures = set()
  85
  86     @property
  87     def failed(self):
  88         if self.other_failures:
  89             return True
  90         return not all([x.result == 0 for x in self.stages])
  91
  92     def __serialize__(self):
  93         return repr(self)
  94
  95     @classmethod
  96     def __deserialize__(cls, text):
  97         return eval(text)
  98
  99     def __str__(self):
 100         if self.other_failures:
 101             return ",".join(self.other_failures)
 102         return "/".join([str(x.result) for x in self.stages])
 103
 104     def broken_host(self):
 105         if "disk full" in self.other_failures:
 106             return True
 107         return False
 108
 109     def regressed_since(self, older):
 110         """Check if this build has regressed since another build."""
 111         if "disk full" in self.other_failures:
 112             return False
 113         if ("timeout" in self.other_failures and
 114             "timeout" in older.other_failures):
 115             # When the timeout happens exactly can differ slightly, so it's
 116             # okay if the numbers are a bit different..
 117             return False
 118         if ("panic" in self.other_failures and
 119             not "panic" in older.other_failures):
 120             return True
 121         if len(self.stages) < len(older.stages):
 122             # Less stages completed
 123             return True
 124         for ((old_name, old_result), (new_name, new_result)) in zip(
 125             older.stages, self.stages):
 126             assert old_name == new_name
 127             if new_result > old_result:
 128                 return True
 129         return False
 130
 131     def __cmp__(self, other):
 132         other_extra = other.other_failures - self.other_failures
 133         self_extra = self.other_failures - other.other_failures
 134         # Give more importance to other failures
 135         if other_extra:
 136             return 1
 137         if self_extra:
 138             return -1
 139
 140         la = len(self.stages)
 141         lb = len(other.stages)
 142         if la > lb:
 143             return 1
 144         elif lb > la:
 145             return -1
 146         else:
 147             return cmp(other.stages, self.stages)
 148
 149     def __repr__(self):
 150         return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
 151
 152
 153 def check_dir_exists(kind, path):
 154     if not os.path.isdir(path):
 155         raise Exception("%s directory %s does not exist" % (kind, path))
 156
 157
 158 def build_status_from_logs(log, err):
 159     """get status of build"""
 160     # FIXME: Perhaps also extract revision here?
 161
 162     test_failures = 0
 163     test_successes = 0
 164     test_seen = 0
 165     ret = BuildStatus()
 166
 167     stages = []
 168     re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
 169     re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
 170
 171     for l in log:
 172         if l.startswith("No space left on device"):
 173             ret.other_failures.add("disk full")
 174             continue
 175         if "Maximum time expired in timelimit" in l: # Ugh.
 176             ret.other_failures.add("timeout")
 177             continue
 178         if "maximum runtime exceeded" in l: # Ugh.
 179             ret.other_failures.add("timeout")
 180             continue
 181         if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
 182             ret.other_failures.add("panic")
 183             continue
 184         if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
 185             test_failures += 1
 186             continue
 187         if l.startswith("testsuite-success: "):
 188             test_successes += 1
 189             continue
 190         m = re_status.match(l)
 191         if m:
 192             stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
 193             if m.group(1) == "TEST":
 194                 test_seen = 1
 195             continue
 196         m = re_action.match(l)
 197         if m and not test_seen:
 198             if m.group(1) == "PASSED":
 199                 stages.append(BuildStageResult("TEST", 0))
 200             else:
 201                 stages.append(BuildStageResult("TEST", 1))
 202             continue
 203
 204     # Scan err file for specific errors
 205     for l in err:
 206         if "No space left on device" in l:
 207             ret.other_failures.add("disk full")
 208
 209     def map_stage(sr):
 210         if sr.name != "TEST":
 211             return sr
 212         # TEST is special
 213         if test_successes + test_failures == 0:
 214             # No granular test output
 215             return BuildStageResult("TEST", sr.result)
 216         if sr.result == 1 and test_failures == 0:
 217             ret.other_failures.add("inconsistent test result")
 218             return BuildStageResult("TEST", -1)
 219         return BuildStageResult("TEST", test_failures)
 220
 221     ret.stages = map(map_stage, stages)
 222     return ret
 223
 224
 225 def revision_from_log(log):
 226     revid = None
 227     for l in log:
 228         if l.startswith("BUILD COMMIT REVISION: "):
 229             revid = l.split(":", 1)[1].strip()
 230     if revid is None:
 231         raise MissingRevisionInfo()
 232     return revid
 233
 234
 235 class NoSuchBuildError(Exception):
 236     """The build with the specified name does not exist."""
 237
 238     def __init__(self, tree, host, compiler, rev=None):
 239         self.tree = tree
 240         self.host = host
 241         self.compiler = compiler
 242         self.rev = rev
 243
 244
 245 class Build(object):
 246     """A single build of a tree on a particular host using a particular compiler.
 247     """
 248
 249     def __init__(self, basename, tree, host, compiler, rev=None):
 250         self.basename = basename
 251         self.tree = tree
 252         self.host = host
 253         self.compiler = compiler
 254         self.revision = rev
 255
 256     def __cmp__(self, other):
 257         return cmp(
 258             (self.upload_time, self.revision, self.host, self.tree, self.compiler),
 259             (other.upload_time, other.revision, other.host, other.tree, other.compiler))
 260
 261     def __eq__(self, other):
 262         return (isinstance(other, Build) and
 263                 self.log_checksum() == other.log_checksum())
 264
 265     def __repr__(self):
 266         if self.revision is not None:
 267             return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
 268         else:
 269             return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
 270
 271     def remove_logs(self):
 272         # In general, basename.log should *always* exist.
 273         if os.path.exists(self.basename+".log"):
 274             os.unlink(self.basename + ".log")
 275         if os.path.exists(self.basename+".err"):
 276             os.unlink(self.basename+".err")
 277
 278     def remove(self):
 279         self.remove_logs()
 280
 281     @property
 282     def upload_time(self):
 283         """get timestamp of build"""
 284         st = os.stat("%s.log" % self.basename)
 285         return st.st_mtime
 286
 287     @property
 288     def age(self):
 289         """get the age of build"""
 290         return time.time() - self.upload_time
 291
 292     def read_log(self):
 293         """read full log file"""
 294         try:
 295             return open(self.basename+".log", "r")
 296         except IOError:
 297             raise LogFileMissing()
 298
 299     def read_err(self):
 300         """read full err file"""
 301         try:
 302             return open(self.basename+".err", 'r')
 303         except IOError:
 304             # No such file
 305             return StringIO()
 306
 307     def log_checksum(self):
 308         f = self.read_log()
 309         try:
 310             return hashlib.sha1(f.read()).hexdigest()
 311         finally:
 312             f.close()
 313
 314     def summary(self):
 315         revid = self.revision_details()
 316         status = self.status()
 317         return BuildSummary(self.host, self.tree, self.compiler, revid, status)
 318
 319     def revision_details(self):
 320         """get the revision of build
 321
 322         :return: revision id
 323         """
 324         f = self.read_log()
 325         try:
 326             return revision_from_log(f)
 327         finally:
 328             f.close()
 329
 330     def status(self):
 331         """get status of build
 332
 333         :return: tuple with build status
 334         """
 335         log = self.read_log()
 336         try:
 337             err = self.read_err()
 338             try:
 339                 return build_status_from_logs(log, err)
 340             finally:
 341                 err.close()
 342         finally:
 343             log.close()
 344
 345     def err_count(self):
 346         """get status of build"""
 347         file = self.read_err()
 348         return len(file.readlines())
 349
 350
 351 class UploadBuildResultStore(object):
 352
 353     def __init__(self, path):
 354         """Open the database.
 355
 356         :param path: Build result base directory
 357         """
 358         self.path = path
 359
 360     def get_all_builds(self):
 361         for name in os.listdir(self.path):
 362             try:
 363                 (build, tree, host, compiler, extension) = name.split(".")
 364             except ValueError:
 365                 continue
 366             if build != "build" or extension != "log":
 367                 continue
 368             yield self.get_build(tree, host, compiler)
 369
 370     def build_fname(self, tree, host, compiler):
 371         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 372
 373     def has_host(self, host):
 374         for name in os.listdir(self.path):
 375             try:
 376                 if name.split(".")[2] == host:
 377                     return True
 378             except IndexError:
 379                 pass
 380         return False
 381
 382     def get_build(self, tree, host, compiler):
 383         basename = self.build_fname(tree, host, compiler)
 384         logf = "%s.log" % basename
 385         if not os.path.exists(logf):
 386             raise NoSuchBuildError(tree, host, compiler)
 387         return Build(basename, tree, host, compiler)
 388
 389
 390 class StormBuild(Build):
 391     __storm_table__ = "build"
 392
 393     id = Int(primary=True)
 394     tree = RawStr()
 395     revision = RawStr()
 396     host = RawStr()
 397     compiler = RawStr()
 398     checksum = RawStr()
 399     upload_time = Int(name="age")
 400     status_str = RawStr(name="status")
 401     basename = RawStr()
 402     host_id = Int()
 403     tree_id = Int()
 404     compiler_id = Int()
 405
 406     def status(self):
 407         return BuildStatus.__deserialize__(self.status_str)
 408
 409     def revision_details(self):
 410         return self.revision
 411
 412     def log_checksum(self):
 413         return self.checksum
 414
 415     def remove(self):
 416         super(StormBuild, self).remove()
 417         Store.of(self).remove(self)
 418
 419     def remove_logs(self):
 420         super(StormBuild, self).remove_logs()
 421         self.basename = None
 422
 423
 424 class BuildResultStore(object):
 425     """The build farm build result database."""
 426
 427     def __init__(self, basedir, store=None):
 428         from buildfarm.sqldb import memory_store
 429         if store is None:
 430             store = memory_store()
 431
 432         self.store = store
 433         self.path = basedir
 434
 435     def __contains__(self, build):
 436         try:
 437             self.get_by_checksum(build.log_checksum())
 438             return True
 439         except NoSuchBuildError:
 440             return False
 441
 442     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 443         from buildfarm.sqldb import Cast
 444         expr = [
 445             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 446             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 447             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 448             ]
 449         if revision is not None:
 450             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 451         if checksum is not None:
 452             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 453         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 454         ret = result.first()
 455         if ret is None:
 456             raise NoSuchBuildError(tree, host, compiler, revision)
 457         return ret
 458
 459     def build_fname(self, tree, host, compiler, rev):
 460         """get the name of the build file"""
 461         return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
 462
 463     def get_all_builds(self):
 464         for l in os.listdir(self.path):
 465             m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
 466             if not m:
 467                 continue
 468             tree = m.group(1)
 469             host = m.group(2)
 470             compiler = m.group(3)
 471             rev = m.group(4)
 472             stat = os.stat(os.path.join(self.path, l))
 473             # skip the current build
 474             if stat.st_nlink == 2:
 475                 continue
 476             yield self.get_build(tree, host, compiler, rev)
 477
 478     def get_old_builds(self, tree, host, compiler):
 479         result = self.store.find(StormBuild,
 480             StormBuild.tree == tree,
 481             StormBuild.host == host,
 482             StormBuild.compiler == compiler)
 483         return result.order_by(Desc(StormBuild.upload_time))
 484
 485     def upload_build(self, build):
 486         from buildfarm.sqldb import Cast, StormHost
 487         try:
 488             existing_build = self.get_by_checksum(build.log_checksum())
 489         except NoSuchBuildError:
 490             pass
 491         else:
 492             # Already present
 493             assert build.tree == existing_build.tree
 494             assert build.host == existing_build.host
 495             assert build.compiler == existing_build.compiler
 496             return existing_build
 497         rev = build.revision_details()
 498
 499         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 500         try:
 501             existing_build = self.get_build(build.tree, build.host, build.compiler, rev)
 502         except NoSuchBuildError:
 503             if os.path.exists(new_basename+".log"):
 504                 os.remove(new_basename+".log")
 505             if os.path.exists(new_basename+".err"):
 506                 os.remove(new_basename+".err")
 507         else:
 508             existing_build.remove_logs()
 509         os.link(build.basename+".log", new_basename+".log")
 510         if os.path.exists(build.basename+".err"):
 511             os.link(build.basename+".err", new_basename+".err")
 512         new_basename = self.build_fname(build.tree, build.host, build.compiler,
 513                 rev)
 514         new_build = StormBuild(new_basename, build.tree, build.host,
 515             build.compiler, rev)
 516         new_build.checksum = build.log_checksum()
 517         new_build.upload_time = build.upload_time
 518         new_build.status_str = build.status().__serialize__()
 519         new_build.basename = new_basename
 520         host = self.store.find(StormHost,
 521             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 522         assert host is not None, "Unable to find host %r" % build.host
 523         new_build.host_id = host.id
 524         self.store.add(new_build)
 525         return new_build
 526
 527     def get_by_checksum(self, checksum):
 528         from buildfarm.sqldb import Cast
 529         result = self.store.find(StormBuild,
 530             Cast(StormBuild.checksum, "TEXT") == checksum)
 531         ret = result.one()
 532         if ret is None:
 533             raise NoSuchBuildError(None, None, None, None)
 534         return ret
 535
 536     def get_previous_revision(self, tree, host, compiler, revision):
 537         from buildfarm.sqldb import Cast
 538         cur_build = self.get_build(tree, host, compiler, revision)
 539
 540         result = self.store.find(StormBuild,
 541             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 542             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 543             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 544             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 545             StormBuild.id < cur_build.id)
 546         result = result.order_by(Desc(StormBuild.id))
 547         prev_build = result.first()
 548         if prev_build is None:
 549             raise NoSuchBuildError(tree, host, compiler, revision)
 550         return prev_build.revision
 551
 552     def get_latest_revision(self, tree, host, compiler):
 553         result = self.store.find(StormBuild,
 554             StormBuild.tree == tree,
 555             StormBuild.host == host,
 556             StormBuild.compiler == compiler)
 557         result = result.order_by(Desc(StormBuild.id))
 558         build = result.first()
 559         if build is None:
 560             raise NoSuchBuildError(tree, host, compiler)
 561         return build.revision