# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+import bz2
from cStringIO import StringIO
import collections
import hashlib
import os
import re
+from storm.locals import Int, RawStr
+from storm.store import Store
+from storm.expr import Desc
import time
+def open_opt_compressed_file(path):
+ try:
+ return bz2.BZ2File(path+".bz2", 'r')
+ except IOError:
+ return open(path, 'r')
+
+
+class Test(object):
+
+ def __init__(self, name):
+ self.name = name
+
+
+
+class TestResult(object):
+
+ def __init__(self, build, test, result):
+ self.build = build
+ self.test = test
+ self.result = result
+
+
class BuildSummary(object):
def __init__(self, host, tree, compiler, revision, status):
return False
if ("panic" in self.other_failures and
not "panic" in older.other_failures):
+ # If this build introduced panics, then that's always worse.
return True
if len(self.stages) < len(older.stages):
# Less stages completed
return True
- for ((old_name, old_result), (new_name, new_result)) in zip(
- older.stages, self.stages):
- assert old_name == new_name
- if new_result > old_result:
+ old_stages = dict(older.stages)
+ new_stages = dict(self.stages)
+ for name, new_result in new_stages.iteritems():
+ try:
+ old_result = old_stages[name]
+ except KeyError:
+ continue
+ if new_result == old_result:
+ continue
+ if new_result < 0 and old_result >= 0:
return True
+ elif new_result >= 0 and old_result < 0:
+ return False
+ else:
+ return (abs(new_result) > abs(old_result))
return False
def __cmp__(self, other):
raise Exception("%s directory %s does not exist" % (kind, path))
+def extract_phase_output(f):
+ name = None
+ output = None
+ re_action = re.compile("^ACTION (PASSED|FAILED):\s+(.*)$")
+ for l in f:
+ if l.startswith("Running action "):
+ name = l[len("Running action "):].strip()
+ output = []
+ continue
+ m = re_action.match(l)
+ if m:
+ assert name == m.group(2).strip(), "%r != %r" % (name, m.group(2))
+ yield name, output
+ name = None
+ output = []
+ elif output is not None:
+ output.append(l)
+
+
+def extract_test_output(f):
+ for name, output in extract_phase_output(f):
+ if name == "test":
+ return output
+ raise NoTestOutput()
+
+
def build_status_from_logs(log, err):
"""get status of build"""
# FIXME: Perhaps also extract revision here?
if l.startswith("No space left on device"):
ret.other_failures.add("disk full")
continue
+ if "Maximum time expired in timelimit" in l: # Ugh.
+ ret.other_failures.add("timeout")
+ continue
if "maximum runtime exceeded" in l: # Ugh.
ret.other_failures.add("timeout")
continue
self.rev = rev
+class NoTestOutput(Exception):
+ """The build did not have any associated test output."""
+
+
class Build(object):
"""A single build of a tree on a particular host using a particular compiler.
"""
"""get the age of build"""
return time.time() - self.upload_time
+ def read_subunit(self):
+ """read the test output as subunit"""
+ return StringIO("".join(extract_test_output(self.read_log())))
+
def read_log(self):
"""read full log file"""
try:
- return open(self.basename+".log", "r")
+ return open_opt_compressed_file(self.basename+".log")
except IOError:
raise LogFileMissing()
+ def has_log(self):
+ try:
+ f = self.read_log()
+ except LogFileMissing:
+ return False
+ else:
+ f.close()
+ return True
+
def read_err(self):
"""read full err file"""
try:
- return open(self.basename+".err", 'r')
+ return open_opt_compressed_file(self.basename+".err")
except IOError:
# No such file
return StringIO()
return Build(basename, tree, host, compiler)
+class StormBuild(Build):
+ __storm_table__ = "build"
+
+ id = Int(primary=True)
+ tree = RawStr()
+ revision = RawStr()
+ host = RawStr()
+ compiler = RawStr()
+ checksum = RawStr()
+ upload_time = Int(name="age")
+ status_str = RawStr(name="status")
+ basename = RawStr()
+ host_id = Int()
+ tree_id = Int()
+ compiler_id = Int()
+
+ def status(self):
+ return BuildStatus.__deserialize__(self.status_str)
+
+ def revision_details(self):
+ return self.revision
+
+ def log_checksum(self):
+ return self.checksum
+
+ def remove(self):
+ super(StormBuild, self).remove()
+ Store.of(self).remove(self)
+
+ def remove_logs(self):
+ super(StormBuild, self).remove_logs()
+ self.basename = None
+
+
class BuildResultStore(object):
"""The build farm build result database."""
- def __init__(self, path):
- """Open the database.
+ def __init__(self, basedir, store=None):
+ from buildfarm.sqldb import memory_store
+ if store is None:
+ store = memory_store()
- :param path: Build result base directory
- """
- self.path = path
+ self.store = store
+ self.path = basedir
def __contains__(self, build):
try:
- if build.revision:
- rev = build.revision
- else:
- rev = build.revision_details()
- self.get_build(build.tree, build.host, build.compiler, rev)
+ self.get_by_checksum(build.log_checksum())
+ return True
except NoSuchBuildError:
return False
- else:
- return True
- def get_build(self, tree, host, compiler, rev, checksum=None):
- basename = self.build_fname(tree, host, compiler, rev)
- logf = "%s.log" % basename
- if not os.path.exists(logf):
- raise NoSuchBuildError(tree, host, compiler, rev)
- return Build(basename, tree, host, compiler, rev)
+ def get_build(self, tree, host, compiler, revision=None, checksum=None):
+ from buildfarm.sqldb import Cast
+ expr = [
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
+ Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
+ ]
+ if revision is not None:
+ expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
+ if checksum is not None:
+ expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
+ result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
+ ret = result.first()
+ if ret is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return ret
def build_fname(self, tree, host, compiler, rev):
"""get the name of the build file"""
yield self.get_build(tree, host, compiler, rev)
def get_old_builds(self, tree, host, compiler):
- """get a list of old builds and their status."""
- ret = []
- for build in self.get_all_builds():
- if build.tree == tree and build.host == host and build.compiler == compiler:
- ret.append(build)
- ret.sort(lambda a, b: cmp(a.upload_time, b.upload_time))
- return ret
+ result = self.store.find(StormBuild,
+ StormBuild.tree == tree,
+ StormBuild.host == host,
+ StormBuild.compiler == compiler)
+ return result.order_by(Desc(StormBuild.upload_time))
def upload_build(self, build):
- rev = build.revision_details()
-
- new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
+ from buildfarm.sqldb import Cast, StormHost
try:
- existing_build = self.get_build(build.tree, build.host, build.compiler, rev)
+ existing_build = self.get_by_checksum(build.log_checksum())
except NoSuchBuildError:
- if os.path.exists(new_basename+".log"):
- os.remove(new_basename+".log")
- if os.path.exists(new_basename+".err"):
- os.remove(new_basename+".err")
+ pass
else:
- existing_build.remove_logs()
+ # Already present
+ assert build.tree == existing_build.tree
+ assert build.host == existing_build.host
+ assert build.compiler == existing_build.compiler
+ return existing_build
+ rev = build.revision_details()
+
+ new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
+ for name in os.listdir(self.path):
+ p = os.path.join(self.path, name)
+ if p.startswith(new_basename+"."):
+ os.remove(p)
os.link(build.basename+".log", new_basename+".log")
if os.path.exists(build.basename+".err"):
os.link(build.basename+".err", new_basename+".err")
- return Build(new_basename, build.tree, build.host, build.compiler, rev)
+ # They are supposed to be in unicode only but since comparision for sumary page depends on them
+ # the unicode conversion is done to avoid duplicates when running query in summary_builds
+ new_build = StormBuild(new_basename, unicode(build.tree), unicode(build.host), unicode(build.compiler), rev)
+ new_build.checksum = build.log_checksum()
+ new_build.upload_time = build.upload_time
+ new_build.status_str = build.status().__serialize__()
+ new_build.basename = new_basename
+ host = self.store.find(StormHost,
+ Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
+ assert host is not None, "Unable to find host %r" % build.host
+ new_build.host_id = host.id
+ self.store.add(new_build)
+ return new_build
+
+ def get_by_checksum(self, checksum):
+ from buildfarm.sqldb import Cast
+ result = self.store.find(StormBuild,
+ Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT")).order_by(Desc(StormBuild.upload_time))
+ ret = result.first()
+ if ret is None:
+ raise NoSuchBuildError(None, None, None, None)
+ return ret
+
+ def get_previous_build(self, tree, host, compiler, revision):
+ from buildfarm.sqldb import Cast
+ cur_build = self.get_build(tree, host, compiler, revision)
+
+ result = self.store.find(StormBuild,
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
+ Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
+ Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
+ StormBuild.id < cur_build.id)
+ result = result.order_by(Desc(StormBuild.id))
+ prev_build = result.first()
+ if prev_build is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return prev_build
+
+ def get_latest_build(self, tree, host, compiler):
+ result = self.store.find(StormBuild,
+ StormBuild.tree == tree,
+ StormBuild.host == host,
+ StormBuild.compiler == compiler)
+ result = result.order_by(Desc(StormBuild.id))
+ build = result.first()
+ if build is None:
+ raise NoSuchBuildError(tree, host, compiler)
+ return build
- def get_previous_revision(self, tree, host, compiler, revision):
- raise NoSuchBuildError(tree, host, compiler, revision)
- def get_latest_revision(self, tree, host, compiler):
- raise NoSuchBuildError(tree, host, compiler)
+class BuildDiff(object):
+ """Represents the difference between two builds."""
+
+ def __init__(self, tree, old, new):
+ self.tree = tree
+ self.old = old
+ self.new = new
+ self.new_rev = new.revision_details()
+ self.new_status = new.status()
+
+ self.old_rev = old.revision_details()
+ self.old_status = old.status()
+
+ def is_regression(self):
+ """Is there a regression in new build since old build?"""
+ return self.new_status.regressed_since(self.old_status)
+
+ def revisions(self):
+ """Returns the revisions introduced since old in new."""
+ branch = self.tree.get_branch()
+ return branch.log(from_rev=self.new.revision, exclude_revs=set([self.old.revision]))
git.samba.org / build-farm.git / blobdiff