2 # Simple database query script for the buildfarm
4 # Copyright (C) Andrew Tridgell <tridge@samba.org> 2001-2005
5 # Copyright (C) Andrew Bartlett <abartlet@samba.org> 2001
6 # Copyright (C) Vance Lankhaar <vance@samba.org> 2002-2005
7 # Copyright (C) Martin Pool <mbp@samba.org> 2001
8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007-2010
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation; either version 2 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License
21 # along with this program; if not, write to the Free Software
22 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 from cStringIO import StringIO
30 from storm.locals import Int, RawStr
31 from storm.store import Store
32 from storm.expr import Desc
36 def open_opt_compressed_file(path):
38 return bz2.BZ2File(path+".bz2", 'r')
40 return open(path, 'r')
45 def __init__(self, name):
50 class TestResult(object):
52 def __init__(self, build, test, result):
58 class BuildSummary(object):
60 def __init__(self, host, tree, compiler, revision, status):
63 self.compiler = compiler
64 self.revision = revision
68 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
71 class MissingRevisionInfo(Exception):
72 """Revision info could not be found in the build log."""
74 def __init__(self, build=None):
78 class LogFileMissing(Exception):
79 """Log file missing."""
82 class BuildStatus(object):
84 def __init__(self, stages=None, other_failures=None):
85 if stages is not None:
86 self.stages = [BuildStageResult(n, r) for (n, r) in stages]
89 if other_failures is not None:
90 self.other_failures = other_failures
92 self.other_failures = set()
96 if self.other_failures:
98 return not all([x.result == 0 for x in self.stages])
100 def __serialize__(self):
104 def __deserialize__(cls, text):
108 if self.other_failures:
109 return ",".join(self.other_failures)
110 return "/".join([str(x.result) for x in self.stages])
112 def broken_host(self):
113 if "disk full" in self.other_failures:
117 def regressed_since(self, older):
118 """Check if this build has regressed since another build."""
119 if "disk full" in self.other_failures:
121 if ("timeout" in self.other_failures and
122 "timeout" in older.other_failures):
123 # When the timeout happens exactly can differ slightly, so it's
124 # okay if the numbers are a bit different..
126 if ("panic" in self.other_failures and
127 not "panic" in older.other_failures):
128 # If this build introduced panics, then that's always worse.
130 if len(self.stages) < len(older.stages):
131 # Less stages completed
133 old_stages = dict(older.stages)
134 new_stages = dict(self.stages)
135 for name, new_result in new_stages.iteritems():
137 old_result = old_stages[name]
140 if new_result == old_result:
142 if new_result < 0 and old_result >= 0:
144 elif new_result >= 0 and old_result < 0:
147 return (abs(new_result) > abs(old_result))
150 def __cmp__(self, other):
151 other_extra = other.other_failures - self.other_failures
152 self_extra = self.other_failures - other.other_failures
153 # Give more importance to other failures
159 la = len(self.stages)
160 lb = len(other.stages)
166 return cmp(other.stages, self.stages)
169 return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
172 def check_dir_exists(kind, path):
173 if not os.path.isdir(path):
174 raise Exception("%s directory %s does not exist" % (kind, path))
177 def extract_phase_output(f):
180 re_action = re.compile("^ACTION (PASSED|FAILED):\s+(.*)$")
182 if l.startswith("Running action "):
183 name = l[len("Running action "):].strip()
186 m = re_action.match(l)
188 assert name == m.group(2).strip(), "%r != %r" % (name, m.group(2))
192 elif output is not None:
196 def extract_test_output(f):
197 for name, output in extract_phase_output(f):
203 def build_status_from_logs(log, err):
204 """get status of build"""
205 # FIXME: Perhaps also extract revision here?
213 re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
214 re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
217 if l.startswith("No space left on device"):
218 ret.other_failures.add("disk full")
220 if "Maximum time expired in timelimit" in l: # Ugh.
221 ret.other_failures.add("timeout")
223 if "maximum runtime exceeded" in l: # Ugh.
224 ret.other_failures.add("timeout")
226 if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
227 ret.other_failures.add("panic")
229 if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
232 if l.startswith("testsuite-success: "):
235 m = re_status.match(l)
237 stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
238 if m.group(1) == "TEST":
241 m = re_action.match(l)
242 if m and not test_seen:
243 if m.group(1) == "PASSED":
244 stages.append(BuildStageResult("TEST", 0))
246 stages.append(BuildStageResult("TEST", 1))
249 # Scan err file for specific errors
251 if "No space left on device" in l:
252 ret.other_failures.add("disk full")
255 if sr.name != "TEST":
258 if test_successes + test_failures == 0:
259 # No granular test output
260 return BuildStageResult("TEST", sr.result)
261 if sr.result == 1 and test_failures == 0:
262 ret.other_failures.add("inconsistent test result")
263 return BuildStageResult("TEST", -1)
264 return BuildStageResult("TEST", test_failures)
266 ret.stages = map(map_stage, stages)
270 def revision_from_log(log):
273 if l.startswith("BUILD COMMIT REVISION: "):
274 revid = l.split(":", 1)[1].strip()
276 raise MissingRevisionInfo()
280 class NoSuchBuildError(Exception):
281 """The build with the specified name does not exist."""
283 def __init__(self, tree, host, compiler, rev=None):
286 self.compiler = compiler
290 class NoTestOutput(Exception):
291 """The build did not have any associated test output."""
295 """A single build of a tree on a particular host using a particular compiler.
298 def __init__(self, basename, tree, host, compiler, rev=None):
299 self.basename = basename
302 self.compiler = compiler
305 def __cmp__(self, other):
307 (self.upload_time, self.revision, self.host, self.tree, self.compiler),
308 (other.upload_time, other.revision, other.host, other.tree, other.compiler))
310 def __eq__(self, other):
311 return (isinstance(other, Build) and
312 self.log_checksum() == other.log_checksum())
315 if self.revision is not None:
316 return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
318 return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
320 def remove_logs(self):
321 # In general, basename.log should *always* exist.
322 if os.path.exists(self.basename+".log"):
323 os.unlink(self.basename + ".log")
324 if os.path.exists(self.basename+".err"):
325 os.unlink(self.basename+".err")
331 def upload_time(self):
332 """get timestamp of build"""
333 st = os.stat("%s.log" % self.basename)
338 """get the age of build"""
339 return time.time() - self.upload_time
341 def read_subunit(self):
342 """read the test output as subunit"""
343 return StringIO("".join(extract_test_output(self.read_log())))
346 """read full log file"""
348 return open_opt_compressed_file(self.basename+".log")
350 raise LogFileMissing()
355 except LogFileMissing:
362 """read full err file"""
364 return open_opt_compressed_file(self.basename+".err")
369 def log_checksum(self):
372 return hashlib.sha1(f.read()).hexdigest()
377 revid = self.revision_details()
378 status = self.status()
379 return BuildSummary(self.host, self.tree, self.compiler, revid, status)
381 def revision_details(self):
382 """get the revision of build
388 return revision_from_log(f)
393 """get status of build
395 :return: tuple with build status
397 log = self.read_log()
399 err = self.read_err()
401 return build_status_from_logs(log, err)
408 """get status of build"""
409 file = self.read_err()
410 return len(file.readlines())
413 class UploadBuildResultStore(object):
415 def __init__(self, path):
416 """Open the database.
418 :param path: Build result base directory
422 def get_all_builds(self):
423 for name in os.listdir(self.path):
425 (build, tree, host, compiler, extension) = name.split(".")
428 if build != "build" or extension != "log":
430 yield self.get_build(tree, host, compiler)
432 def build_fname(self, tree, host, compiler):
433 return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
435 def has_host(self, host):
436 for name in os.listdir(self.path):
438 if name.split(".")[2] == host:
444 def get_build(self, tree, host, compiler):
445 basename = self.build_fname(tree, host, compiler)
446 logf = "%s.log" % basename
447 if not os.path.exists(logf):
448 raise NoSuchBuildError(tree, host, compiler)
449 return Build(basename, tree, host, compiler)
452 class StormBuild(Build):
453 __storm_table__ = "build"
455 id = Int(primary=True)
461 upload_time = Int(name="age")
462 status_str = RawStr(name="status")
469 return BuildStatus.__deserialize__(self.status_str)
471 def revision_details(self):
474 def log_checksum(self):
478 super(StormBuild, self).remove()
479 Store.of(self).remove(self)
481 def remove_logs(self):
482 super(StormBuild, self).remove_logs()
486 class BuildResultStore(object):
487 """The build farm build result database."""
489 def __init__(self, basedir, store=None):
490 from buildfarm.sqldb import memory_store
492 store = memory_store()
497 def __contains__(self, build):
499 self.get_by_checksum(build.log_checksum())
501 except NoSuchBuildError:
504 def get_build(self, tree, host, compiler, revision=None, checksum=None):
505 from buildfarm.sqldb import Cast
507 Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
508 Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
509 Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
511 if revision is not None:
512 expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
513 if checksum is not None:
514 expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
515 result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
518 raise NoSuchBuildError(tree, host, compiler, revision)
521 def build_fname(self, tree, host, compiler, rev):
522 """get the name of the build file"""
523 return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
525 def get_all_builds(self):
526 for l in os.listdir(self.path):
527 m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
532 compiler = m.group(3)
534 stat = os.stat(os.path.join(self.path, l))
535 # skip the current build
536 if stat.st_nlink == 2:
538 yield self.get_build(tree, host, compiler, rev)
540 def get_old_builds(self, tree, host, compiler):
541 result = self.store.find(StormBuild,
542 StormBuild.tree == tree,
543 StormBuild.host == host,
544 StormBuild.compiler == compiler)
545 return result.order_by(Desc(StormBuild.upload_time))
547 def upload_build(self, build):
548 from buildfarm.sqldb import Cast, StormHost
550 existing_build = self.get_by_checksum(build.log_checksum())
551 except NoSuchBuildError:
555 assert build.tree == existing_build.tree
556 assert build.host == existing_build.host
557 assert build.compiler == existing_build.compiler
558 return existing_build
559 rev = build.revision_details()
561 new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
562 for name in os.listdir(self.path):
563 p = os.path.join(self.path, name)
564 if p.startswith(new_basename+"."):
566 os.link(build.basename+".log", new_basename+".log")
567 if os.path.exists(build.basename+".err"):
568 os.link(build.basename+".err", new_basename+".err")
569 #they are supposed to be in unicode only but since comparision for sumary page depends on them the unicode conversion is done to avoid duplicates when running query in summary_builds
570 new_build = StormBuild(new_basename, unicode(build.tree), unicode(build.host), unicode(build.compiler), rev)
571 new_build.checksum = build.log_checksum()
572 new_build.upload_time = build.upload_time
573 new_build.status_str = build.status().__serialize__()
574 new_build.basename = new_basename
575 host = self.store.find(StormHost,
576 Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
577 assert host is not None, "Unable to find host %r" % build.host
578 new_build.host_id = host.id
579 self.store.add(new_build)
582 def get_by_checksum(self, checksum):
583 from buildfarm.sqldb import Cast
584 result = self.store.find(StormBuild,
585 Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT")).order_by(Desc(StormBuild.upload_time))
588 raise NoSuchBuildError(None, None, None, None)
591 def get_previous_build(self, tree, host, compiler, revision):
592 from buildfarm.sqldb import Cast
593 cur_build = self.get_build(tree, host, compiler, revision)
595 result = self.store.find(StormBuild,
596 Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
597 Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
598 Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
599 Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
600 StormBuild.id < cur_build.id)
601 result = result.order_by(Desc(StormBuild.id))
602 prev_build = result.first()
603 if prev_build is None:
604 raise NoSuchBuildError(tree, host, compiler, revision)
607 def get_latest_build(self, tree, host, compiler):
608 result = self.store.find(StormBuild,
609 StormBuild.tree == tree,
610 StormBuild.host == host,
611 StormBuild.compiler == compiler)
612 result = result.order_by(Desc(StormBuild.id))
613 build = result.first()
615 raise NoSuchBuildError(tree, host, compiler)
619 class BuildDiff(object):
620 """Represents the difference between two builds."""
622 def __init__(self, tree, old, new):
626 self.new_rev = new.revision_details()
627 self.new_status = new.status()
629 self.old_rev = old.revision_details()
630 self.old_status = old.status()
632 def is_regression(self):
633 """Is there a regression in new build since old build?"""
634 return self.new_status.regressed_since(self.old_status)
637 """Returns the revisions introduced since old in new."""
638 branch = self.tree.get_branch()
639 return branch.log(from_rev=self.new.revision, exclude_revs=set([self.old.revision]))