buildfarm/__init__.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2010
   5 #
   6 #   This program is free software; you can redistribute it and/or modify
   7 #   it under the terms of the GNU General Public License as published by
   8 #   the Free Software Foundation; either version 2 of the License, or
   9 #   (at your option) any later version.
  10 #
  11 #   This program is distributed in the hope that it will be useful,
  12 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #   GNU General Public License for more details.
  15 #
  16 #   You should have received a copy of the GNU General Public License
  17 #   along with this program; if not, write to the Free Software
  18 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19
  20 import ConfigParser
  21 import os
  22 import re
  23
  24 GIT_ROOT = "/data/git"
  25
  26
  27 class Tree(object):
  28     """A tree to build."""
  29
  30     def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
  31         self.name = name
  32         self.repo = repo
  33         self.scm = scm
  34         self.branch = branch
  35         self.subdir = subdir
  36         self.srcdir = srcdir
  37         self.scm = scm
  38
  39     def get_branch(self):
  40         if self.scm == "git":
  41             from buildfarm.history import GitBranch
  42             return GitBranch(os.path.join(GIT_ROOT, self.repo), self.branch)
  43         else:
  44             raise NotImplementedError(self.scm)
  45
  46     def __repr__(self):
  47         return "<%s %r>" % (self.__class__.__name__, self.name)
  48
  49
  50 def read_trees_from_conf(path):
  51     """Read trees from a configuration file.
  52
  53     :param path: tree path
  54     :return: Dictionary with trees
  55     """
  56     ret = {}
  57     cfp = ConfigParser.ConfigParser()
  58     cfp.read(path)
  59     for s in cfp.sections():
  60         ret[s] = Tree(name=s, **dict(cfp.items(s)))
  61     return ret
  62
  63
  64 def lcov_extract_percentage(f):
  65     """Extract the coverage percentage from the lcov file."""
  66     m = re.search('\<td class="headerItem".*?\>Code\&nbsp\;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', f.read())
  67     if m:
  68         return m.group(1)
  69     else:
  70         return None
  71
  72
  73 class BuildFarm(object):
  74
  75     LCOVHOST = "magni"
  76     OLDAGE = 60*60*4,
  77     DEADAGE = 60*60*24*4
  78
  79     def __init__(self, path=None):
  80         if path is None:
  81             path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
  82         self.path = path
  83         self.webdir = os.path.join(self.path, "web")
  84         if not os.path.isdir(path):
  85             raise Exception("web directory %s does not exist" % self.webdir)
  86         self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
  87         self.builds = self._open_build_results()
  88         self.upload_builds = self._open_upload_build_results()
  89         self.hostdb = self._open_hostdb()
  90         self.compilers = self._load_compilers()
  91         self.lcovdir = os.path.join(self.path, "lcov/data")
  92
  93     def __repr__(self):
  94         return "%s(%r)" % (self.__class__.__name__, self.path)
  95
  96     def _open_build_results(self):
  97         from buildfarm.build import BuildResultStore
  98         path = os.path.join(self.path, "data", "oldrevs")
  99         return BuildResultStore(path)
 100
 101     def _open_upload_build_results(self):
 102         from buildfarm.build import UploadBuildResultStore
 103         path = os.path.join(self.path, "data", "upload")
 104         return UploadBuildResultStore(path)
 105
 106     def _open_hostdb(self):
 107         from buildfarm import hostdb
 108         return hostdb.PlainTextHostDatabase.from_file(os.path.join(self.webdir, "hosts.list"))
 109
 110     def _load_compilers(self):
 111         from buildfarm import util
 112         return set(util.load_list(os.path.join(self.webdir, "compilers.list")))
 113
 114     def commit(self):
 115         pass
 116
 117     def lcov_status(self, tree):
 118         """get status of build"""
 119         from buildfarm.build import NoSuchBuildError
 120         file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
 121         try:
 122             lcov_html = open(file, 'r')
 123         except (OSError, IOError):
 124             # File does not exist
 125             raise NoSuchBuildError(tree, self.LCOVHOST, "lcov")
 126         try:
 127             return lcov_extract_percentage(lcov_html)
 128         finally:
 129             lcov_html.close()
 130
 131     def get_build(self, tree, host, compiler, rev=None, checksum=None):
 132         if rev is not None:
 133             return self.builds.get_build(tree, host, compiler, rev,
 134                 checksum=checksum)
 135         else:
 136             return self.upload_builds.get_build(tree, host, compiler)
 137
 138     def get_new_builds(self):
 139         hostnames = set([host.name for host in self.hostdb.hosts()])
 140         for build in self.upload_builds.get_all_builds():
 141             if (build.tree in self.trees and
 142                 build.compiler in self.compilers and
 143                 build.host in hostnames):
 144                 yield build
 145
 146     def get_last_builds(self):
 147         return sorted(self.get_new_builds(), reverse=True)
 148
 149     def get_tree_builds(self, tree):
 150         ret = []
 151         for build in self.builds.get_all_builds():
 152             if build.tree == tree:
 153                 ret.append(build)
 154         ret.sort(reverse=True)
 155         return ret
 156
 157     def host_last_build(self, host):
 158         return max([build.upload_time for build in self.get_host_builds(host)])
 159
 160     def get_host_builds(self, host):
 161         from buildfarm.build import NoSuchBuildError
 162         ret = []
 163         for compiler in self.compilers:
 164             for tree in sorted(self.trees.keys()):
 165                 try:
 166                     ret.append(self.get_build(tree, host, compiler))
 167                 except NoSuchBuildError:
 168                     pass
 169         ret.sort(reverse=True)
 170         return ret