buildfarm/__init__.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2010
   5 #
   6 #   This program is free software; you can redistribute it and/or modify
   7 #   it under the terms of the GNU General Public License as published by
   8 #   the Free Software Foundation; either version 2 of the License, or
   9 #   (at your option) any later version.
  10 #
  11 #   This program is distributed in the hope that it will be useful,
  12 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #   GNU General Public License for more details.
  15 #
  16 #   You should have received a copy of the GNU General Public License
  17 #   along with this program; if not, write to the Free Software
  18 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19
  20 import ConfigParser
  21 import os
  22 import re
  23
  24
  25 class Tree(object):
  26     """A tree to build."""
  27
  28     def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
  29         self.name = name
  30         self.repo = repo
  31         self.scm = scm
  32         self.branch = branch
  33         self.subdir = subdir
  34         self.srcdir = srcdir
  35         self.scm = scm
  36
  37     def get_branch(self):
  38         from buildfarm.history import GitBranch
  39         return GitBranch(self.repo, self.branch)
  40
  41     def __repr__(self):
  42         return "<%s %r>" % (self.__class__.__name__, self.name)
  43
  44
  45 def read_trees_from_conf(path):
  46     """Read trees from a configuration file.
  47
  48     :param path: tree path
  49     :return: Dictionary with trees
  50     """
  51     ret = {}
  52     cfp = ConfigParser.ConfigParser()
  53     cfp.read(path)
  54     for s in cfp.sections():
  55         ret[s] = Tree(name=s, **dict(cfp.items(s)))
  56     return ret
  57
  58
  59 def lcov_extract_percentage(text):
  60     """Extract the coverage percentage from the lcov file."""
  61     m = re.search('\<td class="headerItem".*?\>Code\&nbsp\;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', text)
  62     if m:
  63         return m.group(1)
  64     else:
  65         return None
  66
  67
  68 class BuildFarm(object):
  69
  70     LCOVHOST = "magni"
  71     OLDAGE = 60*60*4,
  72     DEADAGE = 60*60*24*4
  73
  74     def __init__(self, path=None):
  75         if path is None:
  76             path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
  77         self.path = path
  78         self.webdir = os.path.join(self.path, "web")
  79         if not os.path.isdir(path):
  80             raise Exception("web directory %s does not exist" % self.webdir)
  81         self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
  82         self.builds = self._open_build_results()
  83         self.upload_builds = self._open_upload_build_results()
  84         self.hostdb = self._open_hostdb()
  85         self.compilers = self._load_compilers()
  86         self.lcovdir = os.path.join(self.path, "lcov/data")
  87
  88     def __repr__(self):
  89         return "%s(%r)" % (self.__class__.__name__, self.path)
  90
  91     def _open_build_results(self):
  92         from buildfarm import data
  93         return data.BuildResultStore(os.path.join(self.path, "data", "oldrevs"))
  94
  95     def _open_upload_build_results(self):
  96         from buildfarm import data
  97         return data.UploadBuildResultStore(os.path.join(self.path, "data", "upload"))
  98
  99     def _open_hostdb(self):
 100         from buildfarm import hostdb
 101         return hostdb.HostDatabase(
 102             os.path.join(self.path, "hostdb.sqlite"))
 103
 104     def _load_compilers(self):
 105         from buildfarm import util
 106         return util.load_list(os.path.join(self.webdir, "compilers.list"))
 107
 108     def lcov_status(self, tree):
 109         """get status of build"""
 110         from buildfarm import data, util
 111         file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
 112         try:
 113             lcov_html = util.FileLoad(file)
 114         except (OSError, IOError):
 115             # File does not exist
 116             raise data.NoSuchBuildError(tree, self.LCOVHOST, "lcov")
 117
 118         perc = lcov_extract_percentage(lcov_html)
 119         if perc is None:
 120             ret = ""
 121         else:
 122             ret = perc
 123         return perc
 124
 125     def get_build(self, tree, host, compiler, rev=None):
 126         if rev:
 127             return self.builds.get_build(tree, host, compiler, rev)
 128         else:
 129             return self.upload_builds.get_build(tree, host, compiler)
 130
 131     def get_new_builds(self):
 132         from buildfarm import data
 133         for host in self.hostdb.hosts():
 134             for tree in self.trees:
 135                 for compiler in self.compilers:
 136                     # By building the log file name this way, using only the list of
 137                     # hosts, trees and compilers as input, we ensure we
 138                     # control the inputs
 139                     try:
 140                         yield self.upload_builds.get_build(tree, host.name, compiler)
 141                     except data.NoSuchBuildError:
 142                         continue
 143
 144
 145 class CachingBuildFarm(BuildFarm):
 146
 147     def __init__(self, path=None, readonly=False, cachedirname=None):
 148         self._cachedirname = cachedirname
 149         self.readonly = readonly
 150         super(CachingBuildFarm, self).__init__(path)
 151
 152     def _get_cachedir(self):
 153         if self._cachedirname is not None:
 154             return os.path.join(self.path, self._cachedirname)
 155         else:
 156             return os.path.join(self.path, "cache")
 157
 158     def _open_build_results(self):
 159         from buildfarm import data
 160         return data.CachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
 161                 self._get_cachedir(), readonly=self.readonly)
 162
 163     def _open_upload_build_results(self):
 164         from buildfarm import data
 165         return data.CachingUploadBuildResultStore(os.path.join(self.path, "data", "upload"),
 166                 self._get_cachedir(), readonly=self.readonly)
 167
 168     def lcov_status(self, tree):
 169         """get status of build"""
 170         from buildfarm import data, util
 171         cachefile = os.path.join(self._get_cachedir(),
 172                                     "lcov.%s.%s.status" % (self.LCOVHOST, tree))
 173         file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
 174         try:
 175             st1 = os.stat(file)
 176         except OSError:
 177             # File does not exist
 178             raise data.NoSuchBuildError(tree, self.LCOVHOST, "lcov")
 179         try:
 180             st2 = os.stat(cachefile)
 181         except OSError:
 182             # file does not exist
 183             st2 = None
 184
 185         if st2 and st1.st_ctime <= st2.st_mtime:
 186             ret = util.FileLoad(cachefile)
 187             if ret == "":
 188                 return None
 189             return ret
 190
 191         perc = super(CachingBuildFarm, self).lcov_status(tree)
 192         if not self.readonly:
 193             util.FileSave(cachefile, perc)
 194         return perc