buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     Tree,
  23     )
  24 from buildfarm.data import (
  25     Build,
  26     BuildResultStore,
  27     BuildStatus,
  28     NoSuchBuildError,
  29     )
  30 from buildfarm.hostdb import (
  31     Host,
  32     HostDatabase,
  33     HostAlreadyExists,
  34     NoSuchHost,
  35     )
  36
  37 import os
  38 try:
  39     from pysqlite2 import dbapi2 as sqlite3
  40 except ImportError:
  41     import sqlite3
  42 from storm.database import create_database
  43 from storm.locals import Bool, Desc, Int, Unicode, RawStr
  44 from storm.store import Store
  45
  46
  47 class StormBuild(Build):
  48     __storm_table__ = "build"
  49
  50     id = Int(primary=True)
  51     tree = RawStr()
  52     revision = RawStr()
  53     host = RawStr()
  54     compiler = RawStr()
  55     checksum = RawStr()
  56     upload_time = Int(name="age")
  57     commit = Unicode()
  58     status_str = RawStr(name="status")
  59     commit_revision = RawStr()
  60     basename = RawStr()
  61     host_id = Int()
  62
  63     def status(self):
  64         return BuildStatus.__deserialize__(self.status_str)
  65
  66     def revision_details(self):
  67         return (self.revision, None)
  68
  69     def log_checksum(self):
  70         return self.checksum
  71
  72     def remove(self):
  73         super(StormBuild, self).remove()
  74         Store.of(self).remove(self)
  75
  76
  77 class StormHost(Host):
  78     __storm_table__ = "host"
  79
  80     id = Int(primary=True)
  81     name = RawStr()
  82     owner_name = Unicode(name="owner")
  83     owner_email = Unicode()
  84     password = Unicode()
  85     ssh_access = Bool()
  86     fqdn = RawStr()
  87     platform = Unicode()
  88     permission = Unicode()
  89     last_dead_mail = Int()
  90     join_time = Int()
  91
  92     def _set_owner(self, value):
  93         if value is None:
  94             self.owner_name = None
  95             self.owner_email = None
  96         else:
  97             (self.owner_name, self.owner_email) = value
  98
  99     def _get_owner(self):
 100         if self.owner_name is None:
 101             return None
 102         else:
 103             return (self.owner_name, self.owner_email)
 104
 105     owner = property(_get_owner, _set_owner)
 106
 107
 108 class StormHostDatabase(HostDatabase):
 109
 110     def __init__(self, store=None):
 111         if store is None:
 112             self.store = memory_store()
 113         else:
 114             self.store = store
 115
 116     def createhost(self, name, platform=None, owner=None, owner_email=None,
 117             password=None, permission=None):
 118         """See `HostDatabase.createhost`."""
 119         newhost = StormHost(name, owner=owner, owner_email=owner_email,
 120                 password=password, permission=permission, platform=platform)
 121         try:
 122             self.store.add(newhost)
 123             self.store.flush()
 124         except sqlite3.IntegrityError:
 125             raise HostAlreadyExists(name)
 126         return newhost
 127
 128     def deletehost(self, name):
 129         """Remove a host."""
 130         self.store.remove(self[name])
 131
 132     def hosts(self):
 133         """Retrieve an iterable over all hosts."""
 134         return self.store.find(StormHost).order_by(StormHost.name)
 135
 136     def __getitem__(self, name):
 137         ret = self.store.find(StormHost, StormHost.name==name).one()
 138         if ret is None:
 139             raise NoSuchHost(name)
 140         return ret
 141
 142     def commit(self):
 143         self.store.commit()
 144
 145
 146 class StormCachingBuildResultStore(BuildResultStore):
 147
 148     def __init__(self, basedir, store=None):
 149         super(StormCachingBuildResultStore, self).__init__(basedir)
 150
 151         if store is None:
 152             store = memory_store()
 153
 154         self.store = store
 155
 156     def __contains__(self, build):
 157         return (self._get_by_checksum(build) is not None)
 158
 159     def get_previous_revision(self, tree, host, compiler, revision):
 160         result = self.store.find(StormBuild,
 161             StormBuild.tree == tree,
 162             StormBuild.host == host,
 163             StormBuild.compiler == compiler,
 164             StormBuild.revision == revision)
 165         cur_build = result.any()
 166         if cur_build is None:
 167             raise NoSuchBuildError(tree, host, compiler, revision)
 168
 169         result = self.store.find(StormBuild,
 170             StormBuild.tree == tree,
 171             StormBuild.host == host,
 172             StormBuild.compiler == compiler,
 173             StormBuild.revision != revision,
 174             StormBuild.id < cur_build.id)
 175         result = result.order_by(Desc(StormBuild.id))
 176         prev_build = result.first()
 177         if prev_build is None:
 178             raise NoSuchBuildError(tree, host, compiler, revision)
 179         return prev_build.revision
 180
 181     def get_latest_revision(self, tree, host, compiler):
 182         result = self.store.find(StormBuild,
 183             StormBuild.tree == tree,
 184             StormBuild.host == host,
 185             StormBuild.compiler == compiler)
 186         result = result.order_by(Desc(StormBuild.id))
 187         build = result.first()
 188         if build is None:
 189             raise NoSuchBuildError(tree, host, compiler)
 190         return build.revision
 191
 192     def _get_by_checksum(self, build):
 193         result = self.store.find(StormBuild,
 194             StormBuild.checksum == build.log_checksum())
 195         return result.one()
 196
 197     def upload_build(self, build):
 198         existing_build = self._get_by_checksum(build)
 199         if existing_build is not None:
 200             # Already present
 201             assert build.tree == existing_build.tree
 202             assert build.host == existing_build.host
 203             assert build.compiler == existing_build.compiler
 204             return existing_build
 205         rev, timestamp = build.revision_details()
 206         super(StormCachingBuildResultStore, self).upload_build(build)
 207         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 208         new_build = StormBuild(new_basename, build.tree, build.host,
 209             build.compiler, rev)
 210         new_build.checksum = build.log_checksum()
 211         new_build.upload_time = build.upload_time
 212         new_build.status_str = build.status().__serialize__()
 213         new_build.basename = new_basename
 214         self.store.add(new_build)
 215         return new_build
 216
 217     def get_old_revs(self, tree, host, compiler):
 218         return self.store.find(StormBuild,
 219             StormBuild.tree == tree,
 220             StormBuild.host == host,
 221             StormBuild.compiler == compiler).order_by(Desc(StormBuild.upload_time))
 222
 223
 224 class StormCachingBuildFarm(BuildFarm):
 225
 226     def __init__(self, path=None, store=None, timeout=0.5):
 227         self.timeout = timeout
 228         self.store = store
 229         super(StormCachingBuildFarm, self).__init__(path)
 230
 231     def _get_store(self):
 232         if self.store is not None:
 233             return self.store
 234         db_path = os.path.join(self.path, "db", "hostdb.sqlite")
 235         db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 236         self.store = Store(db)
 237         setup_schema(self.store)
 238         return self.store
 239
 240     def _open_hostdb(self):
 241         return StormHostDatabase(self._get_store())
 242
 243     def _open_build_results(self):
 244         return StormCachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
 245             self._get_store())
 246
 247     def get_host_builds(self, host):
 248         return self._get_store().find(StormBuild,
 249             StormBuild.host==host).group_by(StormBuild.compiler, StormBuild.tree)
 250
 251     def get_tree_builds(self, tree):
 252         result = self._get_store().find(StormBuild, StormBuild.tree==tree)
 253         return result.order_by(Desc(StormBuild.upload_time))
 254
 255     def get_last_builds(self):
 256         return self._get_store().find(StormBuild).group_by(
 257             StormBuild.tree, StormBuild.compiler, StormBuild.host).order_by(
 258                 Desc(StormBuild.upload_time))
 259
 260     def commit(self):
 261         self.store.commit()
 262
 263
 264 class StormTree(Tree):
 265     __storm_table__ = "tree"
 266
 267     id = Int(primary=True)
 268     name = RawStr()
 269     scm = Int()
 270     branch = RawStr()
 271     subdir = RawStr()
 272     repo = RawStr()
 273     scm = RawStr()
 274
 275
 276 def setup_schema(db):
 277     db.execute("PRAGMA foreign_keys = 1;", noresult=True)
 278     db.execute("""
 279 CREATE TABLE IF NOT EXISTS host (
 280     id integer primary key autoincrement,
 281     name blob not null,
 282     owner text,
 283     owner_email text,
 284     password text,
 285     ssh_access int,
 286     fqdn text,
 287     platform text,
 288     permission text,
 289     last_dead_mail int,
 290     join_time int
 291 );""", noresult=True)
 292     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 293     db.execute("""
 294 CREATE TABLE IF NOT EXISTS build (
 295     id integer primary key autoincrement,
 296     tree blob not null,
 297     revision blob,
 298     host blob not null,
 299     host_id integer,
 300     compiler blob not null,
 301     checksum blob,
 302     age int,
 303     status blob,
 304     basename blob,
 305     FOREIGN KEY (host_id) REFERENCES host (id)
 306 );""", noresult=True)
 307     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 308     db.execute("""
 309 CREATE TABLE IF NOT EXISTS tree (
 310     id integer primary key autoincrement,
 311     name blob not null,
 312     scm int,
 313     branch blob,
 314     subdir blob,
 315     repo blob
 316     );""", noresult=True)
 317
 318
 319 def memory_store():
 320     db = create_database("sqlite:")
 321     store = Store(db)
 322     setup_schema(store)
 323     return store