buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     Tree,
  23     )
  24 from buildfarm.data import (
  25     Build,
  26     BuildResultStore,
  27     BuildStatus,
  28     NoSuchBuildError,
  29     )
  30 from buildfarm.hostdb import (
  31     Host,
  32     HostDatabase,
  33     HostAlreadyExists,
  34     NoSuchHost,
  35     )
  36
  37 import os
  38 try:
  39     from pysqlite2 import dbapi2 as sqlite3
  40 except ImportError:
  41     import sqlite3
  42 from storm.database import create_database
  43 from storm.locals import Bool, Desc, Int, Unicode, RawStr
  44 from storm.store import Store
  45
  46
  47 class StormBuild(Build):
  48     __storm_table__ = "build"
  49
  50     id = Int(primary=True)
  51     tree = RawStr()
  52     revision = RawStr()
  53     host = RawStr()
  54     compiler = RawStr()
  55     checksum = RawStr()
  56     upload_time = Int(name="age")
  57     status_str = RawStr(name="status")
  58     basename = RawStr()
  59     host_id = Int()
  60
  61     def status(self):
  62         return BuildStatus.__deserialize__(self.status_str)
  63
  64     def revision_details(self):
  65         return (self.revision, None)
  66
  67     def log_checksum(self):
  68         return self.checksum
  69
  70     def remove(self):
  71         super(StormBuild, self).remove()
  72         Store.of(self).remove(self)
  73
  74     def remove_logs(self):
  75         super(StormBuild, self).remove_logs()
  76         self.basename = None
  77
  78
  79 class StormHost(Host):
  80     __storm_table__ = "host"
  81
  82     id = Int(primary=True)
  83     name = RawStr()
  84     owner_name = Unicode(name="owner")
  85     owner_email = Unicode()
  86     password = Unicode()
  87     ssh_access = Bool()
  88     fqdn = RawStr()
  89     platform = Unicode()
  90     permission = Unicode()
  91     last_dead_mail = Int()
  92     join_time = Int()
  93
  94     def _set_owner(self, value):
  95         if value is None:
  96             self.owner_name = None
  97             self.owner_email = None
  98         else:
  99             (self.owner_name, self.owner_email) = value
 100
 101     def _get_owner(self):
 102         if self.owner_name is None:
 103             return None
 104         else:
 105             return (self.owner_name, self.owner_email)
 106
 107     owner = property(_get_owner, _set_owner)
 108
 109
 110 class StormHostDatabase(HostDatabase):
 111
 112     def __init__(self, store=None):
 113         if store is None:
 114             self.store = memory_store()
 115         else:
 116             self.store = store
 117
 118     def createhost(self, name, platform=None, owner=None, owner_email=None,
 119             password=None, permission=None):
 120         """See `HostDatabase.createhost`."""
 121         newhost = StormHost(name, owner=owner, owner_email=owner_email,
 122                 password=password, permission=permission, platform=platform)
 123         try:
 124             self.store.add(newhost)
 125             self.store.flush()
 126         except sqlite3.IntegrityError:
 127             raise HostAlreadyExists(name)
 128         return newhost
 129
 130     def deletehost(self, name):
 131         """Remove a host."""
 132         self.store.remove(self[name])
 133
 134     def hosts(self):
 135         """Retrieve an iterable over all hosts."""
 136         return self.store.find(StormHost).order_by(StormHost.name)
 137
 138     def __getitem__(self, name):
 139         ret = self.store.find(StormHost, StormHost.name==name).one()
 140         if ret is None:
 141             raise NoSuchHost(name)
 142         return ret
 143
 144     def commit(self):
 145         self.store.commit()
 146
 147
 148 class StormCachingBuildResultStore(BuildResultStore):
 149
 150     def __init__(self, basedir, store=None):
 151         super(StormCachingBuildResultStore, self).__init__(basedir)
 152
 153         if store is None:
 154             store = memory_store()
 155
 156         self.store = store
 157
 158     def __contains__(self, build):
 159         return (self._get_by_checksum(build) is not None)
 160
 161     def get_previous_revision(self, tree, host, compiler, revision):
 162         result = self.store.find(StormBuild,
 163             StormBuild.tree == tree,
 164             StormBuild.host == host,
 165             StormBuild.compiler == compiler,
 166             StormBuild.revision == revision)
 167         cur_build = result.any()
 168         if cur_build is None:
 169             raise NoSuchBuildError(tree, host, compiler, revision)
 170
 171         result = self.store.find(StormBuild,
 172             StormBuild.tree == tree,
 173             StormBuild.host == host,
 174             StormBuild.compiler == compiler,
 175             StormBuild.revision != revision,
 176             StormBuild.id < cur_build.id)
 177         result = result.order_by(Desc(StormBuild.id))
 178         prev_build = result.first()
 179         if prev_build is None:
 180             raise NoSuchBuildError(tree, host, compiler, revision)
 181         return prev_build.revision
 182
 183     def get_latest_revision(self, tree, host, compiler):
 184         result = self.store.find(StormBuild,
 185             StormBuild.tree == tree,
 186             StormBuild.host == host,
 187             StormBuild.compiler == compiler)
 188         result = result.order_by(Desc(StormBuild.id))
 189         build = result.first()
 190         if build is None:
 191             raise NoSuchBuildError(tree, host, compiler)
 192         return build.revision
 193
 194     def _get_by_checksum(self, build):
 195         result = self.store.find(StormBuild,
 196             StormBuild.checksum == build.log_checksum())
 197         return result.one()
 198
 199     def upload_build(self, build):
 200         existing_build = self._get_by_checksum(build)
 201         if existing_build is not None:
 202             # Already present
 203             assert build.tree == existing_build.tree
 204             assert build.host == existing_build.host
 205             assert build.compiler == existing_build.compiler
 206             return existing_build
 207         rev, timestamp = build.revision_details()
 208         super(StormCachingBuildResultStore, self).upload_build(build)
 209         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 210         new_build = StormBuild(new_basename, build.tree, build.host,
 211             build.compiler, rev)
 212         new_build.checksum = build.log_checksum()
 213         new_build.upload_time = build.upload_time
 214         new_build.status_str = build.status().__serialize__()
 215         new_build.basename = new_basename
 216         self.store.add(new_build)
 217         return new_build
 218
 219     def get_old_builds(self, tree, host, compiler):
 220         result = self.store.find(StormBuild,
 221             StormBuild.tree == tree,
 222             StormBuild.host == host,
 223             StormBuild.compiler == compiler)
 224         return result.order_by(Desc(StormBuild.upload_time))
 225
 226     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 227         expr = [
 228             StormBuild.tree == tree,
 229             StormBuild.host == host,
 230             StormBuild.compiler == compiler,
 231             ]
 232         if revision is not None:
 233             expr.append(StormBuild.revision == revision)
 234         if checksum is not None:
 235             expr.append(StormBuild.checksum == checksum)
 236         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 237         ret = result.first()
 238         if ret is None:
 239             raise NoSuchBuildError(tree, host, compiler, revision)
 240         return ret
 241
 242
 243 def distinct_builds(builds):
 244     done = set()
 245     for build in builds:
 246         key = (build.tree, build.compiler, build.host)
 247         if key in done:
 248             continue
 249         done.add(key)
 250         yield build
 251
 252
 253 class StormCachingBuildFarm(BuildFarm):
 254
 255     def __init__(self, path=None, store=None, timeout=0.5):
 256         self.timeout = timeout
 257         self.store = store
 258         super(StormCachingBuildFarm, self).__init__(path)
 259
 260     def _get_store(self):
 261         if self.store is not None:
 262             return self.store
 263         db_path = os.path.join(self.path, "db", "hostdb.sqlite")
 264         db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 265         self.store = Store(db)
 266         setup_schema(self.store)
 267         return self.store
 268
 269     def _open_hostdb(self):
 270         return StormHostDatabase(self._get_store())
 271
 272     def _open_build_results(self):
 273         path = os.path.join(self.path, "data", "oldrevs")
 274         return StormCachingBuildResultStore(path, self._get_store())
 275
 276     def get_host_builds(self, host):
 277         result = self._get_store().find(StormBuild, StormBuild.host == host)
 278         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 279
 280     def get_tree_builds(self, tree):
 281         result = self._get_store().find(StormBuild, StormBuild.tree == tree)
 282         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 283
 284     def get_last_builds(self):
 285         result = self._get_store().find(StormBuild)
 286         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 287
 288     def commit(self):
 289         self.store.commit()
 290
 291
 292 class StormTree(Tree):
 293     __storm_table__ = "tree"
 294
 295     id = Int(primary=True)
 296     name = RawStr()
 297     scm = Int()
 298     branch = RawStr()
 299     subdir = RawStr()
 300     repo = RawStr()
 301     scm = RawStr()
 302
 303
 304 def setup_schema(db):
 305     db.execute("PRAGMA foreign_keys = 1;", noresult=True)
 306     db.execute("""
 307 CREATE TABLE IF NOT EXISTS host (
 308     id integer primary key autoincrement,
 309     name blob not null,
 310     owner text,
 311     owner_email text,
 312     password text,
 313     ssh_access int,
 314     fqdn text,
 315     platform text,
 316     permission text,
 317     last_dead_mail int,
 318     join_time int
 319 );""", noresult=True)
 320     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 321     db.execute("""
 322 CREATE TABLE IF NOT EXISTS build (
 323     id integer primary key autoincrement,
 324     tree blob not null,
 325     revision blob,
 326     host blob not null,
 327     host_id integer,
 328     compiler blob not null,
 329     checksum blob,
 330     age int,
 331     status blob,
 332     basename blob,
 333     FOREIGN KEY (host_id) REFERENCES host (id)
 334 );""", noresult=True)
 335     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 336     db.execute("""
 337 CREATE TABLE IF NOT EXISTS tree (
 338     id integer primary key autoincrement,
 339     name blob not null,
 340     scm int,
 341     branch blob,
 342     subdir blob,
 343     repo blob
 344     );""", noresult=True)
 345
 346
 347 def memory_store():
 348     db = create_database("sqlite:")
 349     store = Store(db)
 350     setup_schema(store)
 351     return store