buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     Tree,
  23     )
  24 from buildfarm.data import (
  25     Build,
  26     BuildResultStore,
  27     BuildStatus,
  28     NoSuchBuildError,
  29     )
  30 from buildfarm.hostdb import (
  31     Host,
  32     HostDatabase,
  33     HostAlreadyExists,
  34     NoSuchHost,
  35     )
  36
  37 import os
  38 try:
  39     from pysqlite2 import dbapi2 as sqlite3
  40 except ImportError:
  41     import sqlite3
  42 from storm.database import create_database
  43 from storm.expr import EXPR, FuncExpr, compile
  44 from storm.locals import Bool, Desc, Int, Unicode, RawStr
  45 from storm.store import Store
  46
  47
  48 class Cast(FuncExpr):
  49     __slots__ = ("column", "type")
  50     name = "CAST"
  51
  52     def __init__(self, column, type):
  53         self.column = column
  54         self.type = type
  55
  56 @compile.when(Cast)
  57 def compile_count(compile, cast, state):
  58     state.push("context", EXPR)
  59     column = compile(cast.column, state)
  60     state.pop()
  61     return "CAST(%s AS %s)" % (column, cast.type)
  62
  63
  64 class StormBuild(Build):
  65     __storm_table__ = "build"
  66
  67     id = Int(primary=True)
  68     tree = RawStr()
  69     revision = RawStr()
  70     host = RawStr()
  71     compiler = RawStr()
  72     checksum = RawStr()
  73     upload_time = Int(name="age")
  74     status_str = RawStr(name="status")
  75     basename = RawStr()
  76     host_id = Int()
  77
  78     def status(self):
  79         return BuildStatus.__deserialize__(self.status_str)
  80
  81     def revision_details(self):
  82         return self.revision
  83
  84     def log_checksum(self):
  85         return self.checksum
  86
  87     def remove(self):
  88         super(StormBuild, self).remove()
  89         Store.of(self).remove(self)
  90
  91     def remove_logs(self):
  92         super(StormBuild, self).remove_logs()
  93         self.basename = None
  94
  95
  96 class StormHost(Host):
  97     __storm_table__ = "host"
  98
  99     id = Int(primary=True)
 100     name = RawStr()
 101     owner_name = Unicode(name="owner")
 102     owner_email = Unicode()
 103     password = Unicode()
 104     ssh_access = Bool()
 105     fqdn = RawStr()
 106     platform = Unicode()
 107     permission = Unicode()
 108     last_dead_mail = Int()
 109     join_time = Int()
 110
 111     def _set_owner(self, value):
 112         if value is None:
 113             self.owner_name = None
 114             self.owner_email = None
 115         else:
 116             (self.owner_name, self.owner_email) = value
 117
 118     def _get_owner(self):
 119         if self.owner_name is None:
 120             return None
 121         else:
 122             return (self.owner_name, self.owner_email)
 123
 124     owner = property(_get_owner, _set_owner)
 125
 126
 127 class StormHostDatabase(HostDatabase):
 128
 129     def __init__(self, store=None):
 130         if store is None:
 131             self.store = memory_store()
 132         else:
 133             self.store = store
 134
 135     def createhost(self, name, platform=None, owner=None, owner_email=None,
 136             password=None, permission=None):
 137         """See `HostDatabase.createhost`."""
 138         newhost = StormHost(name, owner=owner, owner_email=owner_email,
 139                 password=password, permission=permission, platform=platform)
 140         try:
 141             self.store.add(newhost)
 142             self.store.flush()
 143         except sqlite3.IntegrityError:
 144             raise HostAlreadyExists(name)
 145         return newhost
 146
 147     def deletehost(self, name):
 148         """Remove a host."""
 149         self.store.remove(self[name])
 150
 151     def hosts(self):
 152         """Retrieve an iterable over all hosts."""
 153         return self.store.find(StormHost).order_by(StormHost.name)
 154
 155     def __getitem__(self, name):
 156         ret = self.store.find(StormHost, StormHost.name==name).one()
 157         if ret is None:
 158             raise NoSuchHost(name)
 159         return ret
 160
 161     def commit(self):
 162         self.store.commit()
 163
 164
 165 class StormCachingBuildResultStore(BuildResultStore):
 166
 167     def __init__(self, basedir, store=None):
 168         super(StormCachingBuildResultStore, self).__init__(basedir)
 169
 170         if store is None:
 171             store = memory_store()
 172
 173         self.store = store
 174
 175     def __contains__(self, build):
 176         return (self._get_by_checksum(build) is not None)
 177
 178     def get_previous_revision(self, tree, host, compiler, revision):
 179         result = self.store.find(StormBuild,
 180             StormBuild.tree == tree,
 181             StormBuild.host == host,
 182             StormBuild.compiler == compiler,
 183             Cast(StormBuild.revision, "TEXT") == revision)
 184         cur_build = result.any()
 185         if cur_build is None:
 186             raise NoSuchBuildError(tree, host, compiler, revision)
 187
 188         result = self.store.find(StormBuild,
 189             StormBuild.tree == tree,
 190             StormBuild.host == host,
 191             StormBuild.compiler == compiler,
 192             Cast(StormBuild.revision, "TEXT") != revision,
 193             StormBuild.id < cur_build.id)
 194         result = result.order_by(Desc(StormBuild.id))
 195         prev_build = result.first()
 196         if prev_build is None:
 197             raise NoSuchBuildError(tree, host, compiler, revision)
 198         return prev_build.revision
 199
 200     def get_latest_revision(self, tree, host, compiler):
 201         result = self.store.find(StormBuild,
 202             StormBuild.tree == tree,
 203             StormBuild.host == host,
 204             StormBuild.compiler == compiler)
 205         result = result.order_by(Desc(StormBuild.id))
 206         build = result.first()
 207         if build is None:
 208             raise NoSuchBuildError(tree, host, compiler)
 209         return build.revision
 210
 211     def _get_by_checksum(self, build):
 212         result = self.store.find(StormBuild,
 213             Cast(StormBuild.checksum, "TEXT") == build.log_checksum())
 214         return result.one()
 215
 216     def upload_build(self, build):
 217         existing_build = self._get_by_checksum(build)
 218         if existing_build is not None:
 219             # Already present
 220             assert build.tree == existing_build.tree
 221             assert build.host == existing_build.host
 222             assert build.compiler == existing_build.compiler
 223             return existing_build
 224         rev = build.revision_details()
 225         super(StormCachingBuildResultStore, self).upload_build(build)
 226         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 227         new_build = StormBuild(new_basename, build.tree, build.host,
 228             build.compiler, rev)
 229         new_build.checksum = build.log_checksum()
 230         new_build.upload_time = build.upload_time
 231         new_build.status_str = build.status().__serialize__()
 232         new_build.basename = new_basename
 233         self.store.add(new_build)
 234         return new_build
 235
 236     def get_old_builds(self, tree, host, compiler):
 237         result = self.store.find(StormBuild,
 238             StormBuild.tree == tree,
 239             StormBuild.host == host,
 240             StormBuild.compiler == compiler)
 241         return result.order_by(Desc(StormBuild.upload_time))
 242
 243     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 244         expr = [
 245             StormBuild.tree == tree,
 246             StormBuild.host == host,
 247             StormBuild.compiler == compiler,
 248             ]
 249         if revision is not None:
 250             expr.append(Cast(StormBuild.revision, "TEXT") == revision)
 251         if checksum is not None:
 252             expr.append(Cast(StormBuild.checksum, "TEXT") == checksum)
 253         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 254         ret = result.first()
 255         if ret is None:
 256             raise NoSuchBuildError(tree, host, compiler, revision)
 257         return ret
 258
 259
 260 def distinct_builds(builds):
 261     done = set()
 262     for build in builds:
 263         key = (build.tree, build.compiler, build.host)
 264         if key in done:
 265             continue
 266         done.add(key)
 267         yield build
 268
 269
 270 class StormCachingBuildFarm(BuildFarm):
 271
 272     def __init__(self, path=None, store=None, timeout=0.5):
 273         self.timeout = timeout
 274         self.store = store
 275         super(StormCachingBuildFarm, self).__init__(path)
 276
 277     def _get_store(self):
 278         if self.store is not None:
 279             return self.store
 280         db_path = os.path.join(self.path, "db", "hostdb.sqlite")
 281         db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 282         self.store = Store(db)
 283         setup_schema(self.store)
 284         return self.store
 285
 286     def _open_hostdb(self):
 287         return StormHostDatabase(self._get_store())
 288
 289     def _open_build_results(self):
 290         path = os.path.join(self.path, "data", "oldrevs")
 291         return StormCachingBuildResultStore(path, self._get_store())
 292
 293     def get_host_builds(self, host):
 294         result = self._get_store().find(StormBuild, StormBuild.host == host)
 295         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 296
 297     def get_tree_builds(self, tree):
 298         result = self._get_store().find(StormBuild, Cast(StormBuild.tree, "TEXT") == tree)
 299         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 300
 301     def get_last_builds(self):
 302         result = self._get_store().find(StormBuild)
 303         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 304
 305     def get_revision_builds(self, tree, revision=None):
 306         return self._get_store().find(StormBuild,
 307             Cast(StormBuild.tree, "TEXT") == tree,
 308             Cast(StormBuild.revision, "TEXT") == revision)
 309
 310     def commit(self):
 311         self.store.commit()
 312
 313
 314 class StormTree(Tree):
 315     __storm_table__ = "tree"
 316
 317     id = Int(primary=True)
 318     name = RawStr()
 319     scm = Int()
 320     branch = RawStr()
 321     subdir = RawStr()
 322     repo = RawStr()
 323     scm = RawStr()
 324
 325
 326 def setup_schema(db):
 327     db.execute("PRAGMA foreign_keys = 1;", noresult=True)
 328     db.execute("""
 329 CREATE TABLE IF NOT EXISTS host (
 330     id integer primary key autoincrement,
 331     name blob not null,
 332     owner text,
 333     owner_email text,
 334     password text,
 335     ssh_access int,
 336     fqdn text,
 337     platform text,
 338     permission text,
 339     last_dead_mail int,
 340     join_time int
 341 );""", noresult=True)
 342     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 343     db.execute("""
 344 CREATE TABLE IF NOT EXISTS build (
 345     id integer primary key autoincrement,
 346     tree blob not null,
 347     revision blob,
 348     host blob not null,
 349     host_id integer,
 350     compiler blob not null,
 351     checksum blob,
 352     age int,
 353     status blob,
 354     basename blob,
 355     FOREIGN KEY (host_id) REFERENCES host (id)
 356 );""", noresult=True)
 357     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 358     db.execute("""
 359 CREATE TABLE IF NOT EXISTS tree (
 360     id integer primary key autoincrement,
 361     name blob not null,
 362     scm int,
 363     branch blob,
 364     subdir blob,
 365     repo blob
 366     );""", noresult=True)
 367
 368
 369 def memory_store():
 370     db = create_database("sqlite:")
 371     store = Store(db)
 372     setup_schema(store)
 373     return store