# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-from buildfarm.sqldb import distinct_builds, Cast, StormBuild, setup_schema, StormCachingBuildResultStore, StormHostDatabase
+from buildfarm.sqldb import distinct_builds, Cast, StormBuild, setup_schema, StormHostDatabase
from buildfarm.tree import Tree
from storm.database import create_database
from storm.expr import Desc
def _open_build_results(self):
path = os.path.join(self.path, "data", "oldrevs")
- return StormCachingBuildResultStore(path, self._get_store())
+ from buildfarm.build import BuildResultStore
+ return BuildResultStore(path, self._get_store())
def _open_upload_build_results(self):
from buildfarm.build import UploadBuildResultStore
import hashlib
import os
import re
+from storm.locals import Int, RawStr
+from storm.store import Store
+from storm.expr import Desc
import time
return Build(basename, tree, host, compiler)
+class StormBuild(Build):
+ __storm_table__ = "build"
+
+ id = Int(primary=True)
+ tree = RawStr()
+ revision = RawStr()
+ host = RawStr()
+ compiler = RawStr()
+ checksum = RawStr()
+ upload_time = Int(name="age")
+ status_str = RawStr(name="status")
+ basename = RawStr()
+ host_id = Int()
+ tree_id = Int()
+ compiler_id = Int()
+
+ def status(self):
+ return BuildStatus.__deserialize__(self.status_str)
+
+ def revision_details(self):
+ return self.revision
+
+ def log_checksum(self):
+ return self.checksum
+
+ def remove(self):
+ super(StormBuild, self).remove()
+ Store.of(self).remove(self)
+
+ def remove_logs(self):
+ super(StormBuild, self).remove_logs()
+ self.basename = None
+
+
class BuildResultStore(object):
"""The build farm build result database."""
- def __init__(self, path):
- """Open the database.
+ def __init__(self, basedir, store=None):
+ from buildfarm.sqldb import memory_store
+ if store is None:
+ store = memory_store()
- :param path: Build result base directory
- """
- self.path = path
+ self.store = store
+ self.path = basedir
def __contains__(self, build):
try:
- if build.revision:
- rev = build.revision
- else:
- rev = build.revision_details()
- self.get_build(build.tree, build.host, build.compiler, rev)
+ self.get_by_checksum(build.log_checksum())
+ return True
except NoSuchBuildError:
return False
- else:
- return True
- def get_build(self, tree, host, compiler, rev, checksum=None):
- basename = self.build_fname(tree, host, compiler, rev)
- logf = "%s.log" % basename
- if not os.path.exists(logf):
- raise NoSuchBuildError(tree, host, compiler, rev)
- return Build(basename, tree, host, compiler, rev)
+ def get_build(self, tree, host, compiler, revision=None, checksum=None):
+ from buildfarm.sqldb import Cast
+ expr = [
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
+ Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
+ ]
+ if revision is not None:
+ expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
+ if checksum is not None:
+ expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
+ result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
+ ret = result.first()
+ if ret is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return ret
def build_fname(self, tree, host, compiler, rev):
"""get the name of the build file"""
yield self.get_build(tree, host, compiler, rev)
def get_old_builds(self, tree, host, compiler):
- """get a list of old builds and their status."""
- ret = []
- for build in self.get_all_builds():
- if build.tree == tree and build.host == host and build.compiler == compiler:
- ret.append(build)
- ret.sort(lambda a, b: cmp(a.upload_time, b.upload_time))
- return ret
+ result = self.store.find(StormBuild,
+ StormBuild.tree == tree,
+ StormBuild.host == host,
+ StormBuild.compiler == compiler)
+ return result.order_by(Desc(StormBuild.upload_time))
def upload_build(self, build):
+ from buildfarm.sqldb import Cast, StormHost
+ try:
+ existing_build = self.get_by_checksum(build.log_checksum())
+ except NoSuchBuildError:
+ pass
+ else:
+ # Already present
+ assert build.tree == existing_build.tree
+ assert build.host == existing_build.host
+ assert build.compiler == existing_build.compiler
+ return existing_build
rev = build.revision_details()
new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
os.link(build.basename+".log", new_basename+".log")
if os.path.exists(build.basename+".err"):
os.link(build.basename+".err", new_basename+".err")
- return Build(new_basename, build.tree, build.host, build.compiler, rev)
+ new_basename = self.build_fname(build.tree, build.host, build.compiler,
+ rev)
+ new_build = StormBuild(new_basename, build.tree, build.host,
+ build.compiler, rev)
+ new_build.checksum = build.log_checksum()
+ new_build.upload_time = build.upload_time
+ new_build.status_str = build.status().__serialize__()
+ new_build.basename = new_basename
+ host = self.store.find(StormHost,
+ Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
+ assert host is not None, "Unable to find host %r" % build.host
+ new_build.host_id = host.id
+ self.store.add(new_build)
+ return new_build
+
+ def get_by_checksum(self, checksum):
+ from buildfarm.sqldb import Cast
+ result = self.store.find(StormBuild,
+ Cast(StormBuild.checksum, "TEXT") == checksum)
+ ret = result.one()
+ if ret is None:
+ raise NoSuchBuildError(None, None, None, None)
+ return ret
def get_previous_revision(self, tree, host, compiler, revision):
- raise NoSuchBuildError(tree, host, compiler, revision)
+ from buildfarm.sqldb import Cast
+ cur_build = self.get_build(tree, host, compiler, revision)
+
+ result = self.store.find(StormBuild,
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
+ Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
+ Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
+ StormBuild.id < cur_build.id)
+ result = result.order_by(Desc(StormBuild.id))
+ prev_build = result.first()
+ if prev_build is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return prev_build.revision
def get_latest_revision(self, tree, host, compiler):
- raise NoSuchBuildError(tree, host, compiler)
+ result = self.store.find(StormBuild,
+ StormBuild.tree == tree,
+ StormBuild.host == host,
+ StormBuild.compiler == compiler)
+ result = result.order_by(Desc(StormBuild.id))
+ build = result.first()
+ if build is None:
+ raise NoSuchBuildError(tree, host, compiler)
+ return build.revision
Tree,
)
from buildfarm.build import (
- Build,
- BuildResultStore,
- BuildStatus,
- NoSuchBuildError,
+ StormBuild,
Test,
TestResult,
)
NoSuchHost,
)
-import os
+
try:
from pysqlite2 import dbapi2 as sqlite3
except ImportError:
return "CAST(%s AS %s)" % (column, cast.type)
-class StormBuild(Build):
- __storm_table__ = "build"
-
- id = Int(primary=True)
- tree = RawStr()
- revision = RawStr()
- host = RawStr()
- compiler = RawStr()
- checksum = RawStr()
- upload_time = Int(name="age")
- status_str = RawStr(name="status")
- basename = RawStr()
- host_id = Int()
- tree_id = Int()
- compiler_id = Int()
-
- def status(self):
- return BuildStatus.__deserialize__(self.status_str)
-
- def revision_details(self):
- return self.revision
-
- def log_checksum(self):
- return self.checksum
-
- def remove(self):
- super(StormBuild, self).remove()
- Store.of(self).remove(self)
-
- def remove_logs(self):
- super(StormBuild, self).remove_logs()
- self.basename = None
-
-
class StormHost(Host):
__storm_table__ = "host"
self.store.commit()
-class StormCachingBuildResultStore(BuildResultStore):
-
- def __init__(self, basedir, store=None):
- super(StormCachingBuildResultStore, self).__init__(basedir)
-
- if store is None:
- store = memory_store()
-
- self.store = store
-
- def get_by_checksum(self, checksum):
- result = self.store.find(StormBuild,
- Cast(StormBuild.checksum, "TEXT") == checksum)
- ret = result.one()
- if ret is None:
- raise NoSuchBuildError(None, None, None, None)
- return ret
-
- def __contains__(self, build):
- try:
- self.get_by_checksum(build.log_checksum())
- return True
- except NoSuchBuildError:
- return False
-
- def get_previous_revision(self, tree, host, compiler, revision):
- cur_build = self.get_build(tree, host, compiler, revision)
-
- result = self.store.find(StormBuild,
- Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
- Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
- Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
- Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
- StormBuild.id < cur_build.id)
- result = result.order_by(Desc(StormBuild.id))
- prev_build = result.first()
- if prev_build is None:
- raise NoSuchBuildError(tree, host, compiler, revision)
- return prev_build.revision
-
- def get_latest_revision(self, tree, host, compiler):
- result = self.store.find(StormBuild,
- StormBuild.tree == tree,
- StormBuild.host == host,
- StormBuild.compiler == compiler)
- result = result.order_by(Desc(StormBuild.id))
- build = result.first()
- if build is None:
- raise NoSuchBuildError(tree, host, compiler)
- return build.revision
-
- def upload_build(self, build):
- try:
- existing_build = self.get_by_checksum(build.log_checksum())
- except NoSuchBuildError:
- pass
- else:
- # Already present
- assert build.tree == existing_build.tree
- assert build.host == existing_build.host
- assert build.compiler == existing_build.compiler
- return existing_build
- rev = build.revision_details()
- super(StormCachingBuildResultStore, self).upload_build(build)
- new_basename = self.build_fname(build.tree, build.host, build.compiler,
- rev)
- new_build = StormBuild(new_basename, build.tree, build.host,
- build.compiler, rev)
- new_build.checksum = build.log_checksum()
- new_build.upload_time = build.upload_time
- new_build.status_str = build.status().__serialize__()
- new_build.basename = new_basename
- host = self.store.find(StormHost,
- Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
- assert host is not None, "Unable to find host %r" % build.host
- new_build.host_id = host.id
- self.store.add(new_build)
- return new_build
-
- def get_old_builds(self, tree, host, compiler):
- result = self.store.find(StormBuild,
- StormBuild.tree == tree,
- StormBuild.host == host,
- StormBuild.compiler == compiler)
- return result.order_by(Desc(StormBuild.upload_time))
-
- def get_build(self, tree, host, compiler, revision=None, checksum=None):
- expr = [
- Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
- Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
- Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
- ]
- if revision is not None:
- expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
- if checksum is not None:
- expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
- result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
- ret = result.first()
- if ret is None:
- raise NoSuchBuildError(tree, host, compiler, revision)
- return ret
-
-
def distinct_builds(builds):
done = set()
for build in builds:
f.close()
def write_hosts(self, hosts):
- f = open(os.path.join(self.path, "web", "hosts.list"), "w")
- try:
- for name, platform in hosts.iteritems():
- f.write("%s: %s\n" % (name, platform))
- finally:
- f.close()
+ for host in hosts:
+ self.buildfarm.hostdb.createhost(host)
def write_trees(self, trees):
f = open(os.path.join(self.path, "web", "trees.conf"), "w")
from buildfarm.tests import BuildFarmTestCase
-class NonexistantTests(unittest.TestCase):
-
- def test_nonexistant(self):
- self.assertRaises(
- Exception, BuildResultStore, "somedirthatdoesn'texist", None)
-
-
-class BuildResultStoreTestBase(object):
+class BuildResultStoreTests(BuildFarmTestCase):
def setUp(self):
+ super(BuildResultStoreTests, self).setUp()
+ self.buildfarm = BuildFarm(self.path)
self.write_compilers(["cc", "gcc"])
self.write_hosts({"charis": "Some machine",
"myhost": "Another host"})
+ self.x = self.buildfarm.builds
+
+ def test_get_previous_revision_result(self):
+ path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
+BUILD COMMIT REVISION: myrev
+""")
+ self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
+ path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
+BUILD COMMIT REVISION: myotherrev
+""")
+ self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
+ self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "unknown")
+ self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "myrev")
+ self.assertEquals("myrev", self.x.get_previous_revision("tdb", "charis", "cc", "myotherrev"))
+
+ def test_get_latest_revision(self):
+ path = self.create_mock_logfile("tdb", "charis", "cc", "22", contents="""
+BUILD COMMIT REVISION: myrev
+""")
+ self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
+ self.assertEquals("myrev", self.x.get_latest_revision("tdb", "charis", "cc"))
def test_build_fname(self):
self.assertEquals(
def setUp(self):
super(BuildFarmTests, self).setUp()
+ self.buildfarm = BuildFarm(self.path)
self.write_compilers(["cc"])
self.write_hosts({"myhost": "Fedora",
"charis": "Debian"})
self.write_trees({"trivial": {"scm": "git", "repo": "git://foo", "branch": "master"},
"other": {"scm": "git", "repo": "other.git", "branch": "HEAD"}})
+ self.buildfarm.commit()
self.x = BuildFarm(self.path)
def test_get_new_builds_empty(self):
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+from buildfarm import BuildFarm
from buildfarm.build import (
Build,
NoSuchBuildError,
)
from buildfarm.tests import BuildFarmTestCase
-from buildfarm.tests.test_build import BuildResultStoreTestBase
from buildfarm.tests.test_hostdb import HostDatabaseTests
from buildfarm.sqldb import (
StormHostDatabase,
self.db = StormHostDatabase()
-class StormCachingBuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
- def setUp(self):
- BuildFarmTestCase.setUp(self)
- BuildResultStoreTestBase.setUp(self)
- self.x = self.buildfarm.builds
-
- def test_get_previous_revision_result(self):
- path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
-BUILD COMMIT REVISION: myrev
-""")
- self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
- path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
-BUILD COMMIT REVISION: myotherrev
-""")
- self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
- self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "unknown")
- self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "myrev")
- self.assertEquals("myrev", self.x.get_previous_revision("tdb", "charis", "cc", "myotherrev"))
-
- def test_get_latest_revision(self):
- path = self.create_mock_logfile("tdb", "charis", "cc", "22", contents="""
-BUILD COMMIT REVISION: myrev
-""")
- self.x.upload_build(Build(path[:-4], "tdb", "charis", "cc"))
- self.assertEquals("myrev", self.x.get_latest_revision("tdb", "charis", "cc"))
parser.add_option("--port", help="Port to listen on [localhost:8000]",
default="localhost:8000", type=str)
opts, args = parser.parse_args()
- from buildfarm.sqldb import BuildFarm
+ from buildfarm import BuildFarm
buildfarm = BuildFarm()
buildApp = BuildFarmApp(buildfarm)
from wsgiref.simple_server import make_server
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from buildfarm.sqldb import BuildFarm
+from buildfarm import BuildFarm
import optparse
import sys
MissingRevisionInfo,
NoSuchBuildError,
)
-from buildfarm.sqldb import BuildFarm
+from buildfarm import BuildFarm
from buildfarm.web import build_uri
from email.mime.text import MIMEText
import logging
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from buildfarm.sqldb import BuildFarm
+from buildfarm import BuildFarm
from buildfarm.web import host_uri
import optparse
import smtplib
git.samba.org / build-farm.git / commitdiff