Scan for builds then verify, rather than probing.

diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index a608f7967efb10420a2542197fa9f2be629103f0..40522f73885661baebe0d7946bed95ca180dce12 100644 (file)
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -103,7 +103,7 @@ class BuildFarm(object):
 
     def _load_compilers(self):
         from buildfarm import util
-        return util.load_list(os.path.join(self.webdir, "compilers.list"))
+        return set(util.load_list(os.path.join(self.webdir, "compilers.list")))
 
     def lcov_status(self, tree):
         """get status of build"""
@@ -129,17 +129,10 @@ class BuildFarm(object):
             return self.upload_builds.get_build(tree, host, compiler)
 
     def get_new_builds(self):
-        from buildfarm import data
-        for host in self.hostdb.hosts():
-            for tree in self.trees:
-                for compiler in self.compilers:
-                    # By building the log file name this way, using only the list of
-                    # hosts, trees and compilers as input, we ensure we
-                    # control the inputs
-                    try:
-                        yield self.upload_builds.get_build(tree, host.name, compiler)
-                    except data.NoSuchBuildError:
-                        continue
+        hosts = set([host.name for host in self.hostdb.hosts()])
+        for build in self.upload_builds.get_new_builds():
+            if build.tree in self.trees and build.compiler in self.compilers and build.host in hosts:
+                yield build
 
 
 class CachingBuildFarm(BuildFarm):

diff --git a/buildfarm/data.py b/buildfarm/data.py
index fb34ad79c82aa4f780eaeab25b9a0846f477d181..0eaae74295122b9e8b102d5ab09fdd8bac68dab8 100644 (file)
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -55,7 +55,7 @@ class BuildStatus(object):
     def __str__(self):
         if self.other_failures:
             return ",".join(self.other_failures)
-        return "/".join(self._status_tuple())
+        return "/".join(map(str, self._status_tuple()))
 
     def broken_host(self):
         if "disk full" in self.other_failures:
@@ -365,6 +365,16 @@ class UploadBuildResultStore(object):
         """
         self.path = path
 
+    def get_new_builds(self):
+        for name in os.listdir(self.path):
+            try:
+                (build, tree, host, compiler, extension) = name.split(".")
+            except ValueError:
+                continue
+            if build != "build" or extension != "log":
+                continue
+            yield self.get_build(tree, host, compiler)
+
     def build_fname(self, tree, host, compiler):
         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 

diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py
index f89cfb1772fa83433f3187079ee4a7e1a985e174..9824c51d97604498accd843faf60014c91b7ce6d 100755 (executable)
--- a/buildfarm/tests/test_data.py
+++ b/buildfarm/tests/test_data.py
@@ -298,6 +298,15 @@ class UploadBuildResultStoreTestBase(object):
             self.x.build_fname("mytree", "myhost", "cc"),
             "%s/data/upload/build.mytree.myhost.cc" % self.path)
 
+    def test_get_new_builds(self):
+        self.assertEquals([], list(self.x.get_new_builds()))
+        path = self.create_mock_logfile("tdb", "charis", "cc")
+        new_builds = list(self.x.get_new_builds())
+        self.assertEquals(1, len(new_builds))
+        self.assertEquals("tdb", new_builds[0].tree)
+        self.assertEquals("charis", new_builds[0].host)
+        self.assertEquals("cc", new_builds[0].compiler)
+
 
 class UploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFarmTestCase):