buildfarm/tests/test_data.py

   1 #!/usr/bin/python
   2 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2010
   3 #
   4 #   This program is free software; you can redistribute it and/or modify
   5 #   it under the terms of the GNU General Public License as published by
   6 #   the Free Software Foundation; either version 3 of the License, or
   7 #   (at your option) any later version.
   8 #
   9 #   This program is distributed in the hope that it will be useful,
  10 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #   GNU General Public License for more details.
  13 #
  14 #   You should have received a copy of the GNU General Public License
  15 #   along with this program; if not, write to the Free Software
  16 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 from cStringIO import StringIO
  19 import os
  20 import testtools
  21 import unittest
  22
  23 from buildfarm import data
  24
  25 from buildfarm.tests import BuildFarmTestCase
  26
  27
  28 class NonexistantTests(unittest.TestCase):
  29
  30     def test_nonexistant(self):
  31         self.assertRaises(
  32             Exception, data.BuildResultStore, "somedirthatdoesn'texist", None)
  33
  34
  35 class BuildResultStoreTestBase(object):
  36
  37     def test_build_fname(self):
  38         self.assertEquals(
  39             self.x.build_fname("mytree", "myhost", "cc", 123),
  40             "%s/data/oldrevs/build.mytree.myhost.cc-123" % self.path)
  41
  42     def test_build_remove(self):
  43         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  44                 "BUILD COMMIT REVISION: 12\n")
  45         build = self.x.get_build("tdb", "charis", "cc", "12")
  46         logname = build.basename + ".log"
  47         build.remove()
  48         self.assertFalse(os.path.exists(logname))
  49         self.assertRaises(data.NoSuchBuildError, self.x.get_build, "tdb", "charis", "cc", "12")
  50
  51     def test_build_repr(self):
  52         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  53             "BUILD COMMIT REVISION: 12\n")
  54         build = self.x.get_build("tdb", "charis", "cc", "12")
  55         self.assertEquals("<%s: revision 12 of tdb on charis using cc>" % build.__class__.__name__, repr(build))
  56
  57     def test_get_build_nonexistant(self):
  58         self.assertRaises(data.NoSuchBuildError, self.x.get_build, "tdb",
  59             "charis", "cc", "12")
  60
  61     def test_build_upload_time(self):
  62         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  63                 "BUILD COMMIT REVISION: 12\n", mtime=5)
  64         build = self.x.get_build("tdb", "charis", "cc", "12")
  65         self.assertEquals(5, build.upload_time)
  66
  67     def test_read_log(self):
  68         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  69             stdout_contents="This is what a log file looks like.\n"
  70             "BUILD COMMIT REVISION: 12\n")
  71         build = self.x.get_build("tdb", "charis", "cc", "12")
  72         self.assertEquals("This is what a log file looks like.\n"
  73                           "BUILD COMMIT REVISION: 12\n",
  74                           build.read_log().read())
  75
  76     def test_read_err(self):
  77         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  78             stdout_contents="BUILD COMMIT REVISION: 12\n",
  79             stderr_contents="This is what an stderr file looks like.")
  80         build = self.x.get_build("tdb", "charis", "cc", "12")
  81         self.assertEquals("This is what an stderr file looks like.",
  82                 build.read_err().read())
  83
  84     def test_read_err_nofile(self):
  85         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  86                 stdout_contents="BUILD COMMIT REVISION: 12\n")
  87         build = self.x.get_build("tdb", "charis", "cc", "12")
  88         self.assertEquals("", build.read_err().read())
  89
  90     def test_revision_details(self):
  91         self.upload_mock_logfile(self.x, "tdb", "charis", "cc", stdout_contents="""
  92 BUILD COMMIT REVISION: 43
  93 bla
  94 BUILD COMMIT TIME: 3 August 2010
  95 """)
  96         build = self.x.get_build("tdb", "charis", "cc", "43")
  97         rev = build.revision_details()
  98         self.assertIsInstance(rev, str)
  99         self.assertEquals("43", rev)
 100
 101     def test_revision_details_no_timestamp(self):
 102         self.upload_mock_logfile(self.x, "tdb", "charis", "cc", stdout_contents="""
 103 BUILD COMMIT REVISION: 43
 104 BUILD REVISION: 42
 105 BLA
 106 """)
 107         build = self.x.get_build("tdb", "charis", "cc", "43")
 108         self.assertEquals("43", build.revision_details())
 109
 110     def test_err_count(self):
 111         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
 112             stdout_contents="BUILD COMMIT REVISION: 12\n",
 113             stderr_contents="""error1
 114 error2
 115 error3""")
 116         build = self.x.get_build("tdb", "charis", "cc", "12")
 117         self.assertEquals(3, build.err_count())
 118
 119     def test_upload_build(self):
 120         path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
 121 BUILD COMMIT REVISION: myrev
 122 """)
 123         build = data.Build(path[:-4], "tdb", "charis", "cc")
 124         self.x.upload_build(build)
 125         uploaded_build = self.x.get_build("tdb", "charis", "cc", "myrev")
 126         self.assertEquals(uploaded_build.log_checksum(), build.log_checksum())
 127
 128     def test_upload_build_no_rev(self):
 129         path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
 130 """)
 131         build = data.Build(path[:-4], "tdb", "charis", "cc")
 132         self.assertRaises(Exception, self.x.upload_build, build)
 133
 134     def test_get_previous_revision(self):
 135         self.assertRaises(data.NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12")
 136
 137     def test_get_latest_revision_none(self):
 138         self.assertRaises(data.NoSuchBuildError, self.x.get_latest_revision, "tdb", "charis", "cc")
 139
 140     def test_get_old_builds_none(self):
 141         self.assertEquals([],
 142             list(self.x.get_old_builds("tdb", "charis", "gcc")))
 143
 144     def test_get_old_builds(self):
 145         path = self.create_mock_logfile("tdb", "charis", "cc",
 146             contents="""
 147 BUILD COMMIT REVISION: 12
 148 """)
 149         build = data.Build(path[:-4], "tdb", "charis", "cc")
 150         b1 = self.x.upload_build(build)
 151         path = self.create_mock_logfile("tdb", "charis", "cc",
 152             contents="""
 153 BUILD COMMIT REVISION: 15
 154 """)
 155         build = data.Build(path[:-4], "tdb", "charis", "cc")
 156         b2 = self.x.upload_build(build)
 157         path = self.create_mock_logfile("tdb", "charis", "cc",
 158             contents="""
 159 BUILD COMMIT REVISION: 15
 160 """)
 161         self.assertEquals([b1, b2],
 162             list(self.x.get_old_builds("tdb", "charis", "cc")))
 163
 164
 165 class BuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
 166
 167     def setUp(self):
 168         super(BuildResultStoreTests, self).setUp()
 169
 170         self.x = data.BuildResultStore(
 171             os.path.join(self.path, "data", "oldrevs"))
 172
 173
 174 class BuildStatusFromLogs(testtools.TestCase):
 175
 176     def parse_logs(self, log, err):
 177         return data.build_status_from_logs(StringIO(log), StringIO(err))
 178
 179     def test_nothing(self):
 180         s = self.parse_logs("", "")
 181         self.assertEquals([], s.stages)
 182         self.assertEquals(set(), s.other_failures)
 183
 184     def test_disk_full(self):
 185         self.assertEquals(set(["disk full"]),
 186             self.parse_logs("foo\nbar\nNo space left on device\nla\n",
 187                 "").other_failures)
 188         self.assertEquals(set(["disk full"]),
 189             self.parse_logs(
 190                 "", "foo\nbar\nNo space left on device\nla\n").other_failures)
 191
 192     def test_timeout(self):
 193         self.assertEquals(set(["timeout"]),
 194             self.parse_logs("foo\nbar\nmaximum runtime exceeded\nla\n",
 195                 "").other_failures)
 196
 197     def test_failed_test(self):
 198         log = """
 199 TEST STATUS:1
 200 """
 201         res = self.parse_logs(log, "")
 202         self.assertEquals(res.stages, [
 203             ("TEST", 1)])
 204
 205     def test_failed_test_whitespace(self):
 206         log = """
 207 TEST STATUS:  1
 208 """
 209         res = self.parse_logs(log, "")
 210         self.assertEquals(res.stages,
 211             [("TEST", 1)])
 212
 213     def test_failed_test_noise(self):
 214         log = """
 215 CONFIGURE STATUS: 2
 216 TEST STATUS:  1
 217 CC_CHECKER STATUS:      2
 218 """
 219         res = self.parse_logs(log, "")
 220         self.assertEquals(res.stages,
 221             [("CONFIGURE", 2), ("TEST", 1), ("CC_CHECKER", 2)])
 222
 223     def test_no_test_output(self):
 224         log = """
 225 CONFIGURE STATUS: 2
 226 TEST STATUS: 0
 227 CC_CHECKER STATUS:      2
 228 """
 229         res = self.parse_logs(log, "")
 230         self.assertEquals(res.stages,
 231             [("CONFIGURE", 2), ("TEST", 0), ("CC_CHECKER", 2)])
 232
 233     def test_granular_test(self):
 234         log = """
 235 CONFIGURE STATUS: 2
 236 testsuite-success: toto
 237 testsuite-failure: foo
 238 testsuite-failure: bar
 239 testsuite-failure: biz
 240 TEST STATUS: 1
 241 CC_CHECKER STATUS:      2
 242 """
 243         res = self.parse_logs(log, "")
 244         self.assertEquals(res.stages,
 245             [("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 246
 247
 248 class BuildStatusTest(testtools.TestCase):
 249
 250     def test_cmp_equal(self):
 251         a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 252         b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 253
 254         self.assertEquals(cmp(a, b), 0)
 255
 256     def test_cmp_empty(self):
 257         self.assertEquals(cmp(data.BuildStatus(), data.BuildStatus()), 0)
 258
 259     def test_cmp_other_failures(self):
 260         self.assertEquals(cmp(
 261             data.BuildStatus((), set(["foo"])), data.BuildStatus((), set(["foo"]))),
 262             0)
 263
 264     def test_cmp_intermediate_errors(self):
 265         a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)])
 266         b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 7), ("CC_CHECKER", 3)])
 267         self.assertEquals(cmp(a, b), 1)
 268
 269     def test_cmp_bigger(self):
 270         a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)])
 271         b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 272         c = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3)])
 273         d = data.BuildStatus([], set(["super error"]))
 274         e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 275
 276         # less stage means smaller, more error/higher error code means smaller as well
 277         self.assertEquals(cmp(b, a), 1)
 278
 279         self.assertEquals(cmp(a, c), 1)
 280
 281         self.assertEquals(cmp(a, d), 1)
 282
 283         self.assertEquals(cmp(b, e), 1)
 284
 285     def test_cmp_smaller(self):
 286         a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 287         b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)])
 288         c = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3)])
 289         d = data.BuildStatus([], set(["super error"]))
 290         e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 291
 292         # less stage means smaller, more error/higher error code means smaller as well
 293         self.assertEquals(cmp(a, b), -1)
 294
 295         self.assertEquals(cmp(c, b), -1)
 296
 297         self.assertEquals(cmp(d, c), -1)
 298
 299         self.assertEquals(cmp(e, c), -1)
 300
 301     def test_cmp_with_other_failures(self):
 302         d = data.BuildStatus([], set(["super error"]))
 303         e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 304         self.assertEquals(cmp(d, e), -1)
 305
 306     def test_str(self):
 307         a = data.BuildStatus([("CONFIGURE", 3), ("BUILD", 2)])
 308         self.assertEquals("3/2", str(a))
 309
 310     def test_str_other_failures(self):
 311         a = data.BuildStatus([("CONFIGURE", 3), ("BUILD", 2)], set(["panic"]))
 312         self.assertEquals("panic", str(a))
 313
 314
 315 class BuildStatusRegressedSinceTests(testtools.TestCase):
 316
 317     def assertRegressedSince(self, expected, old_status, new_status):
 318         (stages1, other_failures1) = old_status
 319         (stages2, other_failures2) = new_status
 320         a = data.BuildStatus(stages1, set(other_failures1))
 321         b = data.BuildStatus(stages2, set(other_failures2))
 322         self.assertEquals(expected, b.regressed_since(a))
 323
 324     def test_same(self):
 325         self.assertRegressedSince(
 326             False,
 327             ([("CONFIGURE", 2)], []),
 328             ([("CONFIGURE", 2)], []))
 329
 330     def test_same_panic(self):
 331         self.assertRegressedSince(
 332             False,
 333             ([("CONFIGURE", 2)], ["panic"]),
 334             ([("CONFIGURE", 2)], ["panic"]))
 335
 336     def test_other_failures_gone(self):
 337         self.assertRegressedSince(
 338             True,
 339             ([("CONFIGURE", 0)], ["panic"]),
 340             ([("CONFIGURE", 2)], ["panic"]))
 341
 342     def test_more_stages_completed(self):
 343         self.assertRegressedSince(
 344             False,
 345             ([("CONFIGURE", 0)], []),
 346             ([("CONFIGURE", 0), ("BUILD", 0)], []))
 347
 348     def test_less_errors(self):
 349         self.assertRegressedSince(
 350             False,
 351             ([("CONFIGURE", 0), ("BUILD", 0), ("TEST", 0), ("INSTALL", 1)], []),
 352             ([("CONFIGURE", 0), ("BUILD", 0), ("TEST", 0), ("INSTALL", 0)], []))
 353
 354     def test_no_longer_inconsistent(self):
 355         self.assertRegressedSince(
 356             False,
 357             ([("CONFIGURE", 0)], ["inconsistent test result"]),
 358             ([("CONFIGURE", 0)], []))
 359
 360
 361 class UploadBuildResultStoreTestBase(object):
 362
 363     def test_build_fname(self):
 364         self.assertEquals(
 365             self.x.build_fname("mytree", "myhost", "cc"),
 366             "%s/data/upload/build.mytree.myhost.cc" % self.path)
 367
 368     def test_get_all_builds(self):
 369         self.assertEquals([], list(self.x.get_all_builds()))
 370         path = self.create_mock_logfile("tdb", "charis", "cc")
 371         new_builds = list(self.x.get_all_builds())
 372         self.assertEquals(1, len(new_builds))
 373         self.assertEquals("tdb", new_builds[0].tree)
 374         self.assertEquals("charis", new_builds[0].host)
 375         self.assertEquals("cc", new_builds[0].compiler)
 376
 377
 378 class UploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFarmTestCase):
 379
 380     def setUp(self):
 381         super(UploadBuildResultStoreTests, self).setUp()
 382
 383         self.x = data.UploadBuildResultStore(
 384             os.path.join(self.path, "data", "upload"))
 385
 386
 387