lib/subunit/python/subunit/tests/test_test_results.py

   1 #
   2 #  subunit: extensions to Python unittest to get test results from subprocesses.
   3 #  Copyright (C) 2009  Robert Collins <robertc@robertcollins.net>
   4 #
   5 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
   6 #  license at the users choice. A copy of both licenses are available in the
   7 #  project source as Apache-2.0 and BSD. You may not use this file except in
   8 #  compliance with one of these two licences.
   9 #
  10 #  Unless required by applicable law or agreed to in writing, software
  11 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
  12 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
  13 #  license you chose for the specific language governing permissions and
  14 #  limitations under that license.
  15 #
  16
  17 import csv
  18 import datetime
  19 import sys
  20 import unittest
  21
  22 from testtools import TestCase
  23 from testtools.compat import StringIO
  24 from testtools.content import (
  25     text_content,
  26     TracebackContent,
  27     )
  28 from testtools.testresult.doubles import ExtendedTestResult
  29
  30 import subunit
  31 import subunit.iso8601 as iso8601
  32 import subunit.test_results
  33
  34 import testtools
  35
  36
  37 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
  38
  39     def __init__(self, decorated):
  40         self._calls = 0
  41         super(LoggingDecorator, self).__init__(decorated)
  42
  43     def _before_event(self):
  44         self._calls += 1
  45
  46
  47 class AssertBeforeTestResult(LoggingDecorator):
  48     """A TestResult for checking preconditions."""
  49
  50     def __init__(self, decorated, test):
  51         self.test = test
  52         super(AssertBeforeTestResult, self).__init__(decorated)
  53
  54     def _before_event(self):
  55         self.test.assertEqual(1, self.earlier._calls)
  56         super(AssertBeforeTestResult, self)._before_event()
  57
  58
  59 class TimeCapturingResult(unittest.TestResult):
  60
  61     def __init__(self):
  62         super(TimeCapturingResult, self).__init__()
  63         self._calls = []
  64
  65     def time(self, a_datetime):
  66         self._calls.append(a_datetime)
  67
  68
  69 class TestHookedTestResultDecorator(unittest.TestCase):
  70
  71     def setUp(self):
  72         # An end to the chain
  73         terminal = unittest.TestResult()
  74         # Asserts that the call was made to self.result before asserter was
  75         # called.
  76         asserter = AssertBeforeTestResult(terminal, self)
  77         # The result object we call, which much increase its call count.
  78         self.result = LoggingDecorator(asserter)
  79         asserter.earlier = self.result
  80         self.decorated = asserter
  81
  82     def tearDown(self):
  83         # The hook in self.result must have been called
  84         self.assertEqual(1, self.result._calls)
  85         # The hook in asserter must have been called too, otherwise the
  86         # assertion about ordering won't have completed.
  87         self.assertEqual(1, self.decorated._calls)
  88
  89     def test_startTest(self):
  90         self.result.startTest(self)
  91
  92     def test_startTestRun(self):
  93         self.result.startTestRun()
  94
  95     def test_stopTest(self):
  96         self.result.stopTest(self)
  97
  98     def test_stopTestRun(self):
  99         self.result.stopTestRun()
 100
 101     def test_addError(self):
 102         self.result.addError(self, subunit.RemoteError())
 103
 104     def test_addError_details(self):
 105         self.result.addError(self, details={})
 106
 107     def test_addFailure(self):
 108         self.result.addFailure(self, subunit.RemoteError())
 109
 110     def test_addFailure_details(self):
 111         self.result.addFailure(self, details={})
 112
 113     def test_addSuccess(self):
 114         self.result.addSuccess(self)
 115
 116     def test_addSuccess_details(self):
 117         self.result.addSuccess(self, details={})
 118
 119     def test_addSkip(self):
 120         self.result.addSkip(self, "foo")
 121
 122     def test_addSkip_details(self):
 123         self.result.addSkip(self, details={})
 124
 125     def test_addExpectedFailure(self):
 126         self.result.addExpectedFailure(self, subunit.RemoteError())
 127
 128     def test_addExpectedFailure_details(self):
 129         self.result.addExpectedFailure(self, details={})
 130
 131     def test_addUnexpectedSuccess(self):
 132         self.result.addUnexpectedSuccess(self)
 133
 134     def test_addUnexpectedSuccess_details(self):
 135         self.result.addUnexpectedSuccess(self, details={})
 136
 137     def test_progress(self):
 138         self.result.progress(1, subunit.PROGRESS_SET)
 139
 140     def test_wasSuccessful(self):
 141         self.result.wasSuccessful()
 142
 143     def test_shouldStop(self):
 144         self.result.shouldStop
 145
 146     def test_stop(self):
 147         self.result.stop()
 148
 149     def test_time(self):
 150         self.result.time(None)
 151
 152
 153 class TestAutoTimingTestResultDecorator(unittest.TestCase):
 154
 155     def setUp(self):
 156         # And end to the chain which captures time events.
 157         terminal = TimeCapturingResult()
 158         # The result object under test.
 159         self.result = subunit.test_results.AutoTimingTestResultDecorator(
 160             terminal)
 161         self.decorated = terminal
 162
 163     def test_without_time_calls_time_is_called_and_not_None(self):
 164         self.result.startTest(self)
 165         self.assertEqual(1, len(self.decorated._calls))
 166         self.assertNotEqual(None, self.decorated._calls[0])
 167
 168     def test_no_time_from_progress(self):
 169         self.result.progress(1, subunit.PROGRESS_CUR)
 170         self.assertEqual(0, len(self.decorated._calls))
 171
 172     def test_no_time_from_shouldStop(self):
 173         self.decorated.stop()
 174         self.result.shouldStop
 175         self.assertEqual(0, len(self.decorated._calls))
 176
 177     def test_calling_time_inhibits_automatic_time(self):
 178         # Calling time() outputs a time signal immediately and prevents
 179         # automatically adding one when other methods are called.
 180         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 181         self.result.time(time)
 182         self.result.startTest(self)
 183         self.result.stopTest(self)
 184         self.assertEqual(1, len(self.decorated._calls))
 185         self.assertEqual(time, self.decorated._calls[0])
 186
 187     def test_calling_time_None_enables_automatic_time(self):
 188         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 189         self.result.time(time)
 190         self.assertEqual(1, len(self.decorated._calls))
 191         self.assertEqual(time, self.decorated._calls[0])
 192         # Calling None passes the None through, in case other results care.
 193         self.result.time(None)
 194         self.assertEqual(2, len(self.decorated._calls))
 195         self.assertEqual(None, self.decorated._calls[1])
 196         # Calling other methods doesn't generate an automatic time event.
 197         self.result.startTest(self)
 198         self.assertEqual(3, len(self.decorated._calls))
 199         self.assertNotEqual(None, self.decorated._calls[2])
 200
 201
 202 class TestTagCollapsingDecorator(TestCase):
 203
 204     def test_tags_collapsed_outside_of_tests(self):
 205         result = ExtendedTestResult()
 206         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 207         tag_collapser.tags(set(['a']), set())
 208         tag_collapser.tags(set(['b']), set())
 209         tag_collapser.startTest(self)
 210         self.assertEquals(
 211             [('tags', set(['a', 'b']), set([])),
 212              ('startTest', self),
 213              ], result._events)
 214
 215     def test_tags_collapsed_outside_of_tests_are_flushed(self):
 216         result = ExtendedTestResult()
 217         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 218         tag_collapser.startTestRun()
 219         tag_collapser.tags(set(['a']), set())
 220         tag_collapser.tags(set(['b']), set())
 221         tag_collapser.startTest(self)
 222         tag_collapser.addSuccess(self)
 223         tag_collapser.stopTest(self)
 224         tag_collapser.stopTestRun()
 225         self.assertEquals(
 226             [('startTestRun',),
 227              ('tags', set(['a', 'b']), set([])),
 228              ('startTest', self),
 229              ('addSuccess', self),
 230              ('stopTest', self),
 231              ('stopTestRun',),
 232              ], result._events)
 233
 234     def test_tags_forwarded_after_tests(self):
 235         test = subunit.RemotedTestCase('foo')
 236         result = ExtendedTestResult()
 237         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 238         tag_collapser.startTestRun()
 239         tag_collapser.startTest(test)
 240         tag_collapser.addSuccess(test)
 241         tag_collapser.stopTest(test)
 242         tag_collapser.tags(set(['a']), set(['b']))
 243         tag_collapser.stopTestRun()
 244         self.assertEqual(
 245             [('startTestRun',),
 246              ('startTest', test),
 247              ('addSuccess', test),
 248              ('stopTest', test),
 249              ('tags', set(['a']), set(['b'])),
 250              ('stopTestRun',),
 251              ],
 252             result._events)
 253
 254     def test_tags_collapsed_inside_of_tests(self):
 255         result = ExtendedTestResult()
 256         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 257         test = subunit.RemotedTestCase('foo')
 258         tag_collapser.startTest(test)
 259         tag_collapser.tags(set(['a']), set())
 260         tag_collapser.tags(set(['b']), set(['a']))
 261         tag_collapser.tags(set(['c']), set())
 262         tag_collapser.stopTest(test)
 263         self.assertEquals(
 264             [('startTest', test),
 265              ('tags', set(['b', 'c']), set(['a'])),
 266              ('stopTest', test)],
 267             result._events)
 268
 269     def test_tags_collapsed_inside_of_tests_different_ordering(self):
 270         result = ExtendedTestResult()
 271         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 272         test = subunit.RemotedTestCase('foo')
 273         tag_collapser.startTest(test)
 274         tag_collapser.tags(set(), set(['a']))
 275         tag_collapser.tags(set(['a', 'b']), set())
 276         tag_collapser.tags(set(['c']), set())
 277         tag_collapser.stopTest(test)
 278         self.assertEquals(
 279             [('startTest', test),
 280              ('tags', set(['a', 'b', 'c']), set()),
 281              ('stopTest', test)],
 282             result._events)
 283
 284     def test_tags_sent_before_result(self):
 285         # Because addSuccess and friends tend to send subunit output
 286         # immediately, and because 'tags:' before a result line means
 287         # something different to 'tags:' after a result line, we need to be
 288         # sure that tags are emitted before 'addSuccess' (or whatever).
 289         result = ExtendedTestResult()
 290         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
 291         test = subunit.RemotedTestCase('foo')
 292         tag_collapser.startTest(test)
 293         tag_collapser.tags(set(['a']), set())
 294         tag_collapser.addSuccess(test)
 295         tag_collapser.stopTest(test)
 296         self.assertEquals(
 297             [('startTest', test),
 298              ('tags', set(['a']), set()),
 299              ('addSuccess', test),
 300              ('stopTest', test)],
 301             result._events)
 302
 303
 304 class TestTimeCollapsingDecorator(TestCase):
 305
 306     def make_time(self):
 307         # Heh heh.
 308         return datetime.datetime(
 309             2000, 1, self.getUniqueInteger(), tzinfo=iso8601.UTC)
 310
 311     def test_initial_time_forwarded(self):
 312         # We always forward the first time event we see.
 313         result = ExtendedTestResult()
 314         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
 315         a_time = self.make_time()
 316         tag_collapser.time(a_time)
 317         self.assertEquals([('time', a_time)], result._events)
 318
 319     def test_time_collapsed_to_first_and_last(self):
 320         # If there are many consecutive time events, only the first and last
 321         # are sent through.
 322         result = ExtendedTestResult()
 323         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
 324         times = [self.make_time() for i in range(5)]
 325         for a_time in times:
 326             tag_collapser.time(a_time)
 327         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
 328         self.assertEquals(
 329             [('time', times[0]), ('time', times[-1])], result._events[:-1])
 330
 331     def test_only_one_time_sent(self):
 332         # If we receive a single time event followed by a non-time event, we
 333         # send exactly one time event.
 334         result = ExtendedTestResult()
 335         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
 336         a_time = self.make_time()
 337         tag_collapser.time(a_time)
 338         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
 339         self.assertEquals([('time', a_time)], result._events[:-1])
 340
 341     def test_duplicate_times_not_sent(self):
 342         # Many time events with the exact same time are collapsed into one
 343         # time event.
 344         result = ExtendedTestResult()
 345         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
 346         a_time = self.make_time()
 347         for i in range(5):
 348             tag_collapser.time(a_time)
 349         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
 350         self.assertEquals([('time', a_time)], result._events[:-1])
 351
 352     def test_no_times_inserted(self):
 353         result = ExtendedTestResult()
 354         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
 355         a_time = self.make_time()
 356         tag_collapser.time(a_time)
 357         foo = subunit.RemotedTestCase('foo')
 358         tag_collapser.startTest(foo)
 359         tag_collapser.addSuccess(foo)
 360         tag_collapser.stopTest(foo)
 361         self.assertEquals(
 362             [('time', a_time),
 363              ('startTest', foo),
 364              ('addSuccess', foo),
 365              ('stopTest', foo)], result._events)
 366
 367
 368 class TestByTestResultTests(testtools.TestCase):
 369
 370     def setUp(self):
 371         super(TestByTestResultTests, self).setUp()
 372         self.log = []
 373         self.result = subunit.test_results.TestByTestResult(self.on_test)
 374         if sys.version_info >= (3, 0):
 375             self.result._now = iter(range(5)).__next__
 376         else:
 377             self.result._now = iter(range(5)).next
 378
 379     def assertCalled(self, **kwargs):
 380         defaults = {
 381             'test': self,
 382             'tags': set(),
 383             'details': None,
 384             'start_time': 0,
 385             'stop_time': 1,
 386             }
 387         defaults.update(kwargs)
 388         self.assertEqual([defaults], self.log)
 389
 390     def on_test(self, **kwargs):
 391         self.log.append(kwargs)
 392
 393     def test_no_tests_nothing_reported(self):
 394         self.result.startTestRun()
 395         self.result.stopTestRun()
 396         self.assertEqual([], self.log)
 397
 398     def test_add_success(self):
 399         self.result.startTest(self)
 400         self.result.addSuccess(self)
 401         self.result.stopTest(self)
 402         self.assertCalled(status='success')
 403
 404     def test_add_success_details(self):
 405         self.result.startTest(self)
 406         details = {'foo': 'bar'}
 407         self.result.addSuccess(self, details=details)
 408         self.result.stopTest(self)
 409         self.assertCalled(status='success', details=details)
 410
 411     def test_tags(self):
 412         if not getattr(self.result, 'tags', None):
 413             self.skipTest("No tags in testtools")
 414         self.result.tags(['foo'], [])
 415         self.result.startTest(self)
 416         self.result.addSuccess(self)
 417         self.result.stopTest(self)
 418         self.assertCalled(status='success', tags=set(['foo']))
 419
 420     def test_add_error(self):
 421         self.result.startTest(self)
 422         try:
 423             1/0
 424         except ZeroDivisionError:
 425             error = sys.exc_info()
 426         self.result.addError(self, error)
 427         self.result.stopTest(self)
 428         self.assertCalled(
 429             status='error',
 430             details={'traceback': TracebackContent(error, self)})
 431
 432     def test_add_error_details(self):
 433         self.result.startTest(self)
 434         details = {"foo": text_content("bar")}
 435         self.result.addError(self, details=details)
 436         self.result.stopTest(self)
 437         self.assertCalled(status='error', details=details)
 438
 439     def test_add_failure(self):
 440         self.result.startTest(self)
 441         try:
 442             self.fail("intentional failure")
 443         except self.failureException:
 444             failure = sys.exc_info()
 445         self.result.addFailure(self, failure)
 446         self.result.stopTest(self)
 447         self.assertCalled(
 448             status='failure',
 449             details={'traceback': TracebackContent(failure, self)})
 450
 451     def test_add_failure_details(self):
 452         self.result.startTest(self)
 453         details = {"foo": text_content("bar")}
 454         self.result.addFailure(self, details=details)
 455         self.result.stopTest(self)
 456         self.assertCalled(status='failure', details=details)
 457
 458     def test_add_xfail(self):
 459         self.result.startTest(self)
 460         try:
 461             1/0
 462         except ZeroDivisionError:
 463             error = sys.exc_info()
 464         self.result.addExpectedFailure(self, error)
 465         self.result.stopTest(self)
 466         self.assertCalled(
 467             status='xfail',
 468             details={'traceback': TracebackContent(error, self)})
 469
 470     def test_add_xfail_details(self):
 471         self.result.startTest(self)
 472         details = {"foo": text_content("bar")}
 473         self.result.addExpectedFailure(self, details=details)
 474         self.result.stopTest(self)
 475         self.assertCalled(status='xfail', details=details)
 476
 477     def test_add_unexpected_success(self):
 478         self.result.startTest(self)
 479         details = {'foo': 'bar'}
 480         self.result.addUnexpectedSuccess(self, details=details)
 481         self.result.stopTest(self)
 482         self.assertCalled(status='success', details=details)
 483
 484     def test_add_skip_reason(self):
 485         self.result.startTest(self)
 486         reason = self.getUniqueString()
 487         self.result.addSkip(self, reason)
 488         self.result.stopTest(self)
 489         self.assertCalled(
 490             status='skip', details={'reason': text_content(reason)})
 491
 492     def test_add_skip_details(self):
 493         self.result.startTest(self)
 494         details = {'foo': 'bar'}
 495         self.result.addSkip(self, details=details)
 496         self.result.stopTest(self)
 497         self.assertCalled(status='skip', details=details)
 498
 499     def test_twice(self):
 500         self.result.startTest(self)
 501         self.result.addSuccess(self, details={'foo': 'bar'})
 502         self.result.stopTest(self)
 503         self.result.startTest(self)
 504         self.result.addSuccess(self)
 505         self.result.stopTest(self)
 506         self.assertEqual(
 507             [{'test': self,
 508               'status': 'success',
 509               'start_time': 0,
 510               'stop_time': 1,
 511               'tags': set(),
 512               'details': {'foo': 'bar'}},
 513              {'test': self,
 514               'status': 'success',
 515               'start_time': 2,
 516               'stop_time': 3,
 517               'tags': set(),
 518               'details': None},
 519              ],
 520             self.log)
 521
 522
 523 class TestCsvResult(testtools.TestCase):
 524
 525     def parse_stream(self, stream):
 526         stream.seek(0)
 527         reader = csv.reader(stream)
 528         return list(reader)
 529
 530     def test_csv_output(self):
 531         stream = StringIO()
 532         result = subunit.test_results.CsvResult(stream)
 533         if sys.version_info >= (3, 0):
 534             result._now = iter(range(5)).__next__
 535         else:
 536             result._now = iter(range(5)).next
 537         result.startTestRun()
 538         result.startTest(self)
 539         result.addSuccess(self)
 540         result.stopTest(self)
 541         result.stopTestRun()
 542         self.assertEqual(
 543             [['test', 'status', 'start_time', 'stop_time'],
 544              [self.id(), 'success', '0', '1'],
 545              ],
 546             self.parse_stream(stream))
 547
 548     def test_just_header_when_no_tests(self):
 549         stream = StringIO()
 550         result = subunit.test_results.CsvResult(stream)
 551         result.startTestRun()
 552         result.stopTestRun()
 553         self.assertEqual(
 554             [['test', 'status', 'start_time', 'stop_time']],
 555             self.parse_stream(stream))
 556
 557     def test_no_output_before_events(self):
 558         stream = StringIO()
 559         subunit.test_results.CsvResult(stream)
 560         self.assertEqual([], self.parse_stream(stream))
 561
 562
 563 def test_suite():
 564     loader = subunit.tests.TestUtil.TestLoader()
 565     result = loader.loadTestsFromName(__name__)
 566     return result