2 # subunit: extensions to Python unittest to get test results from subprocesses.
3 # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
5 # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6 # license at the users choice. A copy of both licenses are available in the
7 # project source as Apache-2.0 and BSD. You may not use this file except in
8 # compliance with one of these two licences.
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # license you chose for the specific language governing permissions and
14 # limitations under that license.
22 from testtools import TestCase
23 from testtools.compat import StringIO
24 from testtools.content import (
28 from testtools.testresult.doubles import ExtendedTestResult
31 import subunit.iso8601 as iso8601
32 import subunit.test_results
37 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
39 def __init__(self, decorated):
41 super(LoggingDecorator, self).__init__(decorated)
43 def _before_event(self):
47 class AssertBeforeTestResult(LoggingDecorator):
48 """A TestResult for checking preconditions."""
50 def __init__(self, decorated, test):
52 super(AssertBeforeTestResult, self).__init__(decorated)
54 def _before_event(self):
55 self.test.assertEqual(1, self.earlier._calls)
56 super(AssertBeforeTestResult, self)._before_event()
59 class TimeCapturingResult(unittest.TestResult):
62 super(TimeCapturingResult, self).__init__()
65 def time(self, a_datetime):
66 self._calls.append(a_datetime)
69 class TestHookedTestResultDecorator(unittest.TestCase):
73 terminal = unittest.TestResult()
74 # Asserts that the call was made to self.result before asserter was
76 asserter = AssertBeforeTestResult(terminal, self)
77 # The result object we call, which much increase its call count.
78 self.result = LoggingDecorator(asserter)
79 asserter.earlier = self.result
80 self.decorated = asserter
83 # The hook in self.result must have been called
84 self.assertEqual(1, self.result._calls)
85 # The hook in asserter must have been called too, otherwise the
86 # assertion about ordering won't have completed.
87 self.assertEqual(1, self.decorated._calls)
89 def test_startTest(self):
90 self.result.startTest(self)
92 def test_startTestRun(self):
93 self.result.startTestRun()
95 def test_stopTest(self):
96 self.result.stopTest(self)
98 def test_stopTestRun(self):
99 self.result.stopTestRun()
101 def test_addError(self):
102 self.result.addError(self, subunit.RemoteError())
104 def test_addError_details(self):
105 self.result.addError(self, details={})
107 def test_addFailure(self):
108 self.result.addFailure(self, subunit.RemoteError())
110 def test_addFailure_details(self):
111 self.result.addFailure(self, details={})
113 def test_addSuccess(self):
114 self.result.addSuccess(self)
116 def test_addSuccess_details(self):
117 self.result.addSuccess(self, details={})
119 def test_addSkip(self):
120 self.result.addSkip(self, "foo")
122 def test_addSkip_details(self):
123 self.result.addSkip(self, details={})
125 def test_addExpectedFailure(self):
126 self.result.addExpectedFailure(self, subunit.RemoteError())
128 def test_addExpectedFailure_details(self):
129 self.result.addExpectedFailure(self, details={})
131 def test_addUnexpectedSuccess(self):
132 self.result.addUnexpectedSuccess(self)
134 def test_addUnexpectedSuccess_details(self):
135 self.result.addUnexpectedSuccess(self, details={})
137 def test_progress(self):
138 self.result.progress(1, subunit.PROGRESS_SET)
140 def test_wasSuccessful(self):
141 self.result.wasSuccessful()
143 def test_shouldStop(self):
144 self.result.shouldStop
150 self.result.time(None)
153 class TestAutoTimingTestResultDecorator(unittest.TestCase):
156 # And end to the chain which captures time events.
157 terminal = TimeCapturingResult()
158 # The result object under test.
159 self.result = subunit.test_results.AutoTimingTestResultDecorator(
161 self.decorated = terminal
163 def test_without_time_calls_time_is_called_and_not_None(self):
164 self.result.startTest(self)
165 self.assertEqual(1, len(self.decorated._calls))
166 self.assertNotEqual(None, self.decorated._calls[0])
168 def test_no_time_from_progress(self):
169 self.result.progress(1, subunit.PROGRESS_CUR)
170 self.assertEqual(0, len(self.decorated._calls))
172 def test_no_time_from_shouldStop(self):
173 self.decorated.stop()
174 self.result.shouldStop
175 self.assertEqual(0, len(self.decorated._calls))
177 def test_calling_time_inhibits_automatic_time(self):
178 # Calling time() outputs a time signal immediately and prevents
179 # automatically adding one when other methods are called.
180 time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
181 self.result.time(time)
182 self.result.startTest(self)
183 self.result.stopTest(self)
184 self.assertEqual(1, len(self.decorated._calls))
185 self.assertEqual(time, self.decorated._calls[0])
187 def test_calling_time_None_enables_automatic_time(self):
188 time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
189 self.result.time(time)
190 self.assertEqual(1, len(self.decorated._calls))
191 self.assertEqual(time, self.decorated._calls[0])
192 # Calling None passes the None through, in case other results care.
193 self.result.time(None)
194 self.assertEqual(2, len(self.decorated._calls))
195 self.assertEqual(None, self.decorated._calls[1])
196 # Calling other methods doesn't generate an automatic time event.
197 self.result.startTest(self)
198 self.assertEqual(3, len(self.decorated._calls))
199 self.assertNotEqual(None, self.decorated._calls[2])
202 class TestTagCollapsingDecorator(TestCase):
204 def test_tags_collapsed_outside_of_tests(self):
205 result = ExtendedTestResult()
206 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
207 tag_collapser.tags(set(['a']), set())
208 tag_collapser.tags(set(['b']), set())
209 tag_collapser.startTest(self)
211 [('tags', set(['a', 'b']), set([])),
215 def test_tags_collapsed_outside_of_tests_are_flushed(self):
216 result = ExtendedTestResult()
217 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
218 tag_collapser.startTestRun()
219 tag_collapser.tags(set(['a']), set())
220 tag_collapser.tags(set(['b']), set())
221 tag_collapser.startTest(self)
222 tag_collapser.addSuccess(self)
223 tag_collapser.stopTest(self)
224 tag_collapser.stopTestRun()
227 ('tags', set(['a', 'b']), set([])),
229 ('addSuccess', self),
234 def test_tags_forwarded_after_tests(self):
235 test = subunit.RemotedTestCase('foo')
236 result = ExtendedTestResult()
237 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
238 tag_collapser.startTestRun()
239 tag_collapser.startTest(test)
240 tag_collapser.addSuccess(test)
241 tag_collapser.stopTest(test)
242 tag_collapser.tags(set(['a']), set(['b']))
243 tag_collapser.stopTestRun()
247 ('addSuccess', test),
249 ('tags', set(['a']), set(['b'])),
254 def test_tags_collapsed_inside_of_tests(self):
255 result = ExtendedTestResult()
256 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
257 test = subunit.RemotedTestCase('foo')
258 tag_collapser.startTest(test)
259 tag_collapser.tags(set(['a']), set())
260 tag_collapser.tags(set(['b']), set(['a']))
261 tag_collapser.tags(set(['c']), set())
262 tag_collapser.stopTest(test)
264 [('startTest', test),
265 ('tags', set(['b', 'c']), set(['a'])),
269 def test_tags_collapsed_inside_of_tests_different_ordering(self):
270 result = ExtendedTestResult()
271 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
272 test = subunit.RemotedTestCase('foo')
273 tag_collapser.startTest(test)
274 tag_collapser.tags(set(), set(['a']))
275 tag_collapser.tags(set(['a', 'b']), set())
276 tag_collapser.tags(set(['c']), set())
277 tag_collapser.stopTest(test)
279 [('startTest', test),
280 ('tags', set(['a', 'b', 'c']), set()),
284 def test_tags_sent_before_result(self):
285 # Because addSuccess and friends tend to send subunit output
286 # immediately, and because 'tags:' before a result line means
287 # something different to 'tags:' after a result line, we need to be
288 # sure that tags are emitted before 'addSuccess' (or whatever).
289 result = ExtendedTestResult()
290 tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
291 test = subunit.RemotedTestCase('foo')
292 tag_collapser.startTest(test)
293 tag_collapser.tags(set(['a']), set())
294 tag_collapser.addSuccess(test)
295 tag_collapser.stopTest(test)
297 [('startTest', test),
298 ('tags', set(['a']), set()),
299 ('addSuccess', test),
304 class TestTimeCollapsingDecorator(TestCase):
308 return datetime.datetime(
309 2000, 1, self.getUniqueInteger(), tzinfo=iso8601.UTC)
311 def test_initial_time_forwarded(self):
312 # We always forward the first time event we see.
313 result = ExtendedTestResult()
314 tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
315 a_time = self.make_time()
316 tag_collapser.time(a_time)
317 self.assertEquals([('time', a_time)], result._events)
319 def test_time_collapsed_to_first_and_last(self):
320 # If there are many consecutive time events, only the first and last
322 result = ExtendedTestResult()
323 tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
324 times = [self.make_time() for i in range(5)]
326 tag_collapser.time(a_time)
327 tag_collapser.startTest(subunit.RemotedTestCase('foo'))
329 [('time', times[0]), ('time', times[-1])], result._events[:-1])
331 def test_only_one_time_sent(self):
332 # If we receive a single time event followed by a non-time event, we
333 # send exactly one time event.
334 result = ExtendedTestResult()
335 tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
336 a_time = self.make_time()
337 tag_collapser.time(a_time)
338 tag_collapser.startTest(subunit.RemotedTestCase('foo'))
339 self.assertEquals([('time', a_time)], result._events[:-1])
341 def test_duplicate_times_not_sent(self):
342 # Many time events with the exact same time are collapsed into one
344 result = ExtendedTestResult()
345 tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
346 a_time = self.make_time()
348 tag_collapser.time(a_time)
349 tag_collapser.startTest(subunit.RemotedTestCase('foo'))
350 self.assertEquals([('time', a_time)], result._events[:-1])
352 def test_no_times_inserted(self):
353 result = ExtendedTestResult()
354 tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
355 a_time = self.make_time()
356 tag_collapser.time(a_time)
357 foo = subunit.RemotedTestCase('foo')
358 tag_collapser.startTest(foo)
359 tag_collapser.addSuccess(foo)
360 tag_collapser.stopTest(foo)
365 ('stopTest', foo)], result._events)
368 class TestByTestResultTests(testtools.TestCase):
371 super(TestByTestResultTests, self).setUp()
373 self.result = subunit.test_results.TestByTestResult(self.on_test)
374 if sys.version_info >= (3, 0):
375 self.result._now = iter(range(5)).__next__
377 self.result._now = iter(range(5)).next
379 def assertCalled(self, **kwargs):
387 defaults.update(kwargs)
388 self.assertEqual([defaults], self.log)
390 def on_test(self, **kwargs):
391 self.log.append(kwargs)
393 def test_no_tests_nothing_reported(self):
394 self.result.startTestRun()
395 self.result.stopTestRun()
396 self.assertEqual([], self.log)
398 def test_add_success(self):
399 self.result.startTest(self)
400 self.result.addSuccess(self)
401 self.result.stopTest(self)
402 self.assertCalled(status='success')
404 def test_add_success_details(self):
405 self.result.startTest(self)
406 details = {'foo': 'bar'}
407 self.result.addSuccess(self, details=details)
408 self.result.stopTest(self)
409 self.assertCalled(status='success', details=details)
412 if not getattr(self.result, 'tags', None):
413 self.skipTest("No tags in testtools")
414 self.result.tags(['foo'], [])
415 self.result.startTest(self)
416 self.result.addSuccess(self)
417 self.result.stopTest(self)
418 self.assertCalled(status='success', tags=set(['foo']))
420 def test_add_error(self):
421 self.result.startTest(self)
424 except ZeroDivisionError:
425 error = sys.exc_info()
426 self.result.addError(self, error)
427 self.result.stopTest(self)
430 details={'traceback': TracebackContent(error, self)})
432 def test_add_error_details(self):
433 self.result.startTest(self)
434 details = {"foo": text_content("bar")}
435 self.result.addError(self, details=details)
436 self.result.stopTest(self)
437 self.assertCalled(status='error', details=details)
439 def test_add_failure(self):
440 self.result.startTest(self)
442 self.fail("intentional failure")
443 except self.failureException:
444 failure = sys.exc_info()
445 self.result.addFailure(self, failure)
446 self.result.stopTest(self)
449 details={'traceback': TracebackContent(failure, self)})
451 def test_add_failure_details(self):
452 self.result.startTest(self)
453 details = {"foo": text_content("bar")}
454 self.result.addFailure(self, details=details)
455 self.result.stopTest(self)
456 self.assertCalled(status='failure', details=details)
458 def test_add_xfail(self):
459 self.result.startTest(self)
462 except ZeroDivisionError:
463 error = sys.exc_info()
464 self.result.addExpectedFailure(self, error)
465 self.result.stopTest(self)
468 details={'traceback': TracebackContent(error, self)})
470 def test_add_xfail_details(self):
471 self.result.startTest(self)
472 details = {"foo": text_content("bar")}
473 self.result.addExpectedFailure(self, details=details)
474 self.result.stopTest(self)
475 self.assertCalled(status='xfail', details=details)
477 def test_add_unexpected_success(self):
478 self.result.startTest(self)
479 details = {'foo': 'bar'}
480 self.result.addUnexpectedSuccess(self, details=details)
481 self.result.stopTest(self)
482 self.assertCalled(status='success', details=details)
484 def test_add_skip_reason(self):
485 self.result.startTest(self)
486 reason = self.getUniqueString()
487 self.result.addSkip(self, reason)
488 self.result.stopTest(self)
490 status='skip', details={'reason': text_content(reason)})
492 def test_add_skip_details(self):
493 self.result.startTest(self)
494 details = {'foo': 'bar'}
495 self.result.addSkip(self, details=details)
496 self.result.stopTest(self)
497 self.assertCalled(status='skip', details=details)
499 def test_twice(self):
500 self.result.startTest(self)
501 self.result.addSuccess(self, details={'foo': 'bar'})
502 self.result.stopTest(self)
503 self.result.startTest(self)
504 self.result.addSuccess(self)
505 self.result.stopTest(self)
512 'details': {'foo': 'bar'}},
523 class TestCsvResult(testtools.TestCase):
525 def parse_stream(self, stream):
527 reader = csv.reader(stream)
530 def test_csv_output(self):
532 result = subunit.test_results.CsvResult(stream)
533 if sys.version_info >= (3, 0):
534 result._now = iter(range(5)).__next__
536 result._now = iter(range(5)).next
537 result.startTestRun()
538 result.startTest(self)
539 result.addSuccess(self)
540 result.stopTest(self)
543 [['test', 'status', 'start_time', 'stop_time'],
544 [self.id(), 'success', '0', '1'],
546 self.parse_stream(stream))
548 def test_just_header_when_no_tests(self):
550 result = subunit.test_results.CsvResult(stream)
551 result.startTestRun()
554 [['test', 'status', 'start_time', 'stop_time']],
555 self.parse_stream(stream))
557 def test_no_output_before_events(self):
559 subunit.test_results.CsvResult(stream)
560 self.assertEqual([], self.parse_stream(stream))
564 loader = subunit.tests.TestUtil.TestLoader()
565 result = loader.loadTestsFromName(__name__)