selftest/subunithelper.py

   1 # Python module for parsing and generating the Subunit protocol
   2 # (Samba-specific)
   3 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 __all__ = ['parse_results']
  19
  20 import datetime
  21 import re
  22 import sys
  23 import os
  24 from samba import subunit
  25 from samba.subunit.run import TestProtocolClient
  26 import unittest
  27 try:
  28     from dateutil.parser import isoparse as iso_parse_date
  29 except ImportError:
  30     try:
  31         from iso8601 import parse_date as iso_parse_date
  32     except ImportError:
  33         print('Install either python-dateutil >= 2.7.1 or python-iso8601')
  34
  35
  36 VALID_RESULTS = set(['success', 'successful', 'failure', 'fail', 'skip',
  37                      'knownfail', 'error', 'xfail', 'skip-testsuite',
  38                      'testsuite-failure', 'testsuite-xfail',
  39                      'testsuite-success', 'testsuite-error',
  40                      'uxsuccess', 'testsuite-uxsuccess'])
  41
  42
  43 class TestsuiteEnabledTestResult(unittest.TestResult):
  44
  45     def start_testsuite(self, name):
  46         raise NotImplementedError(self.start_testsuite)
  47
  48
  49 def parse_results(msg_ops, statistics, fh):
  50     exitcode = 0
  51     open_tests = {}
  52
  53     for l in fh:
  54         parts = l.split(None, 1)
  55         if not len(parts) == 2 or not l.startswith(parts[0]):
  56             msg_ops.output_msg(l)
  57             continue
  58         command = parts[0].rstrip(":")
  59         arg = parts[1]
  60         if command in ("test", "testing"):
  61             msg_ops.control_msg(l)
  62             name = arg.rstrip()
  63             test = subunit.RemotedTestCase(name)
  64             if name in open_tests:
  65                 msg_ops.addError(open_tests.pop(name), subunit.RemoteError(u"Test already running"))
  66             msg_ops.startTest(test)
  67             open_tests[name] = test
  68         elif command == "time":
  69             msg_ops.control_msg(l)
  70             try:
  71                 dt = iso_parse_date(arg.rstrip("\n"))
  72             except TypeError:
  73                 print("Unable to parse time line: %s" % arg.rstrip("\n"))
  74             else:
  75                 msg_ops.time(dt)
  76         elif command in VALID_RESULTS:
  77             msg_ops.control_msg(l)
  78             result = command
  79             grp = re.match(r"(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
  80             (testname, hasreason) = (grp.group(1), grp.group(2))
  81             if hasreason:
  82                 reason = ""
  83                 # reason may be specified in next lines
  84                 terminated = False
  85                 for l in fh:
  86                     msg_ops.control_msg(l)
  87                     if l == "]\n":
  88                         terminated = True
  89                         break
  90                     else:
  91                         reason += l
  92
  93                 if isinstance(reason, bytes):
  94                     remote_error = subunit.RemoteError(reason.decode("utf-8"))
  95                 else:
  96                     remote_error = subunit.RemoteError(reason)
  97
  98                 if not terminated:
  99                     statistics['TESTS_ERROR'] += 1
 100                     msg_ops.addError(subunit.RemotedTestCase(testname),
 101                                      subunit.RemoteError(u"result (%s) reason (%s) interrupted" % (result, reason)))
 102                     return 1
 103             else:
 104                 reason = None
 105                 remote_error = subunit.RemoteError(u"No reason specified")
 106             if result in ("success", "successful"):
 107                 try:
 108                     test = open_tests.pop(testname)
 109                 except KeyError:
 110                     statistics['TESTS_ERROR'] += 1
 111                     exitcode = 1
 112                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 113                 else:
 114                     statistics['TESTS_EXPECTED_OK'] += 1
 115                     msg_ops.addSuccess(test)
 116             elif result in ("xfail", "knownfail"):
 117                 try:
 118                     test = open_tests.pop(testname)
 119                 except KeyError:
 120                     statistics['TESTS_ERROR'] += 1
 121                     exitcode = 1
 122                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 123                 else:
 124                     statistics['TESTS_EXPECTED_FAIL'] += 1
 125                     msg_ops.addExpectedFailure(test, remote_error)
 126             elif result in ("uxsuccess", ):
 127                 try:
 128                     test = open_tests.pop(testname)
 129                 except KeyError:
 130                     statistics['TESTS_ERROR'] += 1
 131                     exitcode = 1
 132                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 133                 else:
 134                     statistics['TESTS_UNEXPECTED_OK'] += 1
 135                     msg_ops.addUnexpectedSuccess(test)
 136                     exitcode = 1
 137             elif result in ("failure", "fail"):
 138                 try:
 139                     test = open_tests.pop(testname)
 140                 except KeyError:
 141                     statistics['TESTS_ERROR'] += 1
 142                     exitcode = 1
 143                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 144                 else:
 145                     statistics['TESTS_UNEXPECTED_FAIL'] += 1
 146                     exitcode = 1
 147                     msg_ops.addFailure(test, remote_error)
 148             elif result == "skip":
 149                 statistics['TESTS_SKIP'] += 1
 150                 # Allow tests to be skipped without prior announcement of test
 151                 try:
 152                     test = open_tests.pop(testname)
 153                 except KeyError:
 154                     test = subunit.RemotedTestCase(testname)
 155                 msg_ops.addSkip(test, reason)
 156             elif result == "error":
 157                 statistics['TESTS_ERROR'] += 1
 158                 exitcode = 1
 159                 try:
 160                     test = open_tests.pop(testname)
 161                 except KeyError:
 162                     test = subunit.RemotedTestCase(testname)
 163                 msg_ops.addError(test, remote_error)
 164             elif result == "skip-testsuite":
 165                 msg_ops.skip_testsuite(testname)
 166             elif result == "testsuite-success":
 167                 msg_ops.end_testsuite(testname, "success", reason)
 168             elif result == "testsuite-failure":
 169                 msg_ops.end_testsuite(testname, "failure", reason)
 170                 exitcode = 1
 171             elif result == "testsuite-xfail":
 172                 msg_ops.end_testsuite(testname, "xfail", reason)
 173             elif result == "testsuite-uxsuccess":
 174                 msg_ops.end_testsuite(testname, "uxsuccess", reason)
 175                 exitcode = 1
 176             elif result == "testsuite-error":
 177                 msg_ops.end_testsuite(testname, "error", reason)
 178                 exitcode = 1
 179             else:
 180                 raise AssertionError("Recognized but unhandled result %r" %
 181                                      result)
 182         elif command == "testsuite":
 183             msg_ops.start_testsuite(arg.strip())
 184         elif command == "progress":
 185             arg = arg.strip()
 186             if arg == "pop":
 187                 msg_ops.progress(None, subunit.PROGRESS_POP)
 188             elif arg == "push":
 189                 msg_ops.progress(None, subunit.PROGRESS_PUSH)
 190             elif arg[0] in '+-':
 191                 msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
 192             else:
 193                 msg_ops.progress(int(arg), subunit.PROGRESS_SET)
 194         else:
 195             msg_ops.output_msg(l)
 196
 197     while open_tests:
 198         test = subunit.RemotedTestCase(open_tests.popitem()[1])
 199         msg_ops.addError(test, subunit.RemoteError(u"was started but never finished!"))
 200         statistics['TESTS_ERROR'] += 1
 201         exitcode = 1
 202
 203     return exitcode
 204
 205
 206 class SubunitOps(TestProtocolClient, TestsuiteEnabledTestResult):
 207
 208     def progress(self, count, whence):
 209         if whence == subunit.PROGRESS_POP:
 210             self._stream.write("progress: pop\n")
 211         elif whence == subunit.PROGRESS_PUSH:
 212             self._stream.write("progress: push\n")
 213         elif whence == subunit.PROGRESS_SET:
 214             self._stream.write("progress: %d\n" % count)
 215         elif whence == subunit.PROGRESS_CUR:
 216             raise NotImplementedError
 217
 218     # The following are Samba extensions:
 219     def start_testsuite(self, name):
 220         self._stream.write("testsuite: %s\n" % name)
 221
 222     def skip_testsuite(self, name, reason=None):
 223         if reason:
 224             self._stream.write("skip-testsuite: %s [\n%s\n]\n" % (name, reason))
 225         else:
 226             self._stream.write("skip-testsuite: %s\n" % name)
 227
 228     def end_testsuite(self, name, result, reason=None):
 229         if reason:
 230             self._stream.write("testsuite-%s: %s [\n%s\n]\n" % (result, name, reason))
 231         else:
 232             self._stream.write("testsuite-%s: %s\n" % (result, name))
 233
 234     def output_msg(self, msg):
 235         self._stream.write(msg)
 236
 237
 238 def read_test_regexes(*names):
 239     ret = []
 240     files = []
 241     for name in names:
 242         # if we are given a directory, we read all the files it contains
 243         # (except the ones that end with "~").
 244         if os.path.isdir(name):
 245             files.extend([os.path.join(name, x)
 246                           for x in os.listdir(name)
 247                           if x[-1] != '~'])
 248         else:
 249             files.append(name)
 250
 251     for filename in files:
 252         with open(filename, 'r') as f:
 253             for l in f:
 254                 l = l.strip()
 255                 if l == "" or l[0] == "#":
 256                     continue
 257                 if "#" in l:
 258                     (regex, reason) = l.split("#", 1)
 259                     ret.append(re.compile(regex.strip()))
 260                 else:
 261                     ret.append(re.compile(l))
 262
 263     return ret
 264
 265
 266 def find_in_list(regexes, fullname):
 267     for regex in regexes:
 268         if regex.match(fullname):
 269             return True
 270     return False
 271
 272
 273 class ImmediateFail(Exception):
 274     """Raised to abort immediately."""
 275
 276     def __init__(self):
 277         super(ImmediateFail, self).__init__("test failed and fail_immediately set")
 278
 279
 280 class FilterOps(unittest.TestResult):
 281
 282     def control_msg(self, msg):
 283         pass  # We regenerate control messages, so ignore this
 284
 285     def time(self, time):
 286         self._ops.time(time)
 287
 288     def progress(self, delta, whence):
 289         self._ops.progress(delta, whence)
 290
 291     def output_msg(self, msg):
 292         if self.output is None:
 293             sys.stdout.write(msg)
 294         else:
 295             self.output += msg
 296
 297     def startTest(self, test):
 298         self.seen_output = True
 299         test = self._add_prefix(test)
 300         if self.strip_ok_output:
 301             self.output = ""
 302
 303         self._ops.startTest(test)
 304
 305     def _add_prefix(self, test):
 306         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 307
 308     def addError(self, test, err=None):
 309         test = self._add_prefix(test)
 310         self.error_added += 1
 311         self.total_error += 1
 312         self._ops.addError(test, err)
 313         self._ops.writeOutcome(test)
 314         self.output = None
 315         if self.fail_immediately:
 316             raise ImmediateFail()
 317
 318     def addSkip(self, test, reason=None):
 319         self.seen_output = True
 320         test = self._add_prefix(test)
 321         self._ops.addSkip(test, reason)
 322         self._ops.writeOutcome(test)
 323         self.output = None
 324
 325     def addExpectedFailure(self, test, err=None):
 326         test = self._add_prefix(test)
 327         self._ops.addExpectedFailure(test, err)
 328         self._ops.writeOutcome(test)
 329         self.output = None
 330
 331     def addUnexpectedSuccess(self, test):
 332         test = self._add_prefix(test)
 333         self.uxsuccess_added += 1
 334         self.total_uxsuccess += 1
 335         self._ops.addUnexpectedSuccess(test)
 336         self._ops.writeOutcome(test)
 337         if self.output:
 338             self._ops.output_msg(self.output)
 339         self.output = None
 340         if self.fail_immediately:
 341             raise ImmediateFail()
 342
 343     def addFailure(self, test, err=None):
 344         test = self._add_prefix(test)
 345         xfail = find_in_list(self.expected_failures, test.id())
 346         if not xfail:
 347             xfail = find_in_list(self.flapping, test.id())
 348         if xfail:
 349             self.xfail_added += 1
 350             self.total_xfail += 1
 351             self._ops.addExpectedFailure(test, err)
 352             self._ops.writeOutcome(test)
 353         else:
 354             self.fail_added += 1
 355             self.total_fail += 1
 356             self._ops.addFailure(test, err)
 357             self._ops.writeOutcome(test)
 358             if self.output:
 359                 self._ops.output_msg(self.output)
 360             if self.fail_immediately:
 361                 raise ImmediateFail()
 362         self.output = None
 363
 364     def addSuccess(self, test):
 365         test = self._add_prefix(test)
 366         xfail = find_in_list(self.expected_failures, test.id())
 367         if xfail:
 368             self.uxsuccess_added += 1
 369             self.total_uxsuccess += 1
 370             self._ops.addUnexpectedSuccess(test)
 371             self._ops.writeOutcome(test)
 372             if self.output:
 373                 self._ops.output_msg(self.output)
 374             if self.fail_immediately:
 375                 raise ImmediateFail()
 376         else:
 377             self._ops.addSuccess(test)
 378             self._ops.writeOutcome(test)
 379         self.output = None
 380
 381     def skip_testsuite(self, name, reason=None):
 382         self._ops.skip_testsuite(name, reason)
 383
 384     def start_testsuite(self, name):
 385         self._ops.start_testsuite(name)
 386         self.error_added = 0
 387         self.fail_added = 0
 388         self.xfail_added = 0
 389         self.uxsuccess_added = 0
 390
 391     def end_testsuite(self, name, result, reason=None):
 392         xfail = False
 393
 394         if self.xfail_added > 0:
 395             xfail = True
 396         if self.fail_added > 0 or self.error_added > 0 or self.uxsuccess_added > 0:
 397             xfail = False
 398
 399         if xfail and result in ("fail", "failure"):
 400             result = "xfail"
 401
 402         if self.uxsuccess_added > 0 and result != "uxsuccess":
 403             result = "uxsuccess"
 404             if reason is None:
 405                 reason = "Subunit/Filter Reason"
 406             reason += "\n uxsuccess[%d]" % self.uxsuccess_added
 407
 408         if self.fail_added > 0 and result != "failure":
 409             result = "failure"
 410             if reason is None:
 411                 reason = "Subunit/Filter Reason"
 412             reason += "\n failures[%d]" % self.fail_added
 413
 414         if self.error_added > 0 and result != "error":
 415             result = "error"
 416             if reason is None:
 417                 reason = "Subunit/Filter Reason"
 418             reason += "\n errors[%d]" % self.error_added
 419
 420         self._ops.end_testsuite(name, result, reason)
 421         if result not in ("success", "xfail"):
 422             if self.output:
 423                 self._ops.output_msg(self.output)
 424             if self.fail_immediately:
 425                 raise ImmediateFail()
 426         self.output = None
 427
 428     def __init__(self, out, prefix=None, suffix=None, expected_failures=None,
 429                  strip_ok_output=False, fail_immediately=False,
 430                  flapping=None):
 431         self._ops = out
 432         self.seen_output = False
 433         self.output = None
 434         self.prefix = prefix
 435         self.suffix = suffix
 436         if expected_failures is not None:
 437             self.expected_failures = expected_failures
 438         else:
 439             self.expected_failures = []
 440         if flapping is not None:
 441             self.flapping = flapping
 442         else:
 443             self.flapping = []
 444         self.strip_ok_output = strip_ok_output
 445         self.xfail_added = 0
 446         self.fail_added = 0
 447         self.uxsuccess_added = 0
 448         self.total_xfail = 0
 449         self.total_error = 0
 450         self.total_fail = 0
 451         self.total_uxsuccess = 0
 452         self.error_added = 0
 453         self.fail_immediately = fail_immediately
 454
 455
 456 class PerfFilterOps(unittest.TestResult):
 457
 458     def progress(self, delta, whence):
 459         pass
 460
 461     def output_msg(self, msg):
 462         pass
 463
 464     def control_msg(self, msg):
 465         pass
 466
 467     def skip_testsuite(self, name, reason=None):
 468         self._ops.skip_testsuite(name, reason)
 469
 470     def start_testsuite(self, name):
 471         self.suite_has_time = False
 472
 473     def end_testsuite(self, name, result, reason=None):
 474         pass
 475
 476     def _add_prefix(self, test):
 477         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 478
 479     def time(self, time):
 480         self.latest_time = time
 481         self.suite_has_time = True
 482
 483     def get_time(self):
 484         if self.suite_has_time:
 485             return self.latest_time
 486         return datetime.datetime.now(tz=datetime.timezone.utc)
 487
 488     def startTest(self, test):
 489         self.seen_output = True
 490         test = self._add_prefix(test)
 491         self.starts[test.id()] = self.get_time()
 492
 493     def addSuccess(self, test):
 494         test = self._add_prefix(test)
 495         tid = test.id()
 496         if tid not in self.starts:
 497             self._ops.addError(test, "%s succeeded without ever starting!" % tid)
 498         delta = self.get_time() - self.starts[tid]
 499         self._ops.output_msg("elapsed-time: %s: %f\n" % (tid, delta.total_seconds()))
 500
 501     def addFailure(self, test, err=''):
 502         tid = test.id()
 503         delta = self.get_time() - self.starts[tid]
 504         self._ops.output_msg("failure: %s failed after %f seconds (%s)\n" %
 505                              (tid, delta.total_seconds(), err))
 506
 507     def addError(self, test, err=''):
 508         tid = test.id()
 509         delta = self.get_time() - self.starts[tid]
 510         self._ops.output_msg("error: %s failed after %f seconds (%s)\n" %
 511                              (tid, delta.total_seconds(), err))
 512
 513     def __init__(self, out, prefix='', suffix=''):
 514         self._ops = out
 515         self.prefix = prefix or ''
 516         self.suffix = suffix or ''
 517         self.starts = {}
 518         self.seen_output = False
 519         self.suite_has_time = False
 520
 521
 522 class PlainFormatter(TestsuiteEnabledTestResult):
 523
 524     def __init__(self, verbose, immediate, statistics,
 525                  totaltests=None):
 526         super(PlainFormatter, self).__init__()
 527         self.verbose = verbose
 528         self.immediate = immediate
 529         self.statistics = statistics
 530         self.start_time = None
 531         self.test_output = {}
 532         self.suitesfailed = []
 533         self.suites_ok = 0
 534         self.skips = {}
 535         self.index = 0
 536         self.name = None
 537         self._progress_level = 0
 538         self.totalsuites = totaltests
 539         self.last_time = None
 540
 541     @staticmethod
 542     def _format_time(delta):
 543         minutes, seconds = divmod(delta.seconds, 60)
 544         hours, minutes = divmod(minutes, 60)
 545         ret = ""
 546         if hours:
 547             ret += "%dh" % hours
 548         if minutes:
 549             ret += "%dm" % minutes
 550         ret += "%ds" % seconds
 551         return ret
 552
 553     def progress(self, offset, whence):
 554         if whence == subunit.PROGRESS_POP:
 555             self._progress_level -= 1
 556         elif whence == subunit.PROGRESS_PUSH:
 557             self._progress_level += 1
 558         elif whence == subunit.PROGRESS_SET:
 559             if self._progress_level == 0:
 560                 self.totalsuites = offset
 561         elif whence == subunit.PROGRESS_CUR:
 562             raise NotImplementedError
 563
 564     def time(self, dt):
 565         if self.start_time is None:
 566             self.start_time = dt
 567         self.last_time = dt
 568
 569     def start_testsuite(self, name):
 570         self.index += 1
 571         self.name = name
 572
 573         if not self.verbose:
 574             self.test_output[name] = ""
 575
 576         total_tests = (self.statistics['TESTS_EXPECTED_OK'] +
 577                        self.statistics['TESTS_EXPECTED_FAIL'] +
 578                        self.statistics['TESTS_ERROR'] +
 579                        self.statistics['TESTS_UNEXPECTED_FAIL'] +
 580                        self.statistics['TESTS_UNEXPECTED_OK'])
 581
 582         out = "[%d(%d)" % (self.index, total_tests)
 583         if self.totalsuites is not None:
 584             out += "/%d" % self.totalsuites
 585         if self.start_time is not None:
 586             out += " at " + self._format_time(self.last_time - self.start_time)
 587         if self.suitesfailed:
 588             out += ", %d errors" % (len(self.suitesfailed),)
 589         out += "] %s" % name
 590         if self.immediate:
 591             sys.stdout.write(out + "\n")
 592         else:
 593             sys.stdout.write(out + ": ")
 594
 595     def output_msg(self, output):
 596         if self.verbose:
 597             sys.stdout.write(output)
 598         elif self.name is not None:
 599             self.test_output[self.name] += output
 600         else:
 601             sys.stdout.write(output)
 602
 603     def control_msg(self, output):
 604         pass
 605
 606     def end_testsuite(self, name, result, reason):
 607         out = ""
 608         unexpected = False
 609
 610         if name not in self.test_output:
 611             print("no output for name[%s]" % name)
 612
 613         if result in ("success", "xfail"):
 614             self.suites_ok += 1
 615         else:
 616             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 617             if reason is not None:
 618                 self.output_msg("REASON: %s\n" % (reason,))
 619             self.suitesfailed.append(name)
 620             if self.immediate and not self.verbose and name in self.test_output:
 621                 out += self.test_output[name]
 622             unexpected = True
 623
 624         if not self.immediate:
 625             if not unexpected:
 626                 out += " ok\n"
 627             else:
 628                 out += " " + result.upper() + "\n"
 629
 630         sys.stdout.write(out)
 631
 632     def startTest(self, test):
 633         pass
 634
 635     def addSuccess(self, test):
 636         self.end_test(test.id(), "success", False)
 637
 638     def addError(self, test, err=None):
 639         self.end_test(test.id(), "error", True, err)
 640
 641     def addFailure(self, test, err=None):
 642         self.end_test(test.id(), "failure", True, err)
 643
 644     def addSkip(self, test, reason=None):
 645         self.end_test(test.id(), "skip", False, reason)
 646
 647     def addExpectedFailure(self, test, err=None):
 648         self.end_test(test.id(), "xfail", False, err)
 649
 650     def addUnexpectedSuccess(self, test):
 651         self.end_test(test.id(), "uxsuccess", True)
 652
 653     def end_test(self, testname, result, unexpected, err=None):
 654         if not unexpected:
 655             self.test_output[self.name] = ""
 656             if not self.immediate:
 657                 sys.stdout.write({
 658                     'failure': 'f',
 659                     'xfail': 'X',
 660                     'skip': 's',
 661                     'success': '.'}.get(result, "?(%s)" % result))
 662             return
 663
 664         if self.name not in self.test_output:
 665             self.test_output[self.name] = ""
 666
 667         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 668         if err is not None:
 669             self.test_output[self.name] += "REASON: %s\n" % str(err[1]).strip()
 670
 671         if self.immediate and not self.verbose:
 672             sys.stdout.write(self.test_output[self.name])
 673             self.test_output[self.name] = ""
 674
 675         if not self.immediate:
 676             sys.stdout.write({
 677                 'error': 'E',
 678                'failure': 'F',
 679                'uxsuccess': 'U',
 680                'success': 'S'}.get(result, "?"))
 681
 682     def write_summary(self, path):
 683         f = open(path, 'w+')
 684
 685         if self.suitesfailed:
 686             f.write("= Failed tests =\n")
 687
 688             for suite in self.suitesfailed:
 689                 f.write("== %s ==\n" % suite)
 690                 if suite in self.test_output:
 691                     f.write(self.test_output[suite] + "\n\n")
 692
 693             f.write("\n")
 694
 695         if not self.immediate and not self.verbose:
 696             for suite in self.suitesfailed:
 697                 print("=" * 78)
 698                 print("FAIL: %s" % suite)
 699                 if suite in self.test_output:
 700                     print(self.test_output[suite])
 701                 print("")
 702
 703         f.write("= Skipped tests =\n")
 704         for reason in self.skips.keys():
 705             f.write(reason + "\n")
 706             for name in self.skips[reason]:
 707                 f.write("\t%s\n" % name)
 708             f.write("\n")
 709         f.close()
 710
 711         if (not self.suitesfailed and
 712             not self.statistics['TESTS_UNEXPECTED_FAIL'] and
 713             not self.statistics['TESTS_UNEXPECTED_OK'] and
 714             not self.statistics['TESTS_ERROR']):
 715             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 716                   self.statistics['TESTS_EXPECTED_FAIL'])
 717             print("\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok))
 718         else:
 719             print("\nFAILED (%d failures, %d errors and %d unexpected successes in %d testsuites)" % (
 720                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 721                 self.statistics['TESTS_ERROR'],
 722                 self.statistics['TESTS_UNEXPECTED_OK'],
 723                 len(self.suitesfailed)))
 724
 725     def skip_testsuite(self, name, reason="UNKNOWN"):
 726         self.skips.setdefault(reason, []).append(name)
 727         if self.totalsuites:
 728             self.totalsuites -= 1