web/build.py

   1 #!/usr/bin/python
   2 # This CGI script presents the results of the build_farm build
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>     2007-2010
   9 # Copyright (C) Matthieu Patou <mat@matws.net>         2010-2010
  10 #
  11 #   This program is free software; you can redistribute it and/or modify
  12 #   it under the terms of the GNU General Public License as published by
  13 #   the Free Software Foundation; either version 3 of the License, or
  14 #   (at your option) any later version.
  15 #
  16 #   This program is distributed in the hope that it will be useful,
  17 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 #   GNU General Public License for more details.
  20 #
  21 #   You should have received a copy of the GNU General Public License
  22 #   along with this program; if not, write to the Free Software
  23 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  24
  25 # TODO: Allow filtering of the "Recent builds" list to show
  26 # e.g. only broken builds or only builds that you care about.
  27
  28 import os
  29 import sys
  30 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
  31
  32 from buildfarm import (
  33     data,
  34     history,
  35     hostdb,
  36     open_hostdb,
  37     util,
  38     )
  39
  40 import cgi
  41 import re
  42 import time
  43
  44 import wsgiref.util
  45 standalone = 0
  46 webdir = os.path.dirname(__file__)
  47 basedir = os.path.abspath(os.path.join(webdir, ".."))
  48
  49 db = data.BuildResultStore(basedir)
  50 history = history.History(db)
  51 hostsdb = open_hostdb()
  52
  53 compilers = db.compilers
  54 # host.properties are unicode object and the framework expect string object
  55 hosts = dict([(host.name.encode("utf-8"), host) for host in hostsdb.hosts()])
  56 trees = db.trees
  57 OLDAGE = db.OLDAGE
  58
  59 UNPACKED_BASE = "http://svn.samba.org/ftp/unpacked"
  60 GITWEB_BASE = "http://gitweb.samba.org"
  61
  62 # this is automatically filled in
  63 deadhosts = []
  64
  65 def get_param(form, param):
  66     """get a param from the request, after sanitizing it"""
  67     if param not in form:
  68         return None
  69
  70     result = [s.replace(" ", "_") for s in form.getlist(param)]
  71
  72     for entry in result:
  73         if re.match("[^a-zA-Z0-9\-\_\.]", entry):
  74             raise Exception("Parameter %s is invalid" % param)
  75
  76     return result[0]
  77
  78
  79 def build_link(myself, tree, host, compiler, rev, status):
  80     if rev:
  81         opt_rev = ';revision=%s' % rev
  82     else:
  83         opt_rev = ''
  84     return "<a href='%s?function=View+Build;host=%s;tree=%s;compiler=%s%s'>%s</a>" % (
  85            myself, host, tree, compiler, opt_rev, status)
  86
  87
  88 def html_build_status(status):
  89     def span(classname, contents):
  90         return "<span class=\"%s\">%s</span>" % (classname, contents)
  91
  92     def span_status(n, st):
  93         if n == "CC_CHECKER":
  94             if st == 0:
  95                 return span("status checker", "ok")
  96             else:
  97                 return span("status checker", st)
  98
  99         if st is None:
 100             return span("status unknown", "?")
 101         elif st == 0:
 102             return span("status passed", "ok")
 103         else:
 104             return span("status failed", st)
 105
 106     ostatus = ""
 107     if "panic" in status.other_failures:
 108         ostatus += "/"+span("status panic", "PANIC")
 109     if "disk full" in status.other_failures:
 110         ostatus += "/"+span("status failed", "disk full")
 111     if "timeout" in status.other_failures:
 112         ostatus += "/"+span("status failed", "timeout")
 113     if "inconsistent test result" in status.other_failures:
 114         ostatus += "/"+span("status failed", "unexpected return code")
 115     bstatus = "/".join([span_status(n, s) for (n, s) in status.stages])
 116     if bstatus == "":
 117         bstatus = "?"
 118     return bstatus + ostatus
 119
 120
 121 def build_status_html(myself, build):
 122     rawstatus = build.status()
 123     status = html_build_status(rawstatus)
 124     return build_link(myself, build.tree, build.host, build.compiler, build.rev, status)
 125
 126
 127 def red_age(age):
 128     """show an age as a string"""
 129     if age > OLDAGE:
 130         return "<span clsas='old'>%s</span>" % util.dhm_time(age)
 131     return util.dhm_time(age)
 132
 133
 134 def build_status_vals(status):
 135     """translate a status into a set of int representing status"""
 136     status = util.strip_html(status)
 137
 138     status = status.replace("ok", "0")
 139     status = status.replace("-", "0")
 140     status = status.replace("?", "0.1")
 141     status = status.replace("PANIC", "1")
 142
 143     return status.split("/")
 144
 145
 146 def view_summary(myself, output_type):
 147     """view build summary"""
 148     i = 0
 149     cols = 2
 150     broken = 0
 151     broken_count = {}
 152     panic_count = {}
 153     host_count = {}
 154
 155     # zero broken and panic counters
 156     for tree in trees:
 157         broken_count[tree] = 0
 158         panic_count[tree] = 0
 159         host_count[tree] = 0
 160
 161     # set up a variable to store the broken builds table's code, so we can
 162     # output when we want
 163     broken_table = ""
 164     last_host = ""
 165
 166     # for the text report, include the current time
 167     if output_type == 'text':
 168         t = time.gmtime()
 169         yield "Build status as of %s\n\n" % t
 170
 171     for host in hosts.values():
 172         for compiler in compilers:
 173             for tree in trees:
 174                 try:
 175                     build = db.get_build(tree, host.name.encode("utf-8"), compiler)
 176                     status = build_status_html(myself, build)
 177                 except data.NoSuchBuildError:
 178                     continue
 179                 age_mtime = build.age_mtime()
 180                 host_count[tree]+=1
 181
 182                 if "status failed" in status:
 183                     broken_count[tree]+=1
 184                     if "PANIC" in status:
 185                         panic_count[tree]+=1
 186
 187     if output_type == 'text':
 188         yield "Build counts:\n"
 189         yield "%-12s %-6s %-6s %-6s\n" % ("Tree", "Total", "Broken", "Panic")
 190     else:
 191         yield "<div id='build-counts' class='build-section'>"
 192         yield "<h2>Build counts:</h2>"
 193         yield "<table class='real'>"
 194         yield "<thead><tr><th>Tree</th><th>Total</th><th>Broken</th><th>Panic</th><th>Test coverage</th></tr></thead>"
 195         yield "<tbody>"
 196
 197     for tree in sorted(trees.keys()):
 198         if output_type == 'text':
 199             yield "%-12s %-6s %-6s %-6s\n" % (tree, host_count[tree],
 200                     broken_count[tree], panic_count[tree])
 201         else:
 202             yield "<tr>"
 203             yield "<td>%s</td>" % tree_link(myself, tree)
 204             yield "<td>%s</td>" % host_count[tree]
 205             yield "<td>%s</td>" % broken_count[tree]
 206             if panic_count[tree]:
 207                     yield "<td class='panic'>"
 208             else:
 209                     yield "<td>"
 210             yield "%d</td>" % panic_count[tree]
 211             try:
 212                 lcov_status = db.lcov_status(tree)
 213             except data.NoSuchBuildError:
 214                 yield "<td></td>"
 215             else:
 216                 if lcov_status is not None:
 217                     yield "<td><a href=\"/lcov/data/%s/%s\">%s %%</a></td>" % (db.LCOVHOST, tree, lcov_status)
 218                 else:
 219                     yield "<td></td>"
 220             yield "</tr>"
 221
 222     if output_type == 'text':
 223         yield "\n"
 224     else:
 225         yield "</tbody></table>"
 226         yield "</div>"
 227
 228
 229 def revision_link(myself, revision, tree):
 230     """return a link to a particular revision"""
 231
 232     revision = revision.lstrip()
 233     if revision == "0":
 234         return "0"
 235
 236     rev_short = revision
 237     if len(revision) == 40:
 238         rev_short = re.sub("(^.{7}).*", "\\1(git)", rev_short)
 239
 240     return "<a href='%s?function=diff;tree=%s;revision=%s' title='View Diff for %s'>%s</a>" % (myself, tree, revision, revision, rev_short)
 241
 242
 243 def tree_link(myself, tree):
 244     # return a link to a particular tree
 245     branch = ""
 246     if tree in trees:
 247         branch = ":%s" % trees[tree].branch
 248
 249     return "<a href='%s?function=Recent+Builds;tree=%s' title='View recent builds for %s'>%s%s</a>" % (myself, tree, tree, tree, branch)
 250
 251
 252 def view_recent_builds(myself, tree, sort_by):
 253     """Draw the "recent builds" view"""
 254     i = 0
 255     cols = 2
 256     broken = 0
 257     last_host = ""
 258     all_builds = []
 259
 260     def status_cmp(a, b):
 261
 262         #put other failure on top
 263         if len(b.other_failures):
 264             return 1
 265         if len(a.other_failures):
 266             return -1
 267
 268         la = len(a.stages)
 269         lb = len(b.stages)
 270         if la > lb:
 271             return 1
 272         elif lb > la:
 273             return -1
 274         else:
 275             sa = a.stages[-1]
 276             sb = b.stages[-1]
 277             if sa[0] == "TEST" and sa[1] == 0:
 278                 return 1
 279             if sb[0] == "TEST" and sb[1] == 0:
 280                 return -1
 281
 282             return (sb[-1] <= sb[-1])
 283
 284     cmp_funcs = {
 285         "revision": lambda a, b: cmp(a[7], b[7]),
 286         "age": lambda a, b: cmp(a[0], b[0]),
 287         "host": lambda a, b: cmp(a[2], b[2]),
 288         "platform": lambda a, b: cmp(a[1], b[1]),
 289         "compiler": lambda a, b: cmp(a[3], b[3]),
 290         "status": lambda a, b: status_cmp(a[6], b[6]),
 291         }
 292
 293     assert tree in trees, "not a build tree"
 294     assert sort_by in cmp_funcs, "not a valid sort"
 295
 296     t = trees[tree]
 297
 298     for host in hosts.values():
 299         for compiler in compilers:
 300             try:
 301                 build = db.get_build(tree, host.name.encode("utf-8"), compiler)
 302                 status = build_status_html(myself, build)
 303             except data.NoSuchBuildError:
 304                 pass
 305             else:
 306                 age_mtime = build.age_mtime()
 307                 age_ctime = build.age_ctime()
 308                 (revision, commit_revision, revision_time) = build.revision_details()
 309                 if commit_revision:
 310                     revision = commit_revision
 311                 if revision:
 312                     all_builds.append([age_ctime,
 313                                         host.platform.encode("utf-8"),
 314                                         "<a href='%s?function=View+Host;host=%s;tree=%s;compiler=%s#%s'>%s</a>"
 315                                             % (myself, host.name.encode("utf-8"),
 316                                                tree, compiler, host.name.encode("utf-8"),
 317                                                host.name.encode("utf-8")),
 318                                         compiler, tree, status, build.status(),
 319                                         revision_link(myself, revision, tree),
 320                                         revision_time])
 321
 322     all_builds.sort(cmp_funcs[sort_by])
 323
 324     sorturl = "%s?tree=%s;function=Recent+Builds" % (myself, tree)
 325
 326     yield "<div id='recent-builds' class='build-section'>"
 327     yield "<h2>Recent builds of %s (%s branch %s)</h2>" % (tree, t.scm, t.branch)
 328     yield "<table class='real'>"
 329     yield "<thead>"
 330     yield "<tr>"
 331     yield "<th><a href='%s;sortby=age' title='Sort by build age'>Age</a></th>" % sorturl
 332     yield "<th><a href='%s;sortby=revision' title='Sort by build revision'>Revision</a></th>" % sorturl
 333     yield "<th>Tree</th>"
 334     yield "<th><a href='%s;sortby=platform' title='Sort by platform'>Platform</a></th>" % sorturl
 335     yield "<th><a href='%s;sortby=host' title='Sort by host'>Host</a></th>" % sorturl
 336     yield "<th><a href='%s;sortby=compiler' title='Sort by compiler'>Compiler</a></th>" % sorturl
 337     yield "<th><a href='%s;sortby=status' title='Sort by status'>Status</a></th>" % sorturl
 338     yield "<tbody>"
 339
 340     for build in all_builds:
 341         yield "<tr>"
 342         yield "<td>%s</td>" % util.dhm_time(build[0])
 343         yield "<td>%s</td>" % build[7]
 344         yield "<td>%s</td>" % build[4]
 345         yield "<td>%s</td>" % build[1]
 346         yield "<td>%s</td>" % build[2]
 347         yield "<td>%s</td>" % build[3]
 348         yield "<td>%s</td>" % build[5]
 349         yield "</tr>"
 350     yield "</tbody></table>"
 351     yield "</div>"
 352
 353
 354 def draw_dead_hosts(output_type, *deadhosts):
 355     """Draw the "dead hosts" table"""
 356
 357     # don't output anything if there are no dead hosts
 358     if len(deadhosts) == 0:
 359         return
 360
 361     # don't include in text report
 362     if output_type == "text":
 363         return
 364
 365     yield "<div class='build-section' id='dead-hosts'>"
 366     yield "<h2>Dead Hosts:</h2>"
 367     yield "<table class='real'>"
 368     yield "<thead><tr><th>Host</th><th>OS</th><th>Min Age</th></tr></thead>"
 369     yield "<tbody>"
 370
 371     for host in deadhosts:
 372         age_ctime = db.host_age(host)
 373         yield "<tr><td>%s</td><td>%s</td><td>%s</td></tr>" %\
 374                 (host,hosts[host].platform.encode("utf-8"),
 375                  util.dhm_time(age_ctime))
 376
 377     yield "</tbody></table>"
 378     yield "</div>"
 379
 380
 381 def show_oldrevs(myself, tree, host, compiler):
 382     """show the available old revisions, if any"""
 383     revs = db.get_old_revs(tree, host, compiler)
 384
 385     if len(revs) == 0:
 386         return
 387
 388     yield "<h2>Older builds:</h2>"
 389
 390     yield "<table class='real'>"
 391     yield "<thead><tr><th>Revision</th><th>Status</th></tr></thead>"
 392     yield "<tbody>"
 393
 394     lastrev = ""
 395     for rev in revs:
 396         s = html_build_status(rev["STATUS"])
 397         revision = rev["REVISION"]
 398         s = s.replace(revision, "0")
 399         if s == lastrev:
 400             continue
 401         lastrev = s
 402         yield "<tr><td>%s</td><td>%s</td></tr>" % (revision_link(myself, revision, tree), build_link(myself, tree, host, compiler, rev["REVISION"], html_build_status(rev["STATUS"])))
 403
 404     if lastrev != "":
 405         # Only print table if there was any actual data
 406         yield "</tbody></table>"
 407
 408
 409 def view_build(myself, tree, host, compiler, rev, plain_logs=False):
 410     """view one build in detail"""
 411     # ensure the params are valid before using them
 412     assert host in hosts, "unknown host %s" % host
 413     assert compiler in compilers, "unknown compiler %s" % compiler
 414     assert tree in trees, "not a build tree %s" % tree
 415
 416     uname = ""
 417     cflags = ""
 418     config = ""
 419     build = db.get_build(tree, host, compiler, rev)
 420     age_mtime = build.age_mtime()
 421     (revision, commit_revision, revision_time) = build.revision_details()
 422     if commit_revision:
 423         revision = commit_revision
 424     status = build_status_html(myself, build)
 425
 426     if rev:
 427         assert re.match("^[0-9a-fA-F]*$", rev)
 428
 429     f = build.read_log()
 430     try:
 431         log = f.read()
 432     finally:
 433         f.close()
 434     f = build.read_err()
 435     try:
 436         err = f.read()
 437     finally:
 438         f.close()
 439
 440     if log:
 441         log = cgi.escape(log)
 442
 443         m = re.search("(.*)", log)
 444         if m:
 445             uname = m.group(1)
 446         m = re.search("CFLAGS=(.*)", log)
 447         if m:
 448             cflags = m.group(1)
 449         m = re.search("configure options: (.*)", log)
 450         if m:
 451             config = m.group(1)
 452
 453     if err:
 454         err = cgi.escape(err)
 455     yield '<h2>Host information:</h2>'
 456
 457     host_web_file = "../web/%s.html" % host
 458     if os.path.exists(host_web_file):
 459         yield util.FileLoad(host_web_file)
 460
 461     yield "<table class='real'>\n"
 462     yield "<tr><td>Host:</td><td><a href='%s?function=View+Host;host=%s;tree=%s;"\
 463           "compiler=%s#'>%s</a> - %s</td></tr>\n" %\
 464             (myself, host, tree, compiler, host, hosts[host].platform.encode("utf-8"))
 465     yield "<tr><td>Uname:</td><td>%s</td></tr>\n" % uname
 466     yield "<tr><td>Tree:</td><td>%s</td></tr>\n" % tree_link(myself, tree)
 467     yield "<tr><td>Build Revision:</td><td>%s</td></tr>\n" % revision_link(myself, revision, tree)
 468     yield "<tr><td>Build age:</td><td><div class='age'>%s</div></td></tr>\n" % red_age(age_mtime)
 469     yield "<tr><td>Status:</td><td>%s</td></tr>\n" % status
 470     yield "<tr><td>Compiler:</td><td>%s</td></tr>\n" % compiler
 471     yield "<tr><td>CFLAGS:</td><td>%s</td></tr>\n" % cflags
 472     yield "<tr><td>configure options:</td><td>%s</td></tr>\n" % config
 473     yield "</table>\n"
 474
 475     yield "".join(show_oldrevs(myself, tree, host, compiler))
 476
 477     # check the head of the output for our magic string
 478     rev_var = ""
 479     if rev:
 480         rev_var = ";revision=%s" % rev
 481
 482     yield "<div id='log'>"
 483
 484     if not plain_logs:
 485         yield "<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s"\
 486               ";compiler=%s%s;plain=true' title='Switch to bland, non-javascript,"\
 487               " unstyled view'>Plain View</a></p>" % (myself, host, tree, compiler, rev_var)
 488
 489         yield "<div id='actionList'>"
 490         # These can be pretty wide -- perhaps we need to
 491         # allow them to wrap in some way?
 492         if err == "":
 493             yield "<h2>No error log available</h2>\n"
 494         else:
 495             yield "<h2>Error log:</h2>"
 496             yield make_collapsible_html('action', "Error Output", "\n%s" % err, "stderr-0", "errorlog")
 497
 498         if log == "":
 499             yield "<h2>No build log available</h2>"
 500         else:
 501             yield "<h2>Build log:</h2>\n"
 502             yield print_log_pretty(log)
 503
 504         yield "<p><small>Some of the above icons derived from the <a href='http://www.gnome.org'>Gnome Project</a>'s stock icons.</small></p>"
 505         yield "</div>"
 506     else:
 507         yield "<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s;"\
 508               "compiler=%s%s' title='Switch to colourful, javascript-enabled, styled"\
 509               " view'>Enhanced View</a></p>" % (myself, host, tree, compiler, rev_var)
 510         if err == "":
 511             yield "<h2>No error log available</h2>"
 512         else:
 513             yield '<h2>Error log:</h2>\n'
 514             yield '<div id="errorLog"><pre>%s</pre></div>' % err
 515         if log == "":
 516             yield '<h2>No build log available</h2>'
 517         else:
 518             yield '<h2>Build log:</h2>\n'
 519             yield '<div id="buildLog"><pre>%s</pre></div>' % log
 520
 521     yield '</div>'
 522
 523
 524 def view_host(myself, output_type, *requested_hosts):
 525     """print the host's table of information"""
 526
 527     if output_type == 'text':
 528         yield "Host summary:\n"
 529     else:
 530         yield "<div class='build-section' id='build-summary'>"
 531         yield '<h2>Host summary:</h2>'
 532
 533     for host in requested_hosts:
 534         assert host in hosts, "unknown host"
 535
 536     for host in requested_hosts:
 537         # make sure we have some data from it
 538         if not db.has_host(host):
 539             if output_type == 'text':
 540                 yield "<!-- skipping %s -->" % host
 541             continue
 542
 543         row = 0
 544
 545         for compiler in compilers:
 546             for tree in sorted(trees.keys()):
 547                 try:
 548                     build = db.get_build(tree, host, compiler)
 549                 except data.NoSuchBuildError:
 550                     pass
 551                 else:
 552                     (revision, commit_revision, revision_time) = build.revision_details()
 553                     if commit_revision:
 554                         revision = commit_revision
 555                     age_mtime = build.age_mtime()
 556                     age_ctime = build.age_ctime()
 557                     warnings = build.err_count()
 558                     status = build_status_html(myself, build)
 559                     if row == 0:
 560                         if output_type == 'text':
 561                             yield "%-12s %-10s %-10s %-10s %-10s\n" % (
 562                                     "Tree", "Compiler", "Build Age", "Status", "Warnings")
 563                         else:
 564                             yield "<div class='host summary'>"
 565                             yield "<a id='host' name='host'/>"
 566                             yield "<h3>%s - %s</h3>" % (host, hosts[host].platform.encode("utf-8"))
 567                             yield "<table class='real'>"
 568                             yield "<thead><tr><th>Target</th><th>Build<br/>Revision</th><th>Build<br />Age</th><th>Status<br />config/build<br />install/test</th><th>Warnings</th></tr></thead>"
 569                             yield "<tbody>"
 570
 571                     if output_type == 'text':
 572                         yield "%-12s %-10s %-10s %-10s %-10s\n" % (
 573                                 tree, compiler, util.dhm_time(age_mtime),
 574                                 util.strip_html(status), warnings)
 575                     else:
 576                         yield "<tr>"
 577                         yield "<td><span class='tree'>" + tree_link(myself, tree) +"</span>/" + compiler + "</td>"
 578                         yield "<td>" + revision_link(myself, revision, tree) + "</td>"
 579                         yield "<td><div class='age'>" + red_age(age_mtime) + "</div></td>"
 580                         yield "<td><div class='status'>%s</div></td>" % status
 581                         yield "<td>%s</td>" % warnings
 582                         yield "</tr>"
 583                     row+=1
 584         if row != 0:
 585             if output_type == 'text':
 586                 yield "\n"
 587             else:
 588                 yield "</tbody></table>"
 589                 yield "</div>"
 590         else:
 591             deadhosts.append(host)
 592
 593     if output_type != 'text':
 594         yield "</div>"
 595
 596     yield "".join(draw_dead_hosts(output_type, *deadhosts))
 597
 598
 599 def subunit_to_buildfarm_result(subunit_result):
 600     if subunit_result == "success":
 601         return "passed"
 602     elif subunit_result == "error":
 603         return "error"
 604     elif subunit_result == "skip":
 605         return "skipped"
 606     elif subunit_result == "failure":
 607         return "failed"
 608     elif subunit_result == "xfail":
 609         return "xfailed"
 610     else:
 611         return "unknown"
 612
 613 def format_subunit_reason(reason):
 614     reason = re.sub("^\[\n+(.*?)\n+\]$", "\\1", reason)
 615     return "<div class=\"reason\">%s</div>" % reason
 616
 617
 618 def print_log_pretty(log):
 619     # prints the log in a visually appealing manner
 620     global indice
 621     indice = 0
 622
 623     # do some pretty printing for the actions
 624     def pretty_print(m):
 625         global indice
 626         output = m.group(1)
 627         actionName = m.group(2)
 628         status = m.group(3)
 629         # handle pretty-printing of static-analysis tools
 630         if actionName == 'cc_checker':
 631              output = print_log_cc_checker(output)
 632
 633         indice += 1
 634         return make_collapsible_html('action', actionName, output, indice, status)
 635
 636     pattern = re.compile("(Running action\s+([\w\-]+)$(?:\s^.*$)*?\sACTION\ (PASSED|FAILED):\ ([\w\-]+)$)", re.M)
 637     log = pattern.sub(pretty_print, log)
 638
 639     # log is already CGI-escaped, so handle '>' in test name by handling &gt
 640     def format_stage(m):
 641         indice += 1
 642         return make_collapsible_html('test', m.group(1), m.group(2), indice, m.group(3))
 643
 644     log = re.sub("""
 645           --==--==--==--==--==--==--==--==--==--==--.*?
 646           Running\ test\ ([\w\-=,_:\ /.&;]+).*?
 647           --==--==--==--==--==--==--==--==--==--==--
 648               (.*?)
 649           ==========================================.*?
 650           TEST\ (FAILED|PASSED|SKIPPED):.*?
 651           ==========================================\s+
 652         """, format_stage, log)
 653
 654     def format_skip_testsuite(m):
 655         global indice
 656         indice += 1
 657         return make_collapsible_html('test', m.group(1), '', indice, 'skipped')
 658
 659     log = re.sub("skip-testsuite: ([\w\-=,_:\ /.&; \(\)]+).*?",
 660             format_skip_testsuite, log)
 661
 662     def format_testsuite(m):
 663         global indice
 664         testName = m.group(1)
 665         content = m.group(2)
 666         status = subunit_to_buildfarm_result(m.group(3))
 667         if m.group(4):
 668             errorReason = format_subunit_reason(m.group(4))
 669         else:
 670             errorReason = ""
 671         indice += 1
 672         return make_collapsible_html('test', testName, content+errorReason, indice, status)
 673
 674     pattern = re.compile("^testsuite: (.+)$\s((?:^.*$\s)*?)testsuite-(\w+): .*?(?:(\[$\s(?:^.*$\s)*?^\]$)|$)", re.M)
 675     log = pattern.sub(format_testsuite, log)
 676
 677     def format_test(m):
 678         id += 1
 679         return make_collapsible_html('test', m.group(1), m.group(2)+format_subunit_reason(m.group(4)), id, subunit_to_buildfarm_result(m.group(3)))
 680
 681     log = re.sub("""
 682           ^test: ([\w\-=,_:\ /.&; \(\)]+).*?
 683           (.*?)
 684           (success|xfail|failure|skip): [\w\-=,_:\ /.&; \(\)]+( \[.*?\])?.*?
 685        """, format_test, log)
 686
 687     return "<pre>%s</pre>" % log
 688
 689
 690 def print_log_cc_checker(input):
 691     # generate pretty-printed html for static analysis tools
 692     output = ""
 693
 694     # for now, we only handle the IBM Checker's output style
 695     if not re.search("^BEAM_VERSION", input):
 696         return "here"
 697         return input
 698
 699     content = ""
 700     inEntry = False
 701     title = None
 702     status = None
 703
 704     for line in input.splitlines():
 705         # for each line, check if the line is a new entry,
 706         # otherwise, store the line under the current entry.
 707
 708         if line.startswith("-- "):
 709             # got a new entry
 710             if inEntry:
 711                 output += make_collapsible_html('cc_checker', title, content, id, status)
 712             else:
 713                 output += content
 714
 715             # clear maintenance vars
 716             (inEntry, content) = (True, "")
 717
 718             # parse the line
 719             m = re.match("^-- ((ERROR|WARNING|MISTAKE).*?)\s+&gt;&gt;&gt;([a-zA-Z0-9]+_(\w+)_[a-zA-Z0-9]+)", line)
 720
 721             # then store the result
 722             (title, status, id) = ("%s %s" % (m.group(1), m.group(4)), m.group(2), m.group(3))
 723         elif line.startswith("CC_CHECKER STATUS"):
 724             if inEntry:
 725                 output += make_collapsible_html('cc_checker', title, content, id, status)
 726
 727             inEntry = False
 728             content = ""
 729
 730         # not a new entry, so part of the current entry's output
 731         content += "%s\n" % line
 732
 733     output += content
 734
 735     # This function does approximately the same as the following, following
 736     # commented-out regular expression except that the regex doesn't quite
 737     # handle IBM Checker's newlines quite right.
 738     #   $output =~ s{
 739     #                 --\ ((ERROR|WARNING|MISTAKE).*?)\s+
 740     #                        &gt;&gt;&gt
 741     #                 (.*?)
 742     #                 \n{3,}
 743     #               }{make_collapsible_html('cc_checker', "$1 $4", $5, $3, $2)}exgs
 744     return output
 745
 746
 747 def make_collapsible_html(type, title, output, id, status=""):
 748     """generate html for a collapsible section
 749
 750     :param type: the logical type of it. e.g. "test" or "action"
 751     :param title: the title to be displayed
 752     """
 753     if ((status == "" or "failed" == status.lower())):
 754         icon = 'icon_hide_16.png'
 755     else:
 756         icon = 'icon_unhide_16.png'
 757
 758     # trim leading and trailing whitespace
 759     output = output.strip()
 760
 761     # note that we may be inside a <pre>, so we don't put any extra whitespace
 762     # in this html
 763     ret = "<div class='%s unit %s' id='%s-%s'>" % (type, status, type, id)
 764     ret += "<a href=\"javascript:handle('%s');\">" % id
 765     ret += "<img id='img-%s' name='img-%s' alt='%s' src='%s' />" %(id, id, status, icon)
 766     ret += "<div class='%s title'>%s</div></a>" % (type, title)
 767     #ret += " "
 768     ret += "<div class='%s status %s'>%s</div>" % (type, status, status)
 769     ret += "<div class='%s output' id='output-%s'>" % (type, id)
 770     if output and len(output):
 771         ret += "<pre>%s</pre>>" % (output)
 772     ret += "</div></div>"
 773     return ret
 774
 775 def main_menu():
 776     """main page"""
 777
 778     yield "<form method='GET'>"
 779     yield "<div id='build-menu'>"
 780     yield "<select name='host'>"
 781     for name, host in hosts.iteritems():
 782         yield "<option value='%s'>%s -- %s</option>\n" % (name, host.platform.encode("utf-8"), name)
 783     yield "</select>"
 784     yield "<select name='tree'>"
 785     for tree, t in trees.iteritems():
 786         yield "<option value='%s'>%s:%s</option>\n" % (tree, tree, t.branch)
 787     yield "</select>"
 788     yield "<select name='compiler'>"
 789     for compiler in compilers:
 790         yield "<option>%s</option>\n" % compiler
 791     yield "</select>"
 792     yield "<br/>"
 793     yield "<input type='submit' name='function' value='View Build'/>"
 794     yield "<input type='submit' name='function' value='View Host'/>"
 795     yield "<input type='submit' name='function' value='Recent Checkins'/>"
 796     yield "<input type='submit' name='function' value='Summary'/>"
 797     yield "<input type='submit' name='function' value='Recent Builds'/>"
 798     yield "</div>"
 799     yield "</form>"
 800
 801
 802 def diff_pretty(diff):
 803     """pretty up a diff -u"""
 804     # FIXME: JRV 20101109 Use pygments for this
 805     ret = ""
 806     lines = diff.splitlines()
 807
 808     line_types = {
 809             '^diff.*': 'diff_diff',
 810             '^=.*': 'diff_separator',
 811             '^Index:.*': 'diff_index',
 812             '^index.*': 'diff_index',
 813             '^\-.*': 'diff_removed',
 814             '^\+.*': 'diff_added',
 815             '^@@.*': 'diff_fragment_header'
 816             }
 817
 818     for line in lines:
 819         for r in line_types.iterkeys():
 820             if r in line:
 821                 line = "<span class=\"%s\">%s</span>" % (line_types[r], line)
 822                 continue
 823         ret += line
 824
 825     return ret
 826
 827
 828 def web_paths(t, paths):
 829     """change the given source paths into links"""
 830     ret = ""
 831
 832     fmt = None
 833
 834     if t.scm == "git":
 835         r = t.repo
 836         s = t.subdir
 837         b = t.branch
 838         fmt = " <a href=\"%s/?p=%s;a=history;f=%s%%s;h=%s;hb=%s\">%%s</a>" % (GITWEB_BASE, r, s, b, b)
 839     else:
 840         return paths
 841
 842     for m in re.finditer("\s*([^\s]+)", paths):
 843         ret += fmt % (m.group(1), m.group(1))
 844
 845     return ret
 846
 847
 848 def history_row_html(myself, entry, tree):
 849     """show one row of history table"""
 850     msg = cgi.escape(entry["MESSAGE"])
 851     t = time.asctime(time.gmtime(entry["DATE"]))
 852     age = util.dhm_time(time()-entry["DATE"])
 853
 854     t = t.replace(" ", "&nbsp;")
 855
 856     yield """
 857 <div class=\"history_row\">
 858     <div class=\"datetime\">
 859         <span class=\"date\">%s</span><br />
 860         <span class=\"age\">%s ago</span>""" % (t, age)
 861     if entry["REVISION"]:
 862         yield " - <span class=\"revision\">%s</span><br/>" % entry["REVISION"]
 863         revision_url = "revision=%s" % entry["REVISION"]
 864     else:
 865         revision_url = "author=%s" % entry["AUTHOR"]
 866     yield """    </div>
 867     <div class=\"diff\">
 868         <span class=\"html\"><a href=\"%s?function=diff;tree=%s;date=%s;%s\">show diffs</a></span>
 869     <br />
 870         <span class=\"text\"><a href=\"%s?function=text_diff;tree=%s;date=%s;%s\">download diffs</a></span>
 871         <br />
 872         <div class=\"history_log_message\">
 873             <pre>%s</pre>
 874         </div>
 875     </div>
 876     <div class=\"author\">
 877     <span class=\"label\">Author: </span>%s
 878     </div>""" % (myself, tree, entry["DATE"], revision_url,
 879                  myself, tree, entry["DATE"], revision_url,
 880                  msg, entry["AUTHOR"])
 881
 882     t = db.trees.get(tree)
 883
 884     if t is None:
 885         yield "</div>"
 886         return
 887
 888     if entry["FILES"]:
 889         yield "<div class=\"files\"><span class=\"label\">Modified: </span>"
 890         yield web_paths(t, entry["FILES"])
 891         yield "</div>\n"
 892
 893     if entry["ADDED"]:
 894         yield "<div class=\"files\"><span class=\"label\">Added: </span>"
 895         yield web_paths(t, entry["ADDED"])
 896         yield "</div>\n"
 897
 898     if entry["REMOVED"]:
 899         yield "<div class=\"files\"><span class=\"label\">Removed: </span>"
 900         yield web_paths(t, entry["REMOVED"])
 901         yield "</div>\n"
 902
 903     yield "</div>\n"
 904
 905 def history_row_text(entry, tree):
 906     """show one row of history table"""
 907     msg = cgi.escape(entry["MESSAGE"])
 908     t = time.asctime(time.gmtime(entry["DATE"]))
 909     age = util.dhm_time(time()-entry["DATE"])
 910
 911     yield "Author: %s\n" % entry["AUTHOR"]
 912     if entry["REVISION"]:
 913         yield "Revision: %s\n" % entry["REVISION"]
 914     yield "Modified: %s\n" % entry["FILES"]
 915     yield "Added: %s\n" % entry["ADDED"]
 916     yield "Removed: %s\n" % entry["REMOVED"]
 917     yield "\n\n%s\n\n\n" % msg
 918
 919
 920 def show_diff(cmd, diff, text_html):
 921     if text_html == "html":
 922         diff = cgi.escape(diff)
 923         diff = diff_pretty(diff)
 924         ret = "<!-- %s -->\n" % cmd
 925         ret += "<pre>%s</pre>\n" % diff
 926         return ret
 927     else:
 928         return "%s\n" % diff
 929
 930
 931 def buildApp(environ, start_response):
 932     form = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)
 933     fn_name = get_param(form, 'function') or ''
 934     myself = wsgiref.util.application_uri(environ)
 935
 936     if standalone and environ['PATH_INFO']:
 937         dir = os.path.join(os.path.dirname(__file__))
 938         static_file = "%s/%s" % (dir, environ['PATH_INFO'])
 939         if os.path.exists(static_file):
 940             tab = environ['PATH_INFO'].split('.')
 941             if len(tab) > 1:
 942                 extension = tab[-1]
 943                 import mimetypes
 944                 mimetypes.init()
 945                 type = mimetypes.types_map[".%s" % extension]
 946                 start_response('200 OK', [('Content-type', type)])
 947                 data = open(static_file, 'rb').read()
 948                 yield data
 949                 return
 950
 951     if fn_name == 'text_diff':
 952         start_response('200 OK', [('Content-type', 'application/x-diff')])
 953         (title, entry, tree, diffs) = history.diff(get_param(form, 'author'),
 954               get_param(form, 'date'),
 955               get_param(form, 'tree'),
 956               get_param(form, 'revision'))
 957         yield "".join(history_row_text(entry, tree))
 958         for (cmd, diff) in diffs:
 959             yield show_diff(cmd, diff, "text")
 960     elif fn_name == 'Text_Summary':
 961         start_response('200 OK', [('Content-type', 'text/plain')])
 962         yield "".join(view_summary(myself, 'text'))
 963     else:
 964         start_response('200 OK', [('Content-type', 'text/html')])
 965
 966         yield "<html>"
 967         yield "  <head>"
 968         yield "    <title>samba.org build farm</title>"
 969         yield "    <script language='javascript' src='/build_farm.js'></script>"
 970         yield "    <meta name='keywords' contents='Samba SMB CIFS Build Farm'/>"
 971         yield "    <meta name='description' contents='Home of the Samba Build Farm, the automated testing facility.'/>"
 972         yield "    <meta name='robots' contents='noindex'/>"
 973         yield "    <link rel='stylesheet' href='/build_farm.css' type='text/css' media='all'/>"
 974         if standalone:
 975             yield "    <link rel='stylesheet' href='common.css' type='text/css' media='all'/>"
 976         else:
 977             yield "    <link rel='stylesheet' href='http://master.samba.org/samba/style/common.css' type='text/css' media='all'/>"
 978         yield "    <link rel='shortcut icon' href='http://www.samba.org/samba/images/favicon.ico'/>"
 979         yield "  </head>"
 980         yield "<body>"
 981
 982         yield util.FileLoad(os.path.join(webdir, "header2.html"))
 983         yield "".join(main_menu())
 984         yield util.FileLoad(os.path.join(webdir, "header3.html"))
 985         if fn_name == "View_Build":
 986             plain_logs = (get_param(form, "plain") is not None and get_param(form, "plain").lower() in ("yes", "1", "on", "true", "y"))
 987             yield "".join(view_build(myself, get_param(form, "tree"), get_param(form, "host"),
 988                 get_param(form, "compiler"), get_param(form, "revision"), plain_logs))
 989         elif fn_name == "View_Host":
 990             yield "".join(view_host(myself, "html", get_param(form, 'host')))
 991         elif fn_name == "Recent_Builds":
 992             yield "".join(view_recent_builds(myself, get_param(form, "tree"), get_param(form, "sortby") or "revision"))
 993         elif fn_name == "Recent_Checkins":
 994             # validate the tree
 995             tree =  get_param(form, "tree")
 996             t = db.trees[tree]
 997             authors = set(["ALL"])
 998             authors.update(history.authors(tree))
 999
1000             yield "<h2>Recent checkins for %s (%s branch %s)</h2>\n" % (
1001                 tree, t.scm, t.branch)
1002             yield "<form method='GET'>"
1003             yield "Select Author: "
1004             yield "<select name='author'>"
1005             for name in sorted(authors):
1006                 yield "<option>%s</option>" % name
1007             yield "</select>"
1008             yield "<input type='submit' name='sub_function' value='Refresh'/>"
1009             yield "<input type='hidden' name='tree' value='%s'/>" % tree
1010             yield "<input type='hidden' name='function', value='Recent Checkins'/>"
1011             yield "</form>"
1012
1013             for entry, tree in history.history(get_param(form, 'tree'), get_param(form, 'author')):
1014                 yield "".join(history_row_html(myself, entry, tree))
1015             yield "\n"
1016         elif fn_name == "diff":
1017             (title, entry, tree, diffs) = history.diff(get_param(form, 'author'),
1018                     get_param(form, 'date'),
1019                     get_param(form, 'tree'),
1020                     get_param(form, 'revision'))
1021             yield "<h2>%s</h2>" % title
1022             yield "".join(history_row_html(myself, entry, tree))
1023             for (cmd, diff) in diffs:
1024                 yield show_diff(cmd, diff, "html")
1025         elif os.getenv("PATH_INFO") not in (None, "", "/"):
1026             paths = os.getenv("PATH_INFO").split('/')
1027             if paths[1] == "recent":
1028                 yield "".join(view_recent_builds(myself, paths[2], get_param(form, 'sortby') or 'revision'))
1029             elif paths[1] == "host":
1030                 yield "".join(view_host(myself, "html", paths[2]))
1031         else:
1032             yield "".join(view_summary(myself, 'html'))
1033         yield util.FileLoad(os.path.join(webdir, "footer.html"))
1034         yield "</body>"
1035         yield "</html>"
1036
1037 if __name__ == '__main__':
1038     import optparse
1039     parser = optparse.OptionParser("[options]")
1040     parser.add_option("--standalone", help="Run as standalone server (useful for debugging)", action="store_true")
1041     opts, args = parser.parse_args()
1042     if opts.standalone:
1043         standalone = 1
1044         from wsgiref.simple_server import make_server
1045         httpd = make_server('localhost', 8000, buildApp)
1046         print "Serving on port 8000..."
1047         httpd.serve_forever()
1048     else:
1049         import wsgiref.handlers
1050         handler = wsgiref.handlers.CGIHandler()
1051         handler.run(buildApp)