build_test.fns

   1 #!/bin/sh -*- mode: shell-script; -*-
   2
   3 # build_farm -- distributed build/test architecture for samba, rsync, etc
   4
   5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
   6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
   7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
   8
   9 # default maximum runtime for any command
  10 MAXTIME=25200 # 7 hours
  11 SMBD_MAXTIME=18000 # 5 hours for a samba process ..
  12 # default maximum memory size (100M) for any command
  13 MAXMEM=100000
  14 RUN_FROM_BUILD_FARM=yes
  15 export RUN_FROM_BUILD_FARM
  16 export MAXTIME SMBD_MAXTIME
  17
  18 deptrees="";
  19
  20 build_test_fns_id='$Id$'
  21
  22 copy_dir() {
  23         Tsrc=$1
  24         Tdst=$2
  25         pwd
  26         echo rsync -a --delete $Tsrc/ $Tdst
  27         rsync -a --delete $Tsrc/ $Tdst || return 1
  28         return 0
  29 }
  30
  31 #############################
  32 # build a signature of a tree, used to see if we
  33 # need to rebuild
  34 #############################
  35
  36 sum_tree() {
  37         sum_tree_test_root=$1
  38         sum_tree_tree=$2
  39         sum_tree_sum=$3
  40         sum_tree_scm=$4
  41         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
  42         sum build_test build_test.fns >> $sum_tree_sum
  43
  44         if [ -f "$host.fns" ]; then
  45                 sum $host.fns >> $sum_tree_sum
  46         else
  47                 sum generic.fns >> $sum_tree_sum
  48         fi
  49
  50         if [ -f "$test_root/$tree.$scm" ]; then
  51                 sum "$test_root/$tree.$scm" >> $sum_tree_sum
  52         fi
  53
  54         for d in $deptrees; do
  55                 dscm=`choose_scm "$d"`
  56                 if [ -f "$test_root/$d.$dscm" ]; then
  57                 sum "$test_root/$d.$dscm" >> $sum_tree_sum
  58                 fi
  59         done
  60 }
  61
  62 #############################
  63 # send the logs to the master site
  64 #############################
  65
  66 send_logs() {
  67         if [ "$nologreturn" = "yes" ]; then
  68                 echo "skipping log transfer"
  69         else
  70                 log="$1"
  71                 err="$2"
  72                 shift
  73                 shift
  74                 chmod 0644 "$log" "$err"
  75
  76                 # xargs -i is implemented differently or not at all.
  77                 # GNU xargs did not implement "-I" until 4.2.9:
  78                 xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
  79                 status=$?
  80                 if [ x"$status" = x"0" ]; then
  81                         XARGS_IS_GNU=yes
  82                 fi
  83
  84                 if [ x"$XARGS_IS_GNU" = x"yes" ]; then
  85                         XARGS_I="xargs -i"
  86                 else
  87                         XARGS_I="xargs -I '{}'"
  88                 fi
  89
  90                 find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
  91                 find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
  92
  93                 rsync $* -c -q --password-file=.password -z --timeout=200 \
  94                         "$log" "$err" $host@build.samba.org::build_farm_data/
  95         fi
  96 }
  97
  98 #############################
  99 # send the logs when they haven't changed
 100 # the aim is to just update the servers timestamp.
 101 # sending with a very large rsync block size does this
 102 # with minimal network traffic
 103 #############################
 104
 105 send_logs_skip() {
 106         touch "$1" "$2"
 107         send_logs "$1" "$2" -B 10000000
 108 }
 109
 110 ############################
 111 # fetch the latest copy of the tree
 112 ############################
 113
 114 fetch_tree() {
 115         if [ "$norsync" = "yes" ]; then
 116                 echo "skipping tree transfer"
 117         else
 118                 fetchtree=$1
 119                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
 120                         --delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
 121                         samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
 122                         echo "transferred $fetchtree OK"
 123                 else
 124                         echo "transfer of $fetchtree failed code $?"
 125                         return 1
 126                 fi
 127         fi
 128         return 0
 129 }
 130
 131 ############################
 132 # fetch the latest copy of the rev meta info
 133 ############################
 134
 135 fetch_revinfo() {
 136         tree=$1
 137         scm=$2
 138
 139         test -z "$scm" && return 1
 140         test x"$scm" = x"unknown" && return 1
 141         test x"$scm" = x"cvs" && return 1
 142
 143         if [ "$norsync" = "yes" ]; then
 144                 echo "skipping .revinfo.$scm transfer"
 145         else
 146                 if [ -r $test_root/$tree.$scm ]; then
 147                         [ -f $test_root/$tree.$scm.old ] && rm -f $test_root/$tree.$scm.old
 148                         [ -f $test_root/$tree.$scm ] && mv $test_root/$tree.$scm $test_root/$tree.$scm.old
 149                 fi
 150                 rsync -q --timeout=200 -clz --ignore-errors \
 151                         samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
 152         fi
 153         if [ -r $test_root/$tree.$scm ]; then
 154                 return 0;
 155         fi
 156         return 1
 157 }
 158
 159 ############################
 160 # choose the scm that is used for the given project
 161 ############################
 162
 163 choose_scm() {
 164         tree=$1
 165
 166         case "$tree" in
 167                         samba* | rsync | libreplace | talloc | tdb* | ldb | pidl | ccache* | waf*)
 168                         echo "git"
 169                                 return 0
 170                         ;;
 171         esac
 172
 173         echo "svn"
 174         return 0
 175 }
 176
 177 locknesting=0
 178
 179 ############################
 180 # grab a lock file. Not atomic, but close :)
 181 # tries to cope with NFS
 182 ############################
 183
 184 lock_file() {
 185                 if [ -z "$lock_root" ]; then
 186                         lock_root=`pwd`;
 187                 fi
 188
 189                 lckf="$lock_root/$1"
 190                 machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
 191                 pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
 192
 193                 if [ "$pid" = "$$" ]; then
 194                         locknesting=`expr $locknesting + 1`
 195                         echo "lock nesting now $locknesting"
 196                         return 0
 197                 fi
 198
 199                 # We need to assert that the file is > 0 size, as otherwise we never
 200                 # recover from disk full situations
 201                 if test -f "$lckf" && test -s "$lckf"; then
 202                         test x$machine = x$host || {
 203                                 echo "lock file $lckf is valid for other machine $machine"
 204                                 return 1
 205                         }
 206
 207                         kill -0 $pid && {
 208                                 echo "lock file $lckf is valid for process $pid"
 209                                 return 1
 210                         }
 211
 212                         echo "stale lock file $lckf for $machine:$pid"
 213                         cat "$lckf"
 214                         /bin/rm -f "$lckf"
 215                 fi
 216                 echo "$host:$$" > "$lckf"
 217                 return 0
 218 }
 219
 220 ############################
 221 # unlock a lock file
 222 ############################
 223
 224 unlock_file() {
 225         if [ -z "$lock_root" ]; then
 226                 lock_root=`pwd`;
 227         fi
 228         if [ "$locknesting" != "0" ]; then
 229                 locknesting=`expr $locknesting - 1`
 230                 echo "lock nesting now $locknesting"
 231         else
 232                 lckf="$lock_root/$1"
 233                 /bin/rm -f "$lckf"
 234         fi
 235 }
 236
 237 ############################
 238 # run make, and print trace
 239 ############################
 240
 241 do_make() {
 242         # work out correct make command
 243         case "$tree" in
 244             waf*)
 245                 MAKECOMMAND="./waf"
 246                 ;;
 247             *)
 248                 MAKECOMMAND="$MAKE"
 249                 if [ x"$MAKECOMMAND" = x ]; then
 250                     MAKECOMMAND=make
 251                 fi
 252                 ;;
 253         esac
 254
 255         MMTIME=$MAXTIME
 256         # some trees don't need as much time
 257         case "$tree" in
 258                 rsync | tdb* | talloc | libreplace | ccache* | waf*)
 259                         if [ "$compiler" != "checker" ]; then
 260                                 MMTIME=`expr $MMTIME / 5`
 261                         fi
 262                 ;;
 263         esac
 264
 265         # special build for some trees
 266         case "$tree" in
 267             waf*)
 268                 ./waf distclean && ./waf configure build
 269                 ;;
 270         esac
 271
 272
 273         for t in $*; do
 274                 if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
 275                         echo "$MAKECOMMAND $t"
 276                         $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
 277                         status=$?
 278                 else
 279                         # we can parallelize everything and all targets
 280                         if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
 281                                 echo "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
 282                                 $builddir/timelimit $MMTIME "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
 283                                 status=$?
 284                         else
 285                                 echo "$MAKECOMMAND $t"
 286                                 $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
 287                                 status=$?
 288                         fi
 289                 fi
 290
 291                 if [ $status != 0 ]; then
 292                         case "$t" in
 293                                 test|check|installcheck)
 294                                         ;;
 295                                 *)
 296                                         #run again with V=1, so we see failed commands
 297                                         $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t" V=1
 298                                         status=$?
 299                                         ;;
 300                         esac
 301                 fi
 302
 303                 if [ $status != 0 ]; then
 304                         return $status;
 305                 fi
 306
 307         done
 308
 309         return 0
 310 }
 311
 312
 313 ############################
 314 # do the coverage report
 315 ############################
 316
 317 action_lcovreport() {
 318         if [ "$LCOV_REPORT" = "yes" ]; then
 319                 case "$tree" in
 320                 lorikeet-heimdal*)
 321                         lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
 322                         ;;
 323                 samba_3_master*)
 324                         lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
 325                         ;;
 326                 samba_4*|tdb*|talloc|ldb|libreplace)
 327                         lcov --base-directory $builddir/bin --directory $builddir/bin --capture --output-file $builddir/$tree.lcov.info
 328                         ;;
 329                 waf)
 330                         lcov --base-directory $builddir/demos --directory $builddir/demos --capture --output-file $builddir/$tree.lcov.info
 331                         ;;
 332                 *)
 333                         lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
 334                         ;;
 335                 esac
 336                 genhtml -o $builddir/coverage $builddir/$tree.lcov.info
 337                 rc=$?
 338                 echo "return code: $rc"
 339         else
 340                 echo "LCOV_REPORT not set and lcovreport asked"
 341                 echo "Most probably an error please fix !"
 342                 return 1
 343         fi
 344 }
 345
 346 action_callcatcherreport() {
 347         if [ "$CALLCATCHER_REPORT" = "yes" ]; then
 348                 case "$tree" in
 349                 tdb*|talloc|ldb)
 350                         callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/unused-fns.txt
 351                         ;;
 352                 samba_3_master|samba_4*)
 353                         callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/all-unused-fns.txt
 354                         grep -v -f $srcdir/callcatcher-exceptions.grep $builddir/coverage/all-unused-fns.txt > $builddir/coverage/unused-fns.txt
 355                         ;;
 356                 esac
 357                 rc=$?
 358                 echo "return code: $rc"
 359         else
 360                 echo "CALLCATCHER_REPORT not set and callcatcher asked"
 361                 echo "Most probably an error please fix !"
 362                 return 1
 363         fi
 364 }
 365
 366
 367 ############################
 368 # configure the tree
 369 ############################
 370
 371 action_configure() {
 372         # special handling for some trees
 373         case "$tree" in
 374             waf*)
 375                 $builddir/timelimit $MAXTIME ./waf configure
 376                 cstatus=$?
 377                 echo "CONFIGURE STATUS: $cstatus"
 378                 return $cstatus
 379                 ;;
 380         esac
 381
 382         if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
 383                 perl $srcdir/Makefile.PL PREFIX="$prefix"
 384                 cstatus=$?
 385                 echo "CONFIGURE STATUS: $cstatus"
 386                 return $cstatus;
 387         fi
 388
 389         if [ ! -x $srcdir/configure ]; then
 390                 ls -l $srcdir/configure
 391                 echo "$srcdir/configure is missing"
 392                 cstatus=255
 393                 echo "CONFIGURE STATUS: $cstatus"
 394                 return $cstatus;
 395         fi
 396
 397         echo "CFLAGS=$CFLAGS"
 398         echo configure options: $config_and_prefix
 399         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
 400
 401         CC="$CCACHE $compiler"
 402         export CC
 403         $builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
 404         cstatus=$?
 405
 406         if [ x"$cstatus" != x"0" ]; then
 407                 if [ -f config.log ]; then
 408                         echo "contents of config.log:"
 409                         cat config.log
 410                 fi
 411
 412                 # Waf style
 413                 if [ -f bin/config.log ]; then
 414                         echo "contents of config.log:"
 415                         cat bin/config.log
 416                 fi
 417         fi
 418         echo "CONFIGURE STATUS: $cstatus"
 419         return $cstatus
 420 }
 421
 422 ############################
 423 # show the configure log
 424 ############################
 425
 426 action_config_log() {
 427
 428         log_files="config.log bin/config.log"
 429         for f in $log_files; do
 430                 if [ -f $f ]; then
 431                         echo "contents of config.log:"
 432                         cat $f
 433                         return 0
 434                 fi
 435         done
 436         return 0
 437 }
 438
 439 ############################
 440 # show the config.h
 441 ############################
 442
 443 action_config_header() {
 444         hdr_files="config.h include/config.h include/autoconf/config.h bin/default/config.h bin/default/include/config.h bin/default/source3/include/config.h"
 445         for h in $hdr_files; do
 446                 if [ -f $h ]; then
 447                         echo "contents of $h:"
 448                         cat $h
 449                         return 0
 450                 fi
 451         done
 452
 453         return 0
 454 }
 455
 456
 457
 458 ############################
 459 # build the tree
 460 ############################
 461 action_build() {
 462         case "$tree" in
 463         samba_4*)
 464                 do_make everything
 465                 bstatus=$?
 466                 ;;
 467         samba_3*)
 468                 do_make everything torture
 469                 bstatus=$?
 470                 ;;
 471         waf*)
 472                 do_make build
 473                 bstatus=$?
 474                 ;;
 475         *)
 476                 do_make all
 477                 bstatus=$?
 478                 ;;
 479         esac
 480
 481         echo "BUILD STATUS: $bstatus"
 482
 483         return $bstatus
 484 }
 485
 486 ############################
 487 # show static analysis results
 488 ############################
 489
 490 action_cc_checker() {
 491
 492         # default to passing the cc_checker
 493         cccstatus=0
 494
 495         if [ -f ibm_checker.out ]; then
 496                 cat ibm_checker.out
 497                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
 498         fi
 499
 500         echo "CC_CHECKER STATUS: $cccstatus"
 501         return $cccstatus;
 502 }
 503
 504 ############################
 505 # install the tree
 506 ############################
 507
 508 action_install() {
 509         if [ -d $prefix ]; then
 510                 if [ "$noclean" != "yes" ]; then
 511                         rm -rf $prefix
 512                 fi
 513         fi
 514
 515         do_make install
 516         istatus=$?
 517         echo "INSTALL STATUS: $istatus"
 518         return $istatus;
 519 }
 520
 521 ############################
 522 # test the tree
 523 action_test_samba() {
 524         do_make test
 525         totalstatus=$?
 526
 527         # if we produced a test summary then show it
 528         [ -f st/summary ] && {
 529                 echo "TEST SUMMARY"
 530                 cat st/summary
 531         }
 532
 533         return "$totalstatus"
 534 }
 535
 536 action_test_generic() {
 537         CC="$compiler"
 538         export CC
 539         do_make installcheck
 540         totalstatus=$?
 541         echo "TEST STATUS: $totalstatus"
 542         return "$totalstatus"
 543 }
 544
 545 action_test_lorikeet_heimdal() {
 546         CC="$compiler"
 547         export CC
 548         SOCKET_WRAPPER_DIR=`pwd`/sw
 549         mkdir $SOCKET_WRAPPER_DIR
 550         export SOCKET_WRAPPER_DIR
 551         do_make check
 552         totalstatus=$?
 553         SOCKET_WRAPPER_DIR=
 554         export SOCKET_WRAPPER_DIR
 555         echo "TEST STATUS: $totalstatus"
 556         return "$totalstatus"
 557 }
 558
 559
 560 #############################
 561 # attempt some basic tests of functionaility
 562 # starting as basic as possible, and getting incresingly complex
 563 #############################
 564
 565 action_test() {
 566         # Samba needs crufty code of its own for backward
 567         # compatiblity.  I think a better way to do this in the future
 568         # is to just call 'make installcheck'.
 569         case "$tree" in
 570         samba*|smb-build|pidl)
 571                 action_test_samba
 572                 ;;
 573         lorikeet-heimdal*)
 574                 action_test_lorikeet_heimdal
 575                 ;;
 576         *)
 577                 action_test_generic
 578                 ;;
 579         esac
 580 }
 581
 582 #############################
 583 # Do nothing (needed if we have nothing to do for extra_actions)
 584 #############################
 585
 586 action_none() {
 587     return 0;
 588 }
 589
 590 ###########################
 591 # do a test build of a particular tree
 592 # This is the master function called by generic.fns or
 593 # host.fns
 594 ###########################
 595
 596 test_tree() {
 597         tree=$1
 598         source=$2
 599         compiler="$3"
 600         shift
 601         shift
 602         shift
 603         echo "Starting to deal with tree $tree with compiler $compiler"
 604         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null 2>/dev/null; then
 605                 CCACHE="ccache"
 606                 export CCACHE
 607         else
 608                 CCACHE=""
 609         fi
 610
 611         # limit our resource usage
 612         ulimit -t $MAXTIME 2> /dev/null
 613
 614         # max mem size 100M
 615         ulimit -m $MAXMEM 2> /dev/null
 616
 617         # max file size 100M
 618         # darn, this affects sparse files too! disable it
 619         # ulimit -f 100000 2> /dev/null
 620
 621         # try and limit the number of open files to 500, up from 250. That means we'll discover
 622         # fd leaks faster while allowing our very complex make test to run
 623         ulimit -n 500 2> /dev/null
 624
 625         # Keep stuff private
 626         umask 077
 627
 628         if [ -z "$test_root" ]; then
 629                 test_root=`pwd`
 630         fi
 631
 632         log="build.$tree.$host.$compiler.log"
 633         err="build.$tree.$host.$compiler.err"
 634         sum="build.$tree.$host.$compiler.sum"
 635         lck="build.$tree.lck"
 636                 srcdir="$test_root/$tree/$source"
 637
 638         lock_file "$lck" || {
 639                 return
 640         }
 641
 642         # work out what other trees this package depends on
 643         deptrees=""
 644         case "$tree" in
 645                 samba-gtk)
 646                 deptrees="samba_4_0_test"
 647                 ;;
 648         esac
 649
 650         scm=`choose_scm "$tree"`
 651
 652         # pull the entries, if any
 653         # Remove old .svn or .git files
 654         # Move the current .svn org .git to .svn.old or
 655         # .git.old then fetch the new from rsync
 656         if fetch_revinfo "$tree" "$scm"; then
 657                 for d in $deptrees; do
 658                         # If there is dependency substree(s) we add info
 659                         # from the dependency tree so that we
 660                         # can rebuild in case one of them has changed
 661                         dscm=`choose_scm "$d"`
 662                         if [ -f "$test_root/$d.$dscm" ]; then
 663                                 if [ "$d" != "$tree" ]; then
 664                                         cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
 665                                 fi
 666                         fi
 667                 done
 668                 [ -f $test_root/$tree.$compiler.$scm.old ] && rm -f $test_root/$tree.$compiler.$scm.old
 669                 [ -f $test_root/$tree.$compiler.$scm ] && mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
 670                 [ -f $test_root/$tree.$scm ] && cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
 671
 672                 if [ -f $test_root/$tree.$compiler.$scm.old ] && \
 673                                 cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
 674
 675                         echo "skip: $tree.$compiler nothing changed in $scm"
 676                         cd $test_root
 677                         send_logs_skip "$log" "$err"
 678                         unlock_file "$lck"
 679                         return
 680                 fi
 681         fi
 682
 683         # pull the tree
 684         fetch_tree "$tree" || {
 685                 cd $test_root
 686                 unlock_file "$lck"
 687                 return
 688         }
 689
 690         # check for essential files
 691         case "$tree" in
 692             pidl)
 693                 # no generated files
 694                 ;;
 695             waf*)
 696                 if [ ! -x $srcdir/waf ]; then
 697                     echo "skip: $tree.$compiler waf not present, try again next time!"
 698                     cd $test_root
 699                     unlock_file "$lck"
 700                     return
 701                 fi
 702                 ;;
 703             *)
 704                 if [ ! -x $srcdir/configure ]; then
 705                     echo "skip: $tree.$compiler configure not present, try again next time!"
 706                     cd $test_root
 707                     unlock_file "$lck"
 708                     return
 709                 fi
 710                 ;;
 711         esac
 712
 713         echo "Starting build of $tree.$compiler in process $$ at `date`"
 714
 715
 716         # Parameters for the build depending on the tree
 717         case "$tree" in
 718                 *)
 719                         builddir=$srcdir
 720                         export builddir
 721                         ;;
 722         esac
 723
 724         #Fix the user
 725         if [ ! x$USER = x"" ]; then
 726                 whoami=$USER
 727         else
 728                 if [ ! x$LOGNAME = x"" ]; then
 729                         whoami=$LOGNAME
 730                 else
 731                         whoami=build
 732                 fi
 733         fi
 734
 735         # build the timelimit utility
 736         echo "Building timelimit"
 737         mkdir -p $builddir
 738         echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
 739         $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
 740
 741         # build the killbysubdir utility
 742         echo "Building killbysubdir"
 743         echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
 744         $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
 745
 746         prefix="$test_root/prefix/$tree.$compiler"
 747         mkdir -p "$prefix"
 748
 749         # This can be defined in <host>.fns files
 750         sw_config=$config
 751
 752         case "$tree" in
 753         lorikeet-heimdal)
 754                 sw_config="$config --enable-socket-wrapper"
 755                 ;;
 756         samba_4*)
 757                 sw_config="$config --enable-selftest"
 758                 ;;
 759         samba_3_master)
 760                 sw_config="$config --enable-selftest"
 761                 ;;
 762         samba_3*)
 763                 sw_config="$config --enable-socket-wrapper"
 764                 sw_config="$sw_config --enable-nss-wrapper"
 765                 ;;
 766         samba-gtk)
 767                 PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
 768                 export PKG_CONFIG_PATH
 769                 ;;
 770         *)
 771                 testsuite=testsuite
 772                 ;;
 773         esac
 774
 775         if [ "$LCOV_REPORT" = "yes" ]; then
 776                 PRE_GCOV_CFLAGS=$CFLAGS
 777                 PRE_GCOV_LDFLAGS=$LDFLAGS
 778                 GCOV_FLAGS="--coverage"
 779                 CFLAGS="$CFLAGS $GCOV_FLAGS"
 780                 LDFLAGS="$LDFLAGS $GCOV_FLAGS"
 781                 export CFLAGS LDFLAGS
 782         fi
 783
 784         config_and_prefix="$sw_config --prefix=$prefix"
 785
 786         # see if we need to rebuild
 787         sum_tree $test_root $tree $sum $scm
 788         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
 789
 790         if [ -f "$sum.old" ] && cmp "$sum" "$sum.old" > /dev/null; then
 791                 echo "skip: $tree.$compiler nothing changed"
 792                 cd $test_root
 793                 send_logs_skip "$log" "$err"
 794                 unlock_file "$lck"
 795                 echo "Ending build of $tree.$compiler in process $$ at `date`"
 796                 if [ "$LCOV_REPORT" = "yes" ]; then
 797                     CFLAGS=$PRE_GCOV_CFLAGS
 798                     LDFLAGS=$PRE_GCOV_LDFLAGS
 799                     export CFLAGS LDFLAGS
 800                 fi
 801                 return
 802         fi
 803
 804         # we do need to rebuild - save the old sum
 805         [ -f $sum.old ] && /bin/rm -f $sum.old
 806         mv $sum $sum.old
 807
 808         #Action == what to do ie. configure config_log ...
 809         actions="$*"
 810         extra_actions="$EXTRA_ACTIONS"
 811
 812         if [ "$actions" = "" ]; then
 813                 actions="configure config_log config_header build install test"
 814         fi
 815
 816         if [ "$extra_actions" = "" ]; then
 817                 extra_actions="none"
 818         fi
 819
 820         # start the build
 821         (
 822         {
 823                 # we all want to be able to read the output...
 824                 LANG=C
 825                 export LANG
 826
 827                 uname -a
 828
 829                 echo ""
 830                 echo "build_test                  : $build_test_id"
 831                 echo "build_test.fns      : $build_test_fns_id"
 832                 echo "local settings file : $build_test_settings_local_file"
 833                 echo "local functions file: $build_test_fns_local_file"
 834                 echo "used .fns file      : $build_test_used_fns_file"
 835                 echo ""
 836
 837                 # we need to be able to see if a build farm machine is accumulating
 838                 # stuck processes. We do this in two ways, as we don't know what style
 839                 # of ps it will have
 840                 ps xfuw 2> /dev/null
 841                 ps -fu $USER 2> /dev/null
 842
 843                 echo "building $tree with CC=$compiler on $host at "`date`
 844                 echo "builddir=$builddir"
 845                 echo "prefix=$prefix"
 846
 847                 echo "Showing limits"
 848                 ulimit -a 2> /dev/null
 849
 850                 # the following is for non-samba builds only
 851                 if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
 852                         h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
 853                         if [ -n "$h_rev" ]; then
 854                                 echo "HIGHEST SVN REVISION: $h_rev"
 855                         fi
 856                         rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
 857                         if [ -n "$rev" ]; then
 858                         echo "BUILD REVISION: $rev"
 859                         fi
 860                 elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
 861                         csha1=`cat $test_root/$tree.git |head -3 | tail -1`
 862                         if [ -n "$csha1" ]; then
 863                                 echo "BUILD COMMIT REVISION: $csha1"
 864                         fi
 865                         cdate=`cat $test_root/$tree.git |head -4 | tail -1`
 866                         if [ -n "$cdate" ]; then
 867                                 echo "BUILD COMMIT DATE: $cdate"
 868                         fi
 869                         ctime=`cat $test_root/$tree.git |head -2 | tail -1`
 870                         if [ -n "$ctime" ]; then
 871                                 echo "BUILD COMMIT TIME: $ctime"
 872                   fi
 873                 fi
 874
 875                 if [ -x $builddir/killbysubdir ]; then
 876                         echo "$builddir/killbysubdir $builddir in `pwd`"
 877                         $builddir/killbysubdir $builddir
 878                 fi
 879
 880                 for action in $actions; do
 881
 882                         echo Running action $action
 883
 884                         date
 885
 886                         cd $builddir || exit 1
 887                         export srcdir
 888                         df .
 889                         mount
 890                         vmstat
 891
 892                         if [ "x$PREHOOKS" != "x" ]; then
 893                                 for hooks in $PREHOOKS; do
 894                                         if [ "x$hooks" = "x$action" ]; then
 895                                                 ( prehook_$action )
 896                                         fi
 897                                 done
 898                         fi
 899
 900                         ( action_$action )
 901                         action_status=$?
 902
 903                         if [ "x$POSTHOOKS" != "x" ]; then
 904                                 for hooks in $POSTHOOKS; do
 905                                         if [ "x$hooks" = "x$action" ]; then
 906                                                 ( posthook_$action )
 907                                         fi
 908                                 done
 909                         fi
 910
 911                         df .
 912
 913                         if [ $action_status != 0 ]; then
 914                                 echo "ACTION FAILED: $action";
 915                                 echo " return code $action_status $action";
 916                         else
 917                                 echo "ACTION PASSED: $action";
 918                         fi
 919
 920                         if [ $action_status != 0 ]; then
 921                                 break;
 922                         fi
 923                 done
 924
 925                 for action in $extra_actions; do
 926                         if [ "x$action" = "x" ]; then
 927                               break;
 928                         fi
 929
 930                         echo Running action $action
 931
 932                         date
 933
 934                         cd $builddir || exit 1
 935                         export srcdir
 936                         df .
 937                         mount
 938                         vmstat
 939
 940                         if [ "x$PREHOOKS" != "x" ]; then
 941                                 for hooks in $PREHOOKS; do
 942                                         if [ "x$hooks" = "x$action" ]; then
 943                                                 ( prehook_$action )
 944                                         fi
 945                                 done
 946                         fi
 947
 948                         ( action_$action )
 949                         action_status=$?
 950
 951                         if [ "x$POSTHOOKS" != "x" ]; then
 952                                 for hooks in $POSTHOOKS; do
 953                                         if [ "x$hooks" = "x$action" ]; then
 954                                                 ( posthook_$action )
 955                                         fi
 956                                 done
 957                         fi
 958
 959                         df .
 960
 961                         if [ $action_status != 0 ]; then
 962                                 echo "ACTION FAILED: $action";
 963                                 echo " return code $action_status $action";
 964                         else
 965                                 echo "ACTION PASSED: $action";
 966                         fi
 967
 968                         if [ $action_status != 0 ]; then
 969                                 break;
 970                         fi
 971                 done
 972
 973
 974                 if [ "$noclean" = "yes" ]; then
 975                         echo cleanup skipped!
 976                 else
 977                         echo cleaning up
 978                         do_make clean
 979                 fi
 980                 date
 981         } 3>&2 2>&1 1>&3 | tee "$err"
 982         ) > "$log" 2>&1
 983         # be aware the above channel swap may sometimes result in unordered
 984         # stdout/stderr merge
 985
 986         if [ "$LCOV_REPORT" = "yes" ]; then
 987                 chmod u=rwX,g=rX,o=rX -R $builddir/coverage
 988                 rsync -rct -q --password-file=.password -z --timeout=200 \
 989                         $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
 990                 CFLAGS=$PRE_GCOV_CFLAGS
 991                 LDFLAGS=$PRE_GCOV_LDFLAGS
 992                 export CFLAGS LDFLAGS
 993         fi
 994
 995         cd $test_root
 996
 997         /bin/rm -rf $prefix
 998         # send the logs to the master site
 999         send_logs "$log" "$err"
1000
1001         # cleanup
1002         echo "Ending build of $tree.$compiler in process $$ at `date`"
1003         unlock_file "$lck"
1004 }
1005
1006 #########################################################
1007 # if you want to build only one project at a time
1008 # add 'global_lock' after 'per_run_hook' and
1009 # 'global_unlock' to the end of the file
1010 #########################################################
1011
1012 global_lock() {
1013         lock_file "global.lck" || {
1014                 exit 0
1015         }
1016 }
1017
1018 global_unlock() {
1019         unlock_file "global.lck"
1020 }
1021
1022 delete_old_tree() {
1023         otree=$1
1024         test -z "$otree" && return 0;
1025
1026         rm -rf $otree
1027         rm -rf $otree.svn
1028         rm -rf $otree.*.svn
1029         rm -rf $otree.git
1030         rm -rf $otree.*.git
1031         rm -rf build.$otree.*
1032 }
1033
1034 # this is a special fn that allows us to add a "special" hook to the build
1035 # farm that we want to do to the build farm. never leave it empty. instead,
1036 # use ":" as the fn body.
1037 per_run_hook() {
1038         # kill old processes on systems with a known problem
1039         case $host in
1040         nohost)
1041                 echo "just a placeholder";
1042                 ;;
1043         deckchair)
1044                 rm -f deckchair.fns
1045                 ;;
1046         esac
1047
1048         # trim the log if too large
1049         if [ "`wc -c < build.log`" -gt 2000000 ]; then
1050         rm -f build.log
1051         fi
1052
1053         old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2"
1054         old_trees="$old_tree samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze"
1055         old_trees="$old_tree samba_3_X_test samba_3_X_devel samba_3_X_devel samba_3_waf"
1056         for d in $old_trees; do
1057                 delete_old_tree $d
1058         done
1059 }
1060
1061
1062 ######################################################
1063 # main code that is run on each call to the build code
1064 ######################################################
1065 rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
1066
1067
1068 # build.log can grow to an excessive size, trim it beyond 50M
1069 if [ -f build.log ]; then
1070         find build.log -size +100000 -exec /bin/rm '{}' \;
1071 fi