1 #!/bin/sh -*- mode: shell-script; -*-
3 # build_farm -- distributed build/test architecture for samba, rsync, etc
5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
9 # default maximum runtime for any command
10 MAXTIME=25200 # 7 hours
11 SMBD_MAXTIME=18000 # 5 hours for a samba process ..
12 # default maximum memory size (100M) for any command
14 RUN_FROM_BUILD_FARM=yes
15 export RUN_FROM_BUILD_FARM
16 export MAXTIME SMBD_MAXTIME
20 build_test_fns_id='$Id$'
26 echo rsync -a --delete $Tsrc/ $Tdst
27 rsync -a --delete $Tsrc/ $Tdst || return 1
31 #############################
32 # build a signature of a tree, used to see if we
34 #############################
41 find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
42 sum build_test build_test.fns >> $sum_tree_sum
44 if [ -f "$host.fns" ]; then
45 sum $host.fns >> $sum_tree_sum
47 sum generic.fns >> $sum_tree_sum
50 if [ -f "$test_root/$tree.$scm" ]; then
51 sum "$test_root/$tree.$scm" >> $sum_tree_sum
54 for d in $deptrees; do
55 dscm=`choose_scm "$d"`
56 if [ -f "$test_root/$d.$dscm" ]; then
57 sum "$test_root/$d.$dscm" >> $sum_tree_sum
62 #############################
63 # send the logs to the master site
64 #############################
67 if [ "$nologreturn" = "yes" ]; then
68 echo "skipping log transfer"
74 chmod 0644 "$log" "$err"
76 # xargs -i is implemented differently or not at all.
77 # GNU xargs did not implement "-I" until 4.2.9:
78 xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
80 if [ x"$status" = x"0" ]; then
84 if [ x"$XARGS_IS_GNU" = x"yes" ]; then
87 XARGS_I="xargs -I '{}'"
90 find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
91 find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
93 rsync $* -c -q --password-file=.password -z --timeout=200 \
94 "$log" "$err" $host@build.samba.org::build_farm_data/
98 #############################
99 # send the logs when they haven't changed
100 # the aim is to just update the servers timestamp.
101 # sending with a very large rsync block size does this
102 # with minimal network traffic
103 #############################
107 send_logs "$1" "$2" -B 10000000
110 ############################
111 # fetch the latest copy of the tree
112 ############################
115 if [ "$norsync" = "yes" ]; then
116 echo "skipping tree transfer"
119 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
120 --delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
121 samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
122 echo "transferred $fetchtree OK"
124 echo "transfer of $fetchtree failed code $?"
131 ############################
132 # fetch the latest copy of the rev meta info
133 ############################
139 test -z "$scm" && return 1
140 test x"$scm" = x"unknown" && return 1
141 test x"$scm" = x"cvs" && return 1
143 if [ "$norsync" = "yes" ]; then
144 echo "skipping .revinfo.$scm transfer"
146 if [ -r $test_root/$tree.$scm ]; then
147 [ -f $test_root/$tree.$scm.old ] && rm -f $test_root/$tree.$scm.old
148 [ -f $test_root/$tree.$scm ] && mv $test_root/$tree.$scm $test_root/$tree.$scm.old
150 rsync -q --timeout=200 -clz --ignore-errors \
151 samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
153 if [ -r $test_root/$tree.$scm ]; then
159 ############################
160 # choose the scm that is used for the given project
161 ############################
167 samba* | rsync | libreplace | talloc | tdb | ntdb | ldb | pidl | ccache* | waf*)
179 ############################
180 # grab a lock file. Not atomic, but close :)
181 # tries to cope with NFS
182 ############################
185 if [ -z "$lock_root" ]; then
190 machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
191 pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
193 if [ "$pid" = "$$" ]; then
194 locknesting=`expr $locknesting + 1`
195 echo "lock nesting now $locknesting"
199 # We need to assert that the file is > 0 size, as otherwise we never
200 # recover from disk full situations
201 if test -f "$lckf" && test -s "$lckf"; then
202 test x$machine = x$host || {
203 echo "lock file $lckf is valid for other machine $machine"
208 echo "lock file $lckf is valid for process $pid"
212 echo "stale lock file $lckf for $machine:$pid"
216 echo "$host:$$" > "$lckf"
220 ############################
222 ############################
225 if [ -z "$lock_root" ]; then
228 if [ "$locknesting" != "0" ]; then
229 locknesting=`expr $locknesting - 1`
230 echo "lock nesting now $locknesting"
237 ############################
238 # run make, and print trace
239 ############################
242 # work out correct make command
249 if [ x"$MAKECOMMAND" = x ]; then
256 # some trees don't need as much time
258 rsync | tdb | ntdb | talloc | libreplace | ccache* | waf*)
259 if [ "$compiler" != "checker" ]; then
260 MMTIME=`expr $MMTIME / 5`
265 # special build for some trees
268 ./waf distclean && ./waf configure build
274 if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
275 echo "$MAKECOMMAND $t"
276 $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
279 # we can parallelize everything and all targets
280 if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
281 echo "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS" "$t"
282 $builddir/timelimit $MMTIME "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS" "$t"
285 echo "$MAKECOMMAND $t"
286 $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
291 if [ $status != 0 ]; then
293 test|check|installcheck)
296 #run again with V=1, so we see failed commands
297 $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t" V=1
303 if [ $status != 0 ]; then
313 ############################
314 # do the coverage report
315 ############################
317 action_lcovreport() {
318 if [ "$LCOV_REPORT" = "yes" ]; then
321 lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
323 samba_4*|tdb|ntdb|talloc|ldb|libreplace)
324 lcov --base-directory $builddir/bin --directory $builddir/bin --capture --output-file $builddir/$tree.lcov.info
327 lcov --base-directory $builddir/demos --directory $builddir/demos --capture --output-file $builddir/$tree.lcov.info
330 lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
333 genhtml -o $builddir/coverage $builddir/$tree.lcov.info
335 echo "return code: $rc"
337 echo "LCOV_REPORT not set and lcovreport asked"
338 echo "Most probably an error please fix !"
343 action_callcatcherreport() {
344 if [ "$CALLCATCHER_REPORT" = "yes" ]; then
347 callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/unused-fns.txt
350 callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/all-unused-fns.txt
351 grep -v -f $srcdir/callcatcher-exceptions.grep $builddir/coverage/all-unused-fns.txt > $builddir/coverage/unused-fns.txt
355 echo "return code: $rc"
357 echo "CALLCATCHER_REPORT not set and callcatcher asked"
358 echo "Most probably an error please fix !"
364 ############################
366 ############################
369 # special handling for some trees
372 $builddir/timelimit $MAXTIME ./waf configure
374 echo "CONFIGURE STATUS: $cstatus"
379 if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
380 perl $srcdir/Makefile.PL PREFIX="$prefix"
382 echo "CONFIGURE STATUS: $cstatus"
386 if [ ! -x $srcdir/configure ]; then
387 ls -l $srcdir/configure
388 echo "$srcdir/configure is missing"
390 echo "CONFIGURE STATUS: $cstatus"
394 echo "CFLAGS=$CFLAGS"
395 echo configure options: $config_and_prefix
396 echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
398 CC="$CCACHE $compiler"
400 $builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
403 if [ x"$cstatus" != x"0" ]; then
404 if [ -f config.log ]; then
405 echo "contents of config.log:"
410 if [ -f bin/config.log ]; then
411 echo "contents of config.log:"
415 echo "CONFIGURE STATUS: $cstatus"
419 ############################
420 # show the configure log
421 ############################
423 action_config_log() {
425 log_files="config.log bin/config.log"
426 for f in $log_files; do
428 echo "contents of config.log:"
436 ############################
438 ############################
440 action_config_header() {
441 hdr_files="config.h include/config.h include/autoconf/config.h bin/default/config.h bin/default/include/config.h bin/default/source3/include/config.h"
442 for h in $hdr_files; do
444 echo "contents of $h:"
455 ############################
457 ############################
465 do_make everything torture
478 echo "BUILD STATUS: $bstatus"
483 ############################
484 # show static analysis results
485 ############################
487 action_cc_checker() {
489 # default to passing the cc_checker
492 if [ -f ibm_checker.out ]; then
494 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
497 echo "CC_CHECKER STATUS: $cccstatus"
501 ############################
503 ############################
506 if [ -d $prefix ]; then
507 if [ "$noclean" != "yes" ]; then
514 echo "INSTALL STATUS: $istatus"
518 ############################
520 action_test_samba() {
524 # if we produced a test summary then show it
525 [ -f st/summary ] && {
530 return "$totalstatus"
533 action_test_generic() {
538 echo "TEST STATUS: $totalstatus"
539 return "$totalstatus"
542 action_test_lorikeet_heimdal() {
545 SOCKET_WRAPPER_DIR=`pwd`/sw
546 mkdir $SOCKET_WRAPPER_DIR
547 export SOCKET_WRAPPER_DIR
551 export SOCKET_WRAPPER_DIR
552 echo "TEST STATUS: $totalstatus"
553 return "$totalstatus"
557 #############################
558 # attempt some basic tests of functionaility
559 # starting as basic as possible, and getting incresingly complex
560 #############################
563 # Samba needs crufty code of its own for backward
564 # compatiblity. I think a better way to do this in the future
565 # is to just call 'make installcheck'.
567 samba*|smb-build|pidl)
571 action_test_lorikeet_heimdal
579 #############################
580 # Do nothing (needed if we have nothing to do for extra_actions)
581 #############################
587 ###########################
588 # do a test build of a particular tree
589 # This is the master function called by generic.fns or
591 ###########################
600 echo "Starting to deal with tree $tree with compiler $compiler"
601 if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null 2>/dev/null; then
608 # limit our resource usage
609 ulimit -t $MAXTIME 2> /dev/null
612 ulimit -m $MAXMEM 2> /dev/null
615 # darn, this affects sparse files too! disable it
616 # ulimit -f 100000 2> /dev/null
618 # try and limit the number of open files to 500, up from 250. That means we'll discover
619 # fd leaks faster while allowing our very complex make test to run
620 ulimit -n 500 2> /dev/null
625 if [ -z "$test_root" ]; then
629 log="build.$tree.$host.$compiler.log"
630 err="build.$tree.$host.$compiler.err"
631 sum="build.$tree.$host.$compiler.sum"
632 lck="build.$tree.lck"
633 srcdir="$test_root/$tree/$source"
635 lock_file "$lck" || {
639 # work out what other trees this package depends on
643 deptrees="samba_4_0_test"
647 scm=`choose_scm "$tree"`
649 # pull the entries, if any
650 # Remove old .svn or .git files
651 # Move the current .svn org .git to .svn.old or
652 # .git.old then fetch the new from rsync
653 if fetch_revinfo "$tree" "$scm"; then
654 for d in $deptrees; do
655 # If there is dependency substree(s) we add info
656 # from the dependency tree so that we
657 # can rebuild in case one of them has changed
658 dscm=`choose_scm "$d"`
659 if [ -f "$test_root/$d.$dscm" ]; then
660 if [ "$d" != "$tree" ]; then
661 cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
665 [ -f $test_root/$tree.$compiler.$scm.old ] && rm -f $test_root/$tree.$compiler.$scm.old
666 [ -f $test_root/$tree.$compiler.$scm ] && mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
667 [ -f $test_root/$tree.$scm ] && cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
669 if [ -f $test_root/$tree.$compiler.$scm.old ] && \
670 cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
672 echo "skip: $tree.$compiler nothing changed in $scm"
674 send_logs_skip "$log" "$err"
681 fetch_tree "$tree" || {
687 # check for essential files
693 if [ ! -x $srcdir/waf ]; then
694 echo "skip: $tree.$compiler waf not present, try again next time!"
701 if [ ! -x $srcdir/configure ]; then
702 echo "skip: $tree.$compiler configure not present, try again next time!"
710 echo "Starting build of $tree.$compiler in process $$ at `date`"
713 # Parameters for the build depending on the tree
722 if [ ! x$USER = x"" ]; then
725 if [ ! x$LOGNAME = x"" ]; then
732 # build the timelimit utility
733 echo "Building timelimit"
735 echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
736 $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
738 # build the killbysubdir utility
739 echo "Building killbysubdir"
740 echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
741 $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
743 prefix="$test_root/prefix/$tree.$compiler"
746 # This can be defined in <host>.fns files
751 sw_config="$config --enable-socket-wrapper"
754 sw_config="$config --enable-selftest"
755 sw_config="$sw_config --with-perl-lib-install-dir=$prefix/perl.lib"
756 sw_config="$sw_config --with-perl-arch-install-dir=$prefix/perl.arch"
759 sw_config="$config --enable-socket-wrapper"
760 sw_config="$sw_config --enable-nss-wrapper"
763 PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
764 export PKG_CONFIG_PATH
771 if [ "$LCOV_REPORT" = "yes" ]; then
772 PRE_GCOV_CFLAGS=$CFLAGS
773 PRE_GCOV_LDFLAGS=$LDFLAGS
774 GCOV_FLAGS="--coverage"
775 CFLAGS="$CFLAGS $GCOV_FLAGS"
776 LDFLAGS="$LDFLAGS $GCOV_FLAGS"
777 export CFLAGS LDFLAGS
780 config_and_prefix="$sw_config --prefix=$prefix"
782 # see if we need to rebuild
783 sum_tree $test_root $tree $sum $scm
784 echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
786 if [ -f "$sum.old" ] && cmp "$sum" "$sum.old" > /dev/null; then
787 echo "skip: $tree.$compiler nothing changed"
789 send_logs_skip "$log" "$err"
791 echo "Ending build of $tree.$compiler in process $$ at `date`"
792 if [ "$LCOV_REPORT" = "yes" ]; then
793 CFLAGS=$PRE_GCOV_CFLAGS
794 LDFLAGS=$PRE_GCOV_LDFLAGS
795 export CFLAGS LDFLAGS
800 # we do need to rebuild - save the old sum
801 [ -f $sum.old ] && /bin/rm -f $sum.old
804 #Action == what to do ie. configure config_log ...
806 extra_actions="$EXTRA_ACTIONS"
808 if [ "$actions" = "" ]; then
809 actions="configure config_log config_header build install test"
812 if [ "$extra_actions" = "" ]; then
819 # we all want to be able to read the output...
826 echo "build_test : $build_test_id"
827 echo "build_test.fns : $build_test_fns_id"
828 echo "local settings file : $build_test_settings_local_file"
829 echo "local functions file: $build_test_fns_local_file"
830 echo "used .fns file : $build_test_used_fns_file"
833 # we need to be able to see if a build farm machine is accumulating
834 # stuck processes. We do this in two ways, as we don't know what style
837 ps -fu $USER 2> /dev/null
839 echo "building $tree with CC=$compiler on $host at "`date`
840 echo "builddir=$builddir"
841 echo "prefix=$prefix"
843 echo "Showing limits"
844 ulimit -a 2> /dev/null
846 # the following is for non-samba builds only
847 if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
848 h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
849 if [ -n "$h_rev" ]; then
850 echo "HIGHEST SVN REVISION: $h_rev"
852 rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
853 if [ -n "$rev" ]; then
854 echo "BUILD REVISION: $rev"
856 elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
857 csha1=`cat $test_root/$tree.git |head -3 | tail -1`
858 if [ -n "$csha1" ]; then
859 echo "BUILD COMMIT REVISION: $csha1"
861 cdate=`cat $test_root/$tree.git |head -4 | tail -1`
862 if [ -n "$cdate" ]; then
863 echo "BUILD COMMIT DATE: $cdate"
865 ctime=`cat $test_root/$tree.git |head -2 | tail -1`
866 if [ -n "$ctime" ]; then
867 echo "BUILD COMMIT TIME: $ctime"
871 if [ -x $builddir/killbysubdir ]; then
872 echo "$builddir/killbysubdir $builddir in `pwd`"
873 $builddir/killbysubdir $builddir
876 for action in $actions; do
878 echo Running action $action
882 cd $builddir || exit 1
888 if [ "x$PREHOOKS" != "x" ]; then
889 for hooks in $PREHOOKS; do
890 if [ "x$hooks" = "x$action" ]; then
899 if [ "x$POSTHOOKS" != "x" ]; then
900 for hooks in $POSTHOOKS; do
901 if [ "x$hooks" = "x$action" ]; then
909 if [ $action_status != 0 ]; then
910 echo "ACTION FAILED: $action";
911 echo " return code $action_status $action";
913 echo "ACTION PASSED: $action";
916 if [ $action_status != 0 ]; then
921 for action in $extra_actions; do
922 if [ "x$action" = "x" ]; then
926 echo Running action $action
930 cd $builddir || exit 1
936 if [ "x$PREHOOKS" != "x" ]; then
937 for hooks in $PREHOOKS; do
938 if [ "x$hooks" = "x$action" ]; then
947 if [ "x$POSTHOOKS" != "x" ]; then
948 for hooks in $POSTHOOKS; do
949 if [ "x$hooks" = "x$action" ]; then
957 if [ $action_status != 0 ]; then
958 echo "ACTION FAILED: $action";
959 echo " return code $action_status $action";
961 echo "ACTION PASSED: $action";
964 if [ $action_status != 0 ]; then
970 if [ "$noclean" = "yes" ]; then
971 echo cleanup skipped!
977 } 3>&2 2>&1 1>&3 | tee "$err"
979 # be aware the above channel swap may sometimes result in unordered
980 # stdout/stderr merge
982 if [ "$LCOV_REPORT" = "yes" ]; then
983 chmod u=rwX,g=rX,o=rX -R $builddir/coverage
984 rsync -rct -q --password-file=.password -z --timeout=200 \
985 $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
986 CFLAGS=$PRE_GCOV_CFLAGS
987 LDFLAGS=$PRE_GCOV_LDFLAGS
988 export CFLAGS LDFLAGS
994 # send the logs to the master site
995 send_logs "$log" "$err"
998 echo "Ending build of $tree.$compiler in process $$ at `date`"
1002 #########################################################
1003 # if you want to build only one project at a time
1004 # add 'global_lock' after 'per_run_hook' and
1005 # 'global_unlock' to the end of the file
1006 #########################################################
1009 lock_file "global.lck" || {
1015 unlock_file "global.lck"
1020 test -z "$otree" && return 0;
1027 rm -rf build.$otree.*
1030 # this is a special fn that allows us to add a "special" hook to the build
1031 # farm that we want to do to the build farm. never leave it empty. instead,
1032 # use ":" as the fn body.
1034 # kill old processes on systems with a known problem
1037 echo "just a placeholder";
1044 # trim the log if too large
1045 if [ "`wc -c < build.log`" -gt 2000000 ]; then
1049 old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2"
1050 old_trees="$old_tree samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze"
1051 old_trees="$old_tree samba_3_X_test samba_3_X_devel samba_3_X_devel samba_3_waf samba_3_master tdb2"
1052 for d in $old_trees; do
1058 ######################################################
1059 # main code that is run on each call to the build code
1060 ######################################################
1061 rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
1064 # build.log can grow to an excessive size, trim it beyond 50M
1065 if [ -f build.log ]; then
1066 find build.log -size +100000 -exec /bin/rm '{}' \;