update build farm for samba_3_2 branch
[build-farm.git] / build_test.fns
1 # -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
2
3 # build_farm -- distributed build/test architecture for samba, rsync, etc
4
5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
8
9 # default maximum runtime for any command
10 MAXTIME=7200
11 # default maximum memory size (100M) for any command
12 MAXMEM=100000
13 RUN_FROM_BUILD_FARM=yes
14 export RUN_FROM_BUILD_FARM
15
16 deptrees="";
17
18 build_test_fns_id='$Id$'
19
20 #############################
21 # build a signature of a tree, used to see if we
22 # need to rebuild 
23 sum_tree() {
24         sum_tree_test_root=$1
25         sum_tree_tree=$2
26         sum_tree_sum=$3
27         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
28         sum build_test build_test.fns >> $sum_tree_sum
29
30         if [ -f "$host.fns" ]; then
31             sum $host.fns >> $sum_tree_sum
32         else
33             sum generic.fns >> $sum_tree_sum
34         fi
35         if [ -f "$test_root/$tree.svn" ]; then
36             sum "$test_root/$tree.svn" >> $sum_tree_sum
37         fi
38         for d in $deptrees; do
39             if [ -f "$test_root/$d.svn" ]; then
40                 sum "$test_root/$d.svn" >> $sum_tree_sum
41             fi
42         done
43 }
44
45 #############################
46 # send the logs to the master site
47 send_logs() {
48         if [ "$nologreturn" = "yes" ]; then
49                 echo "skipping log transfer"
50         else
51                 log="$1"
52                 err="$2"
53                 shift
54                 shift
55                 chmod 0644 "$log" "$err"
56
57                 XARGS_I="xargs -i"
58                 if [ "`uname`" = "FreeBSD" ]; then
59                         XARGS_I="xargs -I '{}' -R -1"
60                 fi
61                 find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
62                 find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
63
64                 rsync $* -ct -q --password-file=.password -z --timeout=200 \
65                     "$log" "$err" $host@build.samba.org::build_farm_data/
66         fi
67 }
68
69 #############################
70 # send the logs when they haven't changed
71 # the aim is to just update the servers timestamp.
72 # sending with a very large rsync block size does this
73 # with minimal network traffic
74 send_logs_skip() {
75     touch "$1" "$2"
76     send_logs "$1" "$2" -B 10000000
77 }
78
79 ############################
80 # fetch the latest copy of the tree
81 fetch_tree() {
82         if [ "$norsync" = "yes" ]; then
83                 echo "skipping tree transfer"
84         else
85                 fetchtree=$1
86                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
87                         samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
88                         echo "transferred $fetchtree OK"
89                 else
90                         echo "transfer of $fetchtree failed code $?"
91                         return 1
92                 fi
93         fi
94         return 0
95 }
96
97 ############################
98 # fetch the latest copy of the svn entries file
99 fetch_svn() {
100     tree=$1
101 # skip products still in CVS.
102     case "$tree" in
103     ccache | distcc | rsync)
104         return 1
105         ;;
106     *)
107         ;;
108     esac
109     if [ "$norsync" = "yes" ]; then
110         echo "skipping svn transfer"
111     else
112         if [ -r $test_root/$tree.svn ]; then
113                 rm -f $test_root/$tree.svn.old
114             mv $test_root/$tree.svn $test_root/$tree.svn.old
115         fi
116         rsync -q --timeout=200 -clz --ignore-errors \
117             samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn.tmp
118         chmod u+w $test_root/$tree.svn.tmp
119         sort -u < $test_root/$tree.svn.tmp > $test_root/$tree.svn
120         rm -f $test_root/$tree.svn.tmp
121     fi
122     if [ -r $test_root/$tree.svn ]; then
123         return 0;
124     fi
125     return 1
126 }
127
128 locknesting=0
129
130 ############################
131 # grab a lock file. Not atomic, but close :)
132 # tries to cope with NFS
133 lock_file() {
134         if [ -z "$lock_root" ]; then
135           lock_root=`pwd`;
136         fi
137         lckf="$lock_root/$1"
138         machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
139         pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
140
141         if [ "$pid" = "$$" ]; then
142             locknesting=`expr $locknesting + 1`
143             echo "lock nesting now $locknesting"
144             return 0
145         fi
146
147         if test -f "$lckf"; then
148             test $machine = $host || {
149                 echo "lock file $lckf is valid for other machine $machine"
150                 return 1                 
151             }
152             kill -0 $pid && {
153                 echo "lock file $lckf is valid for process $pid"
154                 return 1
155             }
156             echo "stale lock file $lckf for $machine:$pid"
157             cat "$lckf"
158             /bin/rm -f "$lckf"
159         fi
160         echo "$host:$$" > "$lckf"
161         return 0
162 }
163
164 ############################
165 # unlock a lock file
166 unlock_file() {
167         if [ -z "$lock_root" ]; then
168           lock_root=`pwd`;
169         fi
170         if [ "$locknesting" != "0" ]; then
171             locknesting=`expr $locknesting - 1`
172             echo "lock nesting now $locknesting"
173         else 
174             lckf="$lock_root/$1"
175             /bin/rm -f "$lckf"
176         fi
177 }
178
179 ############################
180 # run make, and print trace
181 do_make() {
182
183   if [ x"$MAKE" = x ] 
184   then
185     MAKE=make
186   fi 
187
188   MMTIME=$MAXTIME
189   # some trees don't need as much time
190   case "$tree" in
191         rsync | tdb | talloc | libreplace | ccache | distcc)
192           if [ "$compiler" != "checker" ]; then
193               MMTIME=`expr $MMTIME / 5`
194           fi
195           ;;
196   esac
197   
198     
199   for t in $*; do
200     if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
201       echo "$MAKE $t"
202       ./timelimit $MMTIME "$MAKE" "$t"
203       status=$?
204     else
205       # we can parallelize everything and all targets
206       if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
207         echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
208         ./timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
209         status=$?
210       else
211         echo "$MAKE $t"
212         ./timelimit $MMTIME "$MAKE" "$t"
213         status=$?
214       fi
215     fi
216
217     if [ $status != 0 ]; then
218       return $status;
219     fi
220
221   done
222
223   return 0
224 }      
225
226 ############################
227 # configure the tree
228 action_configure() {
229         if [ ! -x $srcdir/configure ]; then
230             ls -l $srcdir/configure
231             echo "$srcdir/configure is missing"
232             cstatus=255
233             echo "CONFIGURE STATUS: $cstatus"
234             return $cstatus;
235         fi
236         echo "CFLAGS=$CFLAGS"
237         echo configure options: $config_and_prefix
238         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
239         CC="$CCACHE $compiler"
240         export CC
241         ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
242         cstatus=$?
243         echo "CONFIGURE STATUS: $cstatus"
244         if [ -f config.h ]; then
245             echo "contents of config.h:"
246             cat config.h
247         fi
248         if [ -f include/config.h ]; then
249             echo "contents of include/config.h:"
250             cat include/config.h
251         fi
252         return $cstatus;
253 }
254
255 ############################
256 # show the configure log
257 action_config_log() {
258         if [ ! -f config.log ]; then
259             return 0;
260         fi
261         echo "contents of config.log:"
262         cat config.log
263         return 0;
264 }
265
266 ############################
267 # build the tree
268 action_build() {
269         case "$tree" in
270         samba4)
271                 rm -f bin/smbtorture
272                 do_make everything
273                 bstatus=$?
274                 if test x"$bstatus" != x"0"; then
275                         # the 2nd 'make everything' is to work around a bug
276                         # in netbsd make. 
277                         do_make everything
278                         bstatus=$?
279                 fi
280
281                 if [ -n "$smbtorture4" ]; then
282                         if [ ! -f bin/smbtorture ]; then
283                                 do_make bin/smbtorture
284                         fi
285                         if [ -f bin/smbtorture ]; then
286                                 cp bin/smbtorture $smbtorture4
287                         fi
288                 fi
289
290                 ;;
291         samba|samba_3*)
292                 do_make proto everything torture
293                 bstatus=$?
294                 ;;
295         *)
296                 do_make all
297                 bstatus=$?
298                 ;;
299         esac
300
301         echo "BUILD STATUS: $bstatus"
302
303         return $bstatus
304 }
305
306 ############################
307 # show static analysis results
308 action_cc_checker() {
309
310         # default to passing the cc_checker
311         cccstatus=0
312
313         if [ -f ibm_checker.out ]; then
314                 cat ibm_checker.out
315                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
316         fi
317
318         echo "CC_CHECKER STATUS: $cccstatus"
319         return $cccstatus;      
320 }
321
322 ############################
323 # install the tree
324 action_install() {
325         if [ -d $prefix ]; then
326                 if [ "$noclean" != "yes" ]; then
327                     rm -rf $prefix
328                 fi
329         fi
330
331         do_make install
332         istatus=$?
333         echo "INSTALL STATUS: $istatus"
334         return $istatus;
335 }
336
337 ############################
338 # test the tree
339 action_test_samba() {
340         do_make test
341         totalstatus=$?
342         return "$totalstatus"
343 }
344
345 action_test_generic() {
346         CC="$compiler"
347         export CC
348         do_make installcheck
349         totalstatus=$?
350         echo "TEST STATUS: $totalstatus"
351         return "$totalstatus"
352 }
353
354 action_test_lorikeet_heimdal() {
355         CC="$compiler"
356         export CC
357         SOCKET_WRAPPER_DIR=`pwd`/sw
358         mkdir $SOCKET_WRAPPER_DIR
359         export SOCKET_WRAPPER_DIR
360         do_make check
361         totalstatus=$?
362         SOCKET_WRAPPER_DIR=
363         export SOCKET_WRAPPER_DIR
364         echo "TEST STATUS: $totalstatus"
365         return "$totalstatus"
366 }
367
368
369 #############################
370 # attempt some basic tests of functionaility
371 # starting as basic as possible, and getting incresingly complex
372
373 action_test() {
374         # Samba needs crufty code of its own for backward
375         # compatiblity.  I think a better way to do this in the future
376         # is to just call 'make installcheck'.
377         case "$tree" in
378 #        samba_3*)
379 #               echo "testing of samba_3* disabled until cause of runaway processes found (tridge - 7th sep 2006)"
380 #               ;;
381         samba*|smb-build|pidl)
382             action_test_samba
383             ;;
384         lorikeet-heimdal*)
385             action_test_lorikeet_heimdal
386             ;;
387         *)
388             action_test_generic
389             ;;
390         esac
391 }
392
393 ###########################
394 # do a test build of a particular tree
395 test_tree() {
396         tree=$1
397         source=$2
398         compiler="$3"
399         shift
400         shift
401         shift
402         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
403             CCACHE="ccache"
404             export CCACHE
405         else
406             CCACHE=""
407         fi
408
409         # limit our resource usage
410         ulimit -t $MAXTIME 2> /dev/null
411
412         # max mem size 100M
413         ulimit -m $MAXMEM 2> /dev/null
414
415         # max file size 100M
416         # darn, this affects sparse files too! disable it
417         # ulimit -f 100000 2> /dev/null
418
419         # try and limit the number of open files to 150. That means we'll discover
420         # fd leaks faster
421         ulimit -n 150 2> /dev/null
422
423         # Keep stuff private
424         umask 077
425
426         if [ -z "$test_root" ]; then
427                 test_root=`pwd`
428         fi
429
430         log="build.$tree.$host.$compiler.log"
431         err="build.$tree.$host.$compiler.err"
432         sum="build.$tree.$host.$compiler.sum"
433         lck="build.$tree.lck"
434         srcdir="$test_root/$tree/$source"
435
436         lock_file "$lck" || {
437                 return
438         }
439
440         # work out what other trees this package depends on
441         deptrees=""
442         case "$tree" in
443             talloc | tdb)
444                 deptrees="libreplace";
445             ;;
446             ldb)
447                 deptrees="libreplace talloc tdb";
448             ;;
449                 samba-gtk)
450                 deptrees="samba4"
451                 ;;
452         esac
453
454         # pull the svn entries, if any
455         if fetch_svn "$tree"; then
456             for d in $deptrees; do
457                 if [ -f "$test_root/$d.svn" ]; then
458                     if [ "$d" != "$tree" ]; then
459                         cat "$test_root/$d.svn" >> $test_root/$tree.svn
460                     fi
461                 fi
462             done
463             rm -f $test_root/$tree.$compiler.svn.old
464             mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
465             cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
466             if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
467                 echo "skip: $tree.$compiler nothing changed in svn"
468                 cd $test_root
469                 send_logs_skip "$log" "$err"
470                 unlock_file "$lck"
471                 return
472             fi
473         fi
474
475         # pull the tree
476         fetch_tree "$tree" || {
477             cd $test_root
478             unlock_file "$lck"
479             return
480         }
481
482         if [ ! -x $srcdir/configure ] && [ "$tree" != "pidl" ]; then
483                 echo "skip: $tree.$compiler configure not present, try again next time!"
484                 cd $test_root
485                 unlock_file "$lck"
486                 return
487         fi
488
489         echo "Starting build of $tree.$compiler in process $$ at `date`"
490
491         case "$tree" in
492             tdb | talloc | ldb | libreplace)
493                 builddir="$test_root/tmp.$tree.$compiler"
494                 usingtmpbuild=1
495                 if [ -d $builddir ]; then
496                     rm -rf $builddir
497                 fi
498                 mkdir -p $builddir
499                 export builddir
500             ;;
501             *)
502                 builddir=$srcdir
503                 usingtmpbuild=0
504                 export builddir
505             ;;
506         esac
507         
508         if [ ! x$USER = x"" ]; then
509             whoami=$USER
510         else 
511             if [ ! x$LOGNAME = x"" ]; then
512                 whoami=$LOGNAME
513             else
514                 whoami=build
515             fi
516         fi
517
518         prefix="$test_root/prefix/$tree.$compiler"
519         mkdir -p "$prefix"
520
521         smbtorture4=$test_root/smbtorture4
522         export smbtorture4
523
524         sw_config=$config
525
526         case "$tree" in
527         samba4|lorikeet-heimdal)
528             sw_config="$config --enable-socket-wrapper"
529             ;;
530         samba|samba_3*)
531             sw_config="$config --enable-socket-wrapper"
532             if [ -f $smbtorture4 ]; then
533                 # we create a local copy to make sure the same binary is used for all tests
534                 cp $smbtorture4 $smbtorture4.$tree 
535                 sw_config="$sw_config --with-smbtorture4-path=$smbtorture4.$tree"
536             fi
537             ;;
538         samba-gtk)
539                 PKG_CONFIG_PATH="$test_root/prefix/samba4.$compiler/lib/pkgconfig"
540                 export PKG_CONFIG_PATH
541                 ;;
542         ldb)
543             fetch_tree popt
544             ;;
545         talloc)
546             fetch_tree libreplace
547             ;;
548         *)
549             testsuite=testsuite
550             ;;
551         esac
552
553         if [ "$LCOV_REPORT" = "yes" ]; then
554             GCOV_FLAGS="-ftest-coverage -fprofile-arcs"
555             GCOV_LIBS="-lgcov"
556             HOSTCC_CFLAGS="$HOSTCC_CFLAGS $GCOV_FLAGS" 
557             CFLAGS="$CFLAGS $GCOV_FLAGS" 
558             LDFLAGS="$LDFLAGS $GCOV_FLAGS $GCOV_LIBS" 
559             SHLD_FLAGS="$SHLD_FLAGS $GCOV_FLAGS $GCOV_LIBS"
560             export HOSTCC_CFLAGS CFLAGS LDFLAGS SHLD_FLAGS
561         fi
562
563         config_and_prefix="$sw_config --prefix=$prefix"
564
565         # see if we need to rebuild
566         sum_tree $test_root $tree $sum
567         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
568
569         if cmp "$sum" "$sum.old" > /dev/null; then
570                 echo "skip: $tree.$compiler nothing changed"
571                 cd $test_root
572                 send_logs_skip "$log" "$err"
573                 unlock_file "$lck"
574                 echo "Ending build of $tree.$compiler in process $$ at `date`"
575                 return
576         fi
577
578         # we do need to rebuild - save the old sum
579         /bin/rm -f $sum.old
580         mv $sum $sum.old
581
582         actions="$*"
583         
584         if [ "$actions" = "" ]; then
585             actions="configure config_log build install test"
586         fi
587
588         # start the build
589         (
590                 # we all want to be able to read the output...
591                 LANG=C
592                 export LANG
593
594                 uname -a
595
596                 echo ""
597                 echo "build_test          : $build_test_id"
598                 echo "build_test.fns      : $build_test_fns_id"
599                 echo "local settings file : $build_test_settings_local_file"
600                 echo "local functions file: $build_test_fns_local_file"
601                 echo "used .fns file      : $build_test_used_fns_file"
602                 echo ""
603
604                 # we need to be able to see if a build farm machine is accumulating
605                 # stuck processes. We do this in two ways, as we don't know what style
606                 # of ps it will have
607                 ps xfuw 2> /dev/null
608                 ps -fu $USER 2> /dev/null
609
610                 echo "building $tree with CC=$compiler on $host at "`date`
611                 echo "builddir=$builddir"
612                 echo "prefix=$prefix"
613
614                 echo "Showing limits"
615                 ulimit -a 2> /dev/null
616
617                 # build the timelimit utility
618                 echo "Building timelimit"
619                 mkdir -p $builddir
620                 $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
621
622                 if [ -r $test_root/$tree.svn ]; then
623                   h_rev=`grep revision= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
624                   if [ -n "$h_rev" ]; then
625                         echo "HIGHEST SVN REVISION: $h_rev"
626                   fi
627                   rev=`grep committed-rev= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
628                   if [ -n "$rev" ]; then
629                         echo "BUILD REVISION: $rev"
630                   fi
631                 fi
632
633
634                 if [ "$tree" = "pidl" ] 
635                 then
636                         cd $builddir
637                         perl ./Makefile.PL "$prefix"
638                 fi
639
640                 for action in $actions; do
641
642                     echo Running action $action
643
644                     date
645
646                     cd $builddir || exit 1
647                     export srcdir
648                     df .
649                     mount
650                     vmstat
651
652                     ( action_$action )
653                     action_status=$?
654                     
655                     if [ $action_status != 0 ]; then
656                         echo "ACTION FAILED: $action";
657                     else
658                         echo "ACTION PASSED: $action";
659                     fi
660                     
661                     if [ $action_status != 0 ]; then 
662                         break;
663                     fi
664
665                 done
666
667                 if [ "$LCOV_REPORT" = "yes" ]; then
668                     case "$tree" in
669                         lorikeet-heimdal*)
670                             lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
671                             ;;
672                         *)
673                             # ugly hack for s4, as lcov is otherwise not able to find 
674                             # these files
675                             rm -f heimdal/lib/*/{lex,parse}.{gcda,gcno}
676                             lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
677                             ;;
678                     esac
679                     genhtml -o $builddir/coverage $builddir/$tree.lcov.info
680                 fi
681
682                 if [ "$noclean" = "yes" ]; then
683                     echo cleanup skipped!
684                 else
685                     echo cleaning up
686                     do_make clean
687                 fi
688                 date
689         ) > "$log" 2> "$err"
690
691         if [ "$LCOV_REPORT" = "yes" ]; then
692             chmod u=rwX,g=rX,o=rX -R $builddir/coverage
693             rsync -rct -q --password-file=.password -z --timeout=200 \
694                 $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
695         fi
696
697         cd $test_root
698
699         /bin/rm -rf $prefix
700         if [ "$usingtmpbuild" = "1" ]; then
701             if [ "$noclean" = "yes" ]; then
702                 echo builddir cleanup skipped!
703             else
704                 /bin/rm -rf $builddir
705             fi
706         fi
707         # send the logs to the master site
708         send_logs "$log" "$err"
709
710         # cleanup
711         echo "Ending build of $tree.$compiler in process $$ at `date`"
712         unlock_file "$lck"
713 }
714
715 #########################################################
716 # if you want to build only one project at a time
717 # add 'global_lock' after 'per_run_hook' and
718 # 'global_unlock' to the end of the file
719 global_lock() {
720     lock_file "global.lck" || {
721         exit 0
722     }
723 }
724 global_unlock() {
725     unlock_file "global.lck"
726 }
727
728 #########################################################
729 # enable this on a per host basis only when needed please
730 # (at least for the moment)
731 kill_old_processes() {
732     # this should work on systems with linux like ps
733     (ps uxfw | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
734     # and this should work on sysv style ps
735     (ps -fu $USER | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
736 }
737
738
739 # this is a special fn that allows us to add a "special" hook to the build
740 # farm that we want to do to the build farm. never leave it empty. instead,
741 # use ":" as the fn body.
742 per_run_hook() {
743     # kill old processes on systems with a known problem
744     case $host in
745         nohost)
746             echo "just a placeholder";
747             ;;
748         tridge)
749             kill_old_processes
750             ;;
751         deckchair)
752             rm -f deckchair.fns
753             ;;
754     esac
755     # trim the log if too large
756     if [ "`wc -c < build.log`" -gt 2000000 ]; then
757         rm -f build.log
758     fi
759 }
760
761
762 ######################################################
763 # main code that is run on each call to the build code
764 rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
765
766
767 # build.log can grow to an excessive size, trim it beyond 50M
768 if [ -f build.log ]; then
769   find build.log -size +100000 -exec /bin/rm '{}' \;
770 fi
771