1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 # Augment PATH with relevant stubs/ directories. We do this by actually
4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
5 # prepending that to $PATH in rc.local to avoid the PATH reset in
10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
11 EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs"
12 case "$EVENTSCRIPTS_PATH" in
14 *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;;
18 export EVENTSCRIPTS_PATH
20 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
22 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
23 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
24 "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
25 rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
27 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
28 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
30 export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
31 touch "${CTDB_LOGGING#file:}" || \
32 die "Unable to setup logging for \"$CTDB_LOGGING\""
34 if [ -d "${TEST_SUBDIR}/etc" ] ; then
35 cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
36 export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
38 die "Unable to setup \$CTDB_ETCDIR"
41 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
42 cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
43 export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
45 die "Unable to set \$CTDB_BASE"
49 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
51 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
53 That means that no eventscripts can be tested.
55 One possible explanation:
57 You have CTDB installed via RPMs (or similar), so the regular
58 CTDB_BASE directory is in /etc/ctdb/
62 You have done a regular "configure" and "make install" so the tests
63 are installed under /usr/local/.
65 If so, one possible hack to fix this is to create a symlink:
67 ln -s /etc/ctdb /usr/local/etc/ctdb
69 This is nasty but it works... :-)
74 ######################################################################
76 if "$TEST_VERBOSE" ; then
77 debug () { echo "$@" ; }
82 eventscripts_tests_cleanup_hooks=""
85 eventscripts_test_add_cleanup ()
87 eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*"
90 trap 'eval $eventscripts_tests_cleanup_hooks' 0
93 ######################################################################
95 # General setup fakery
99 debug "Setting up shares (3 existing shares)"
100 # Create 3 fake shares/exports.
101 export FAKE_SHARES=""
102 for i in $(seq 1 3) ; do
103 _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
105 FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
108 export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
109 mkdir -p "$FAKE_PROC_NET_BONDING"
110 rm -f "$FAKE_PROC_NET_BONDING"/*
112 export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
113 mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
114 rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
116 # This can only have 2 levels. We don't want to resort to usings
117 # something dangerous like "rm -r" setup time.
118 export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
119 mkdir -p "$FAKE_IP_STATE"
120 rm -f "$FAKE_IP_STATE"/*/*
121 rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
122 rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
125 export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
126 mkdir -p "${CTDB_DBDIR}/persistent"
128 export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
129 export FAKE_TDB_IS_OK
130 export FAKE_DATE_OUTPUT
132 export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
133 export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
139 debug "Marking TCP port \"${_i}\" as not listening"
140 FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
148 # Replace some shares with non-existent ones.
153 export MISSING_SHARES_TEXT=""
154 for _i in $FAKE_SHARES ; do
155 if [ $_n = "$1" ] ; then
157 _i="${_i%_existing}_missing"
158 debug "Replacing share $_n with missing share \"$_i\""
159 rmdir "$_i" 2>/dev/null || true
160 MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
162 _t="${_t}${_t:+ }${_i}"
168 # Setup some fake /proc/net/bonding files with just enough info for
171 # arg1 is interface name, arg2 is currently active slave (use "None"
172 # if none), arg3 is MII status ("up" or "down").
176 _slave="${2:-${_iface}_sl_0}"
178 _mii_subs="${4:-${_mii_s:-up}}"
179 echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
180 cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
181 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
182 Currently Active Slave: $_slave
185 # Status of 1st pretend adapter
186 MII Status: $_mii_subs
187 # Status of 2nd pretend adapter
188 MII Status: $_mii_subs
192 ethtool_interfaces_down ()
195 echo "Marking interface $_i DOWN for ethtool"
196 touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
200 ethtool_interfaces_up ()
203 echo "Marking interface $_i UP for ethtool"
204 rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
210 echo "# ip rule show"
214 while read _p _x _i _x _t ; do
215 # Remove trailing colon after priority/preference.
217 # Only remove rules that match our priority/preference.
218 [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
220 echo "# ip route show table $_t"
221 ip route show table "$_t"
225 # Copied from 13.per_ip_routing for now... so this is lazy testing :-(
226 ipv4_host_addr_to_net ()
231 # Convert the host address to an unsigned long by splitting out
232 # the octets and doing the math.
234 for _o in $(export IFS="." ; echo $_host) ; do
235 _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
238 # Calculate the mask and apply it.
239 _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
240 _net_ul=$(( $_host_ul & $_mask_ul ))
242 # Now convert to a network address one byte at a time.
244 for _o in $(seq 1 4) ; do
245 _net="$(($_net_ul & 255))${_net:+.}${_net}"
246 _net_ul=$(($_net_ul >> 8))
249 echo "${_net}/${_maskbits}"
252 ######################################################################
256 # Evaluate an expression that probably calls functions or uses
257 # variables from the CTDB functions file. This is used for test
262 . "$CTDB_BASE/functions"
267 # Set output for ctdb command. Option 1st argument is return code.
270 _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out"
273 _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc"
274 echo "${1:-0}" >"$_rc"
276 eventscripts_test_add_cleanup "rm -f $_out $_rc"
283 export FAKE_CTDB_NUMNODES="${1:-3}"
284 echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
286 export FAKE_CTDB_PNN="${2:-0}"
287 echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}"
289 export CTDB_PUBLIC_ADDRESSES="${CTDB_BASE}/public_addresses"
290 if [ -n "$3" ] ; then
291 echo "Setting up CTDB_PUBLIC_ADDRESSES: $3"
292 CTDB_PUBLIC_ADDRESSES=$(mktemp)
296 echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES"
298 eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES"
301 export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
303 export FAKE_CTDB_EXTRA_CONFIG="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh"
304 rm -f "$FAKE_CTDB_EXTRA_CONFIG"
306 export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
307 mkdir -p "$FAKE_CTDB_IFACES_DOWN"
308 rm -f "$FAKE_CTDB_IFACES_DOWN"/*
310 export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
311 mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
312 rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
314 export CTDB_PARTIALLY_ONLINE_INTERFACES
316 export FAKE_CTDB_TUNABLES_OK="MonitorInterval TDBMutexEnabled DatabaseHashSize"
317 export FAKE_CTDB_TUNABLES_OBSOLETE="EventScriptUnhealthyOnTimeout"
322 cat >"$FAKE_CTDB_EXTRA_CONFIG"
329 _swap_total="5857276"
331 if [ "$1" = "bad" ] ; then
336 _swap_free="$_swap_total"
341 export FAKE_PROC_MEMINFO="\
349 Active(anon): 1917580 kB
350 Inactive(anon): 523080 kB
351 Active(file): 504524 kB
352 Inactive(file): 496848 kB
355 SwapTotal: ${_swap_total} kB
356 SwapFree: ${_swap_free} kB
359 export FAKE_FREE_M="\
360 total used free shared buffers cached
361 Mem: 3848 3634 213 0 142 ${_mem_cached}
362 -/+ buffers/cache: 2379 ${_mem_free}
365 export CTDB_MONITOR_FREE_MEMORY
366 export CTDB_MONITOR_FREE_MEMORY_WARN
367 export CTDB_CHECK_SWAP_IS_NOT_USED
370 ctdb_get_interfaces ()
372 # The echo/subshell forces all the output onto 1 line.
373 echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}')
376 ctdb_get_1_interface ()
378 _t=$(ctdb_get_interfaces)
382 # Print all public addresses as: interface IP maskbits
383 # Each line is suitable for passing to takeip/releaseip
384 ctdb_get_all_public_addresses ()
386 _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
387 while IFS="/$IFS" read _ip _maskbits _ifaces ; do
388 echo "$_ifaces $_ip $_maskbits"
392 # Print public addresses on this node as: interface IP maskbits
393 # Each line is suitable for passing to takeip/releaseip
394 ctdb_get_my_public_addresses ()
397 read _x # skip header line
399 while IFS="|" read _x _ip _x _iface _x ; do
400 [ -n "$_iface" ] || continue
401 while IFS="/$IFS" read _i _maskbits _x ; do
402 if [ "$_ip" = "$_i" ] ; then
403 echo $_iface $_ip $_maskbits
406 done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
411 # Prints the 1st public address as: interface IP maskbits
412 # This is suitable for passing to takeip/releaseip
413 ctdb_get_1_public_address ()
415 ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; }
418 ctdb_not_implemented ()
420 export CTDB_NOT_IMPLEMENTED="$1"
421 ctdb_not_implemented="\
422 DEBUG: ctdb: command \"$1\" not implemented in stub"
425 ctdb_fake_scriptstatus ()
434 echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
437 ######################################################################
439 setup_ctdb_policy_routing ()
441 service_name="per_ip_routing"
443 export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
444 export CTDB_PER_IP_ROUTING_RULE_PREF=100
445 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
446 export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
448 # Tests need to create and populate this file
449 rm -f "$CTDB_PER_IP_ROUTING_CONF"
452 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
453 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
454 # 1. If $2 is "default" then a default route is also added.
455 create_policy_routing_config ()
458 _should_add_default="$2"
460 ctdb_get_my_public_addresses |
461 if [ "$_num_ips" = "all" ] ; then
464 { head -n "$_num_ips" ; cat >/dev/null ; }
466 while read _dev _ip _bits ; do
467 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
468 _gw="${_net%.*}.1" # a dumb, calculated default
472 if [ "$_should_add_default" = "default" ] ; then
473 echo "$_ip 0.0.0.0/0 $_gw"
475 done >"$CTDB_PER_IP_ROUTING_CONF"
478 # Check the routes against those that are expected. $1 is the number
479 # of assigned IPs to use (<num>, all), defaulting to 1. If $2 is
480 # "default" then expect default routes to have been added.
484 _should_add_default="$2"
489 ctdb_get_my_public_addresses |
490 if [ "$_num_ips" = "all" ] ; then
493 { head -n "$_num_ips" ; cat >/dev/null ; }
495 while read _dev _ip _bits ; do
496 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
497 _gw="${_net%.*}.1" # a dumb, calculated default
499 _policy_rules="${_policy_rules}
500 ${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip "
501 _policy_routes="${_policy_routes}
502 # ip route show table ctdb.$_ip
503 $_net dev $_dev scope link "
505 if [ "$_should_add_default" = "default" ] ; then
506 _policy_routes="${_policy_routes}
507 default via $_gw dev $_dev "
513 0: from all lookup local ${_policy_rules}
514 32766: from all lookup main
515 32767: from all lookup default ${_policy_routes}
518 simple_test_command dump_routes
522 ######################################################################
526 debug "Setting up NAT gateway"
528 natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
529 mkdir -p "$natgw_config_dir"
531 # These will accumulate, 1 per test... but will be cleaned up at
533 export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
536 while read _ip _master _dev ; do
538 if [ "$_master" = "master" ] ; then
539 export FAKE_CTDB_NATGW_MASTER="$_ip"
541 done >"$CTDB_NATGW_NODES"
543 # Assume all of the nodes are on a /24 network and have IPv4
545 read _ip <"$CTDB_NATGW_NODES"
546 export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
548 # These are fixed. Probably don't use the same network for the
549 # private node IPs. To unset the default gateway just set it to
551 export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
552 export CTDB_NATGW_PUBLIC_IFACE="eth1"
553 export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
554 export CTDB_NATGW_SLAVE_ONLY=""
557 ok_natgw_master_ip_addr_show ()
559 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
561 # This is based on CTDB_NATGW_PUBLIC_IP
565 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
566 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
567 inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
568 valid_lft forever preferred_lft forever
572 ok_natgw_slave_ip_addr_show ()
574 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
576 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
577 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
581 ok_natgw_master_static_routes ()
586 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
587 # This is intentionally different to the code in 11.natgw ;-)
590 _net=$(echo "$_i" | sed -e 's|@.*||')
591 _gw=$(echo "$_i" | sed -e 's|.*@||')
595 _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
598 [ -n "$_gw" ] || continue
599 _t="${_t}${_t:+${_nl}}"
600 _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 "
605 ok_natgw_slave_static_routes ()
610 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
611 # This is intentionally different to the code in 11.natgw ;-)
612 _net=$(echo "$_i" | sed -e 's|@.*||')
614 # The interface for the private network isn't specified as
615 # part of the NATGW configuration and isn't part of the
616 # command to add the route. It is implicitly added by "ip
617 # route" but our stub doesn't do this and adds "ethXXX".
618 _t="${_t}${_t:+${_nl}}"
619 _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 "
624 ######################################################################
626 # Samba/winbind fakery
634 if [ "$1" != "down" ] ; then
636 debug "Marking Samba services as up, listening and managed by CTDB"
637 # Get into known state.
638 eventscript_call ctdb_service_managed
640 # All possible service names for all known distros.
641 for i in "smb" "nmb" "samba" ; do
642 service "$i" force-started
645 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
646 export CTDB_MANAGED_SERVICES="foo samba bar"
648 export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
649 export FAKE_WBINFO_FAIL="no"
651 # Some things in 50.samba are backgrounded and waited for. If
652 # we don't sleep at all then timeouts can happen. This avoids
654 export FAKE_SLEEP_FORCE=0.1
656 debug "Marking Samba services as down, not listening and not managed by CTDB"
657 # Get into known state.
658 eventscript_call ctdb_service_unmanaged
660 # All possible service names for all known distros.
661 for i in "smb" "nmb" "samba" ; do
662 service "$i" force-stopped
665 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
666 export CTDB_MANAGED_SERVICES="foo bar"
667 unset CTDB_MANAGES_SAMBA
669 export FAKE_TCP_LISTEN=""
670 export FAKE_WBINFO_FAIL="yes"
673 # This is ugly but if this file isn't removed before each test
674 # then configuration changes between tests don't stick.
675 rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache"
682 service_name="winbind"
684 if [ "$1" != "down" ] ; then
686 debug "Marking Winbind service as up and managed by CTDB"
687 # Get into known state.
688 eventscript_call ctdb_service_managed
690 service "winbind" force-started
692 export CTDB_MANAGED_SERVICES="foo winbind bar"
694 export FAKE_WBINFO_FAIL="no"
697 debug "Marking Winbind service as down and not managed by CTDB"
698 # Get into known state.
699 eventscript_call ctdb_service_unmanaged
701 service "winbind" force-stopped
703 export CTDB_MANAGED_SERVICES="foo bar"
704 unset CTDB_MANAGES_WINBIND
706 export FAKE_WBINFO_FAIL="yes"
712 debug "Making wbinfo commands fail"
713 FAKE_WBINFO_FAIL="yes"
716 ######################################################################
726 export FAKE_RPCINFO_SERVICES=""
728 export CTDB_NFS_SKIP_SHARE_CHECK="no"
730 export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
731 export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS
733 # Reset the failcounts for nfs services.
734 eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
736 if [ "$1" != "down" ] ; then
737 debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
739 eventscript_call ctdb_service_managed
740 service "nfs" force-started # might not be enough
742 export CTDB_MANAGED_SERVICES="foo nfs bar"
744 rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
746 debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
748 eventscript_call ctdb_service_unmanaged
749 service "nfs" force-stopped # might not be enough
750 eventscript_call startstop_nfs stop
752 export CTDB_MANAGED_SERVICES="foo bar"
753 unset CTDB_MANAGES_NFS
760 export CTDB_NFS_SERVER_MODE="ganesha"
761 if [ "$1" != "down" ] ; then
762 export CTDB_MANAGES_NFS="yes"
765 # We do not support testing the Ganesha-nfsd-specific part of the
767 export CTDB_SKIP_GANESHA_NFSD_CHECK="yes"
768 export CTDB_NFS_SKIP_SHARE_CHECK="yes"
774 debug "Marking RPC service \"${_i}\" as unavailable"
775 FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
782 debug "Marking RPC service \"${_i}\" as available"
787 nlockmgr) _t="3:4" ;;
789 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
792 FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
796 # Set the required result for a particular RPC program having failed
797 # for a certain number of iterations. This is probably still a work
798 # in progress. Note that we could hook aggressively
799 # nfs_check_rpc_service() to try to implement this but we're better
800 # off testing nfs_check_rpc_service() using independent code... even
801 # if it is incomplete and hacky. So, if the 60.nfs eventscript
802 # changes and the tests start to fail then it may be due to this
803 # function being incomplete.
804 rpc_set_service_failure_response ()
807 # The number of failures defaults to the iteration number. This
808 # will be true when we fail from the 1st iteration... but we need
809 # the flexibility to set the number of failures.
810 _numfails="${2:-${iteration}}"
812 _etc="$CTDB_ETCDIR" # shortcut for readability
813 for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
814 if [ -r "$_c" ] ; then
820 # A handy newline. :-)
827 _file=$(ls "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9]."${_progname}.check")
828 [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
830 while read _op _li _actions ; do
837 if [ "$_op" != "%" ] ; then
838 if [ $_numfails $_op $_li ] ; then
842 if [ $(($_numfails $_op $_li)) -eq 0 ] ; then
849 for _action in $_actions ; do
855 nfsd) _ver=3 ; _pn="nfs" ;;
856 lockd) _ver=4 ; _pn="nlockmgr" ;;
857 statd) _pn="status" ;;
860 ERROR: $_pn failed RPC check:
861 rpcinfo: RPC: Program not registered
862 program $_pn version $_ver is not available"
865 _p="rpc.${_progname}"
873 Trying to restart NFS service"
875 if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
876 for _pid in $FAKE_NFSD_THREAD_PIDS ; do
879 ${_bg}Stack trace for nfsd[${_pid}]:
880 ${_bg}[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
886 ${_bg}Starting nfslock: OK
887 ${_bg}Starting nfs: OK"
891 Trying to restart lock manager service
892 ${_bg}Starting nfslock: OK"
895 _t="Trying to restart $_progname [${_p}]"
896 if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
897 for _pid in $FAKE_RPC_THREAD_PIDS ; do
900 Stack trace for ${_p}[${_pid}]:
901 [<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
905 _out="${_out}${_out:+${_nl}}${_t}"
911 required_result $_rc "$_out"
917 ######################################################################
923 service_name="vsftpd"
925 if [ "$1" != "down" ] ; then
926 die "setup_vsftpd up not implemented!!!"
928 debug "Setting up VSFTPD environment: service down, not managed by CTDB"
930 eventscript_call ctdb_service_unmanaged
931 service vsftpd force-stopped
933 export CTDB_MANAGED_SERVICES="foo"
934 unset CTDB_MANAGES_VSFTPD
938 ######################################################################
944 if [ "$1" != "down" ] ; then
945 die "setup_httpd up not implemented!!!"
947 debug "Setting up HTTPD environment: service down, not managed by CTDB"
949 for service_name in "apache2" "httpd" ; do
950 eventscript_call ctdb_service_unmanaged
951 service "$service_name" force-stopped
954 export CTDB_MANAGED_SERVICES="foo"
955 unset CTDB_MANAGES_HTTPD
959 ######################################################################
969 echo "Marking ${_t} as having no active paths"
970 FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
975 CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
978 export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
979 export FAKE_SLEEP_FORCE=0.1
982 ######################################################################
984 # Result and test functions
986 # Set some globals and print the summary.
991 _f=$(basename "$0" ".sh")
993 # Remaining format should be NN.service.event.NNN or NN.service.NNN:
1006 die "Internal error - unknown testcase filename format"
1009 printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
1015 CTDB_BASE="$CTDB_BASE"
1016 CTDB_ETCDIR="$CTDB_ETCDIR"
1017 ctdb client is "$(which ctdb)"
1018 ip command is "$(which ip)"
1022 # Run an eventscript once. The test passes if the return code and
1023 # output match those required.
1025 # Any args are passed to the eventscript.
1029 [ -n "$event" ] || die 'simple_test: $event not set'
1031 _extra_header=$(_extra_header)
1033 echo "Running eventscript \"$script $event${1:+ }$*\""
1035 if $TEST_COMMAND_TRACE ; then
1040 _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
1042 result_check "$_extra_header"
1045 simple_test_event ()
1047 # If something has previously failed then don't continue.
1049 $_passed || return 1
1052 echo "=================================================="
1056 simple_test_command ()
1058 # If something has previously failed then don't continue.
1060 $_passed || return 1
1062 echo "=================================================="
1063 echo "Running command \"$*\""
1069 # Run an eventscript iteratively.
1070 # - 1st argument is the number of iterations.
1071 # - 2nd argument is something to eval to do setup for every iteration.
1072 # The easiest thing to do here is to define a function and pass it
1074 # - Subsequent arguments come in pairs: an iteration number and
1075 # something to eval for that iteration. Each time an iteration
1076 # number is matched the associated argument is given to eval after
1077 # the default setup is done. The iteration numbers need to be given
1078 # in ascending order.
1080 # Some optional args can be given *before* these, surrounded by extra
1081 # "--" args. These args are passed to the eventscript. Quoting is
1084 # One use of the 2nd and further arguments is to call
1085 # required_result() to change what is expected of a particular
1089 [ -n "$event" ] || die 'simple_test: $event not set'
1092 if [ "$1" = "--" ] ; then
1094 while [ "$1" != "--" ] ; do
1095 args="${args}${args:+ }$1"
1105 echo "Running $_repeats iterations of \"$script $event\" $args"
1109 for iteration in $(seq 1 $_repeats) ; do
1110 # This is inefficient because the iteration-specific setup
1111 # might completely replace the default one. However, running
1112 # the default is good because it allows you to revert to a
1113 # particular result without needing to specify it explicitly.
1114 eval $_setup_default
1115 if [ $iteration = "$1" ] ; then
1121 if $TEST_COMMAND_TRACE ; then
1126 _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1)
1129 _fout=$(echo "$_out" | result_filter)
1131 if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
1138 result_print "$_passed" "$_out" "$_rc" "Iteration $iteration"
1141 result_footer "$_result" "$(_extra_header)"