1 # Hey Emacs, this is a -*- shell-script -*- !!! :-)
3 # Augment PATH with relevant stubs/ directories. We do this by actually
4 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
5 # prepending that to $PATH in rc.local to avoid the PATH reset in
10 if [ -d "${TEST_SUBDIR}/stubs" ] ; then
11 EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs"
12 case "$EVENTSCRIPTS_PATH" in
14 *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;;
18 export EVENTSCRIPTS_PATH
20 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
22 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
23 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
24 "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
25 rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
27 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
28 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
30 export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
31 touch "${CTDB_LOGGING#file:}" || \
32 die "Unable to setup logging for \"$CTDB_LOGGING\""
34 if [ -d "${TEST_SUBDIR}/etc" ] ; then
35 cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
36 export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
38 die "Unable to setup \$CTDB_ETCDIR"
41 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
42 cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR"
43 export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb"
45 die "Unable to set \$CTDB_BASE"
49 if [ ! -d "${CTDB_BASE}/events.d" ] ; then
51 ERROR: Directory ${CTDB_BASE}/events.d does not exist.
53 That means that no eventscripts can be tested.
55 One possible explanation:
57 You have CTDB installed via RPMs (or similar), so the regular
58 CTDB_BASE directory is in /etc/ctdb/
62 You have done a regular "configure" and "make install" so the tests
63 are installed under /usr/local/.
65 If so, one possible hack to fix this is to create a symlink:
67 ln -s /etc/ctdb /usr/local/etc/ctdb
69 This is nasty but it works... :-)
74 ######################################################################
76 if "$TEST_VERBOSE" ; then
77 debug () { echo "$@" ; }
82 ######################################################################
84 # General setup fakery
88 debug "Setting up shares (3 existing shares)"
89 # Create 3 fake shares/exports.
91 for i in $(seq 1 3) ; do
92 _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing"
94 FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}"
97 export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding"
98 mkdir -p "$FAKE_PROC_NET_BONDING"
99 rm -f "$FAKE_PROC_NET_BONDING"/*
101 export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down"
102 mkdir -p "$FAKE_ETHTOOL_LINK_DOWN"
103 rm -f "$FAKE_ETHTOOL_LINK_DOWN"/*
105 # This can only have 2 levels. We don't want to resort to usings
106 # something dangerous like "rm -r" setup time.
107 export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state"
108 mkdir -p "$FAKE_IP_STATE"
109 rm -f "$FAKE_IP_STATE"/*/*
110 rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true
111 rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true
114 export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
115 mkdir -p "${CTDB_DBDIR}/persistent"
117 export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
118 export FAKE_TDB_IS_OK
119 export FAKE_DATE_OUTPUT
121 export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
122 export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
128 debug "Marking TCP port \"${_i}\" as not listening"
129 FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
137 # Replace some shares with non-existent ones.
142 export MISSING_SHARES_TEXT=""
143 for _i in $FAKE_SHARES ; do
144 if [ $_n = "$1" ] ; then
146 _i="${_i%_existing}_missing"
147 debug "Replacing share $_n with missing share \"$_i\""
148 rmdir "$_i" 2>/dev/null || true
149 MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}")
151 _t="${_t}${_t:+ }${_i}"
157 # Setup some fake /proc/net/bonding files with just enough info for
160 # arg1 is interface name, arg2 is currently active slave (use "None"
161 # if none), arg3 is MII status ("up" or "down").
165 _slave="${2:-${_iface}_sl_0}"
167 _mii_subs="${4:-${_mii_s:-up}}"
168 echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s"
169 cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF
170 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
171 Currently Active Slave: $_slave
174 # Status of 1st pretend adapter
175 MII Status: $_mii_subs
176 # Status of 2nd pretend adapter
177 MII Status: $_mii_subs
181 ethtool_interfaces_down ()
184 echo "Marking interface $_i DOWN for ethtool"
185 touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
189 ethtool_interfaces_up ()
192 echo "Marking interface $_i UP for ethtool"
193 rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}"
199 echo "# ip rule show"
203 while read _p _x _i _x _t ; do
204 # Remove trailing colon after priority/preference.
206 # Only remove rules that match our priority/preference.
207 [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
209 echo "# ip route show table $_t"
210 ip route show table "$_t"
214 # Copied from 13.per_ip_routing for now... so this is lazy testing :-(
215 ipv4_host_addr_to_net ()
220 # Convert the host address to an unsigned long by splitting out
221 # the octets and doing the math.
223 for _o in $(export IFS="." ; echo $_host) ; do
224 _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
227 # Calculate the mask and apply it.
228 _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
229 _net_ul=$(( $_host_ul & $_mask_ul ))
231 # Now convert to a network address one byte at a time.
233 for _o in $(seq 1 4) ; do
234 _net="$(($_net_ul & 255))${_net:+.}${_net}"
235 _net_ul=$(($_net_ul >> 8))
238 echo "${_net}/${_maskbits}"
241 ######################################################################
245 # Evaluate an expression that probably calls functions or uses
246 # variables from the CTDB functions file. This is used for test
251 . "$CTDB_BASE/functions"
256 # For now this creates the same public addresses each time. However,
257 # it could be made more flexible.
258 setup_public_addresses ()
260 if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \
261 "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then
262 rm "$CTDB_PUBLIC_ADDRESSES"
265 export CTDB_PUBLIC_ADDRESSES=$(mktemp \
266 --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \
267 "public-addresses-XXXXXXXX")
269 echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}"
270 cat >"$CTDB_PUBLIC_ADDRESSES" <<EOF
283 # Need to cope with ctdb_get_pnn(). If a test changes PNN then it
284 # needs to be using a different state directory, otherwise the wrong
285 # PNN can already be cached in the state directory.
288 export FAKE_CTDB_PNN="$1"
289 echo "Setting up PNN ${FAKE_CTDB_PNN}"
291 export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb/${FAKE_CTDB_PNN}"
292 mkdir -p "$CTDB_VARDIR"
299 export FAKE_CTDB_NUMNODES="${1:-3}"
300 echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes"
302 ctdb_set_pnn "${2:-0}"
304 setup_public_addresses
306 export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
308 export FAKE_CTDB_EXTRA_CONFIG="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh"
309 rm -f "$FAKE_CTDB_EXTRA_CONFIG"
311 export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
312 mkdir -p "$FAKE_CTDB_IFACES_DOWN"
313 rm -f "$FAKE_CTDB_IFACES_DOWN"/*
315 export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
316 mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
317 rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
319 export CTDB_PARTIALLY_ONLINE_INTERFACES
321 export FAKE_CTDB_TUNABLES_OK="MonitorInterval TDBMutexEnabled DatabaseHashSize"
322 export FAKE_CTDB_TUNABLES_OBSOLETE="EventScriptUnhealthyOnTimeout"
327 cat >"$FAKE_CTDB_EXTRA_CONFIG"
334 _swap_total="5857276"
336 if [ "$1" = "bad" ] ; then
341 _swap_free="$_swap_total"
346 export FAKE_PROC_MEMINFO="\
354 Active(anon): 1917580 kB
355 Inactive(anon): 523080 kB
356 Active(file): 504524 kB
357 Inactive(file): 496848 kB
360 SwapTotal: ${_swap_total} kB
361 SwapFree: ${_swap_free} kB
364 export FAKE_FREE_M="\
365 total used free shared buffers cached
366 Mem: 3848 3634 213 0 142 ${_mem_cached}
367 -/+ buffers/cache: 2379 ${_mem_free}
370 export CTDB_MONITOR_FREE_MEMORY
371 export CTDB_MONITOR_FREE_MEMORY_WARN
372 export CTDB_CHECK_SWAP_IS_NOT_USED
375 ctdb_get_interfaces ()
377 # The echo/subshell forces all the output onto 1 line.
378 echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}')
381 ctdb_get_1_interface ()
383 _t=$(ctdb_get_interfaces)
387 # Print all public addresses as: interface IP maskbits
388 # Each line is suitable for passing to takeip/releaseip
389 ctdb_get_all_public_addresses ()
391 _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
392 while IFS="/$IFS" read _ip _maskbits _ifaces ; do
393 echo "$_ifaces $_ip $_maskbits"
397 # Print public addresses on this node as: interface IP maskbits
398 # Each line is suitable for passing to takeip/releaseip
399 ctdb_get_my_public_addresses ()
402 read _x # skip header line
404 while IFS="|" read _x _ip _x _iface _x ; do
405 [ -n "$_iface" ] || continue
406 while IFS="/$IFS" read _i _maskbits _x ; do
407 if [ "$_ip" = "$_i" ] ; then
408 echo $_iface $_ip $_maskbits
411 done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
416 # Prints the 1st public address as: interface IP maskbits
417 # This is suitable for passing to takeip/releaseip
418 ctdb_get_1_public_address ()
420 ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; }
423 ctdb_not_implemented ()
425 export CTDB_NOT_IMPLEMENTED="$1"
426 ctdb_not_implemented="\
427 DEBUG: ctdb: command \"$1\" not implemented in stub"
430 ctdb_fake_scriptstatus ()
439 echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
442 ######################################################################
444 setup_ctdb_policy_routing ()
446 service_name="per_ip_routing"
448 export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
449 export CTDB_PER_IP_ROUTING_RULE_PREF=100
450 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
451 export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
453 # Tests need to create and populate this file
454 rm -f "$CTDB_PER_IP_ROUTING_CONF"
457 # Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF.
458 # $1 is the number of assigned IPs to use (<num>, all), defaulting to
459 # 1. If $2 is "default" then a default route is also added.
460 create_policy_routing_config ()
463 _should_add_default="$2"
465 ctdb_get_my_public_addresses |
466 if [ "$_num_ips" = "all" ] ; then
469 { head -n "$_num_ips" ; cat >/dev/null ; }
471 while read _dev _ip _bits ; do
472 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
473 _gw="${_net%.*}.1" # a dumb, calculated default
477 if [ "$_should_add_default" = "default" ] ; then
478 echo "$_ip 0.0.0.0/0 $_gw"
480 done >"$CTDB_PER_IP_ROUTING_CONF"
483 # Check the routes against those that are expected. $1 is the number
484 # of assigned IPs to use (<num>, all), defaulting to 1. If $2 is
485 # "default" then expect default routes to have been added.
489 _should_add_default="$2"
494 ctdb_get_my_public_addresses |
495 if [ "$_num_ips" = "all" ] ; then
498 { head -n "$_num_ips" ; cat >/dev/null ; }
500 while read _dev _ip _bits ; do
501 _net=$(ipv4_host_addr_to_net "$_ip" "$_bits")
502 _gw="${_net%.*}.1" # a dumb, calculated default
504 _policy_rules="${_policy_rules}
505 ${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip "
506 _policy_routes="${_policy_routes}
507 # ip route show table ctdb.$_ip
508 $_net dev $_dev scope link "
510 if [ "$_should_add_default" = "default" ] ; then
511 _policy_routes="${_policy_routes}
512 default via $_gw dev $_dev "
518 0: from all lookup local ${_policy_rules}
519 32766: from all lookup main
520 32767: from all lookup default ${_policy_routes}
523 simple_test_command dump_routes
527 ######################################################################
529 ctdb_catdb_format_pairs ()
533 while read _k _v ; do
534 _kn=$(echo -n "$_k" | wc -c)
535 _vn=$(echo -n "$_v" | wc -c)
537 key(${_kn}) = "${_k}"
540 data(${_vn}) = "${_v}"
543 _count=$(($_count + 1))
546 echo "Dumped ${_count} records"
549 check_ctdb_tdb_statd_state ()
551 ctdb_get_my_public_addresses |
552 while read _x _sip _x ; do
554 echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
557 ctdb_catdb_format_pairs | {
559 simple_test_command ctdb catdb ctdb.tdb
563 check_statd_callout_smnotify ()
565 _state_even=$(( $(date '+%s') / 2 * 2))
566 _state_odd=$(($_state_even + 1))
570 ctdb_get_my_public_addresses |
571 while read _x _sip _x ; do
574 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
575 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
576 --client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
577 --client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
582 simple_test_event "notify"
586 ######################################################################
590 debug "Setting up NAT gateway"
592 natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
593 mkdir -p "$natgw_config_dir"
595 # These will accumulate, 1 per test... but will be cleaned up at
597 export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
600 while read _ip _master _dev ; do
602 if [ "$_master" = "master" ] ; then
603 export FAKE_CTDB_NATGW_MASTER="$_ip"
605 done >"$CTDB_NATGW_NODES"
607 # Assume all of the nodes are on a /24 network and have IPv4
609 read _ip <"$CTDB_NATGW_NODES"
610 export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
612 # These are fixed. Probably don't use the same network for the
613 # private node IPs. To unset the default gateway just set it to
615 export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
616 export CTDB_NATGW_PUBLIC_IFACE="eth1"
617 export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
618 export CTDB_NATGW_SLAVE_ONLY=""
621 ok_natgw_master_ip_addr_show ()
623 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
625 # This is based on CTDB_NATGW_PUBLIC_IP
629 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
630 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
631 inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
632 valid_lft forever preferred_lft forever
636 ok_natgw_slave_ip_addr_show ()
638 _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
640 1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
641 link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
645 ok_natgw_master_static_routes ()
650 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
651 # This is intentionally different to the code in 11.natgw ;-)
654 _net=$(echo "$_i" | sed -e 's|@.*||')
655 _gw=$(echo "$_i" | sed -e 's|.*@||')
659 _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
662 [ -n "$_gw" ] || continue
663 _t="${_t}${_t:+${_nl}}"
664 _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 "
666 _t=$(echo "$_t" | sort)
670 ok_natgw_slave_static_routes ()
675 for _i in $CTDB_NATGW_STATIC_ROUTES ; do
676 # This is intentionally different to the code in 11.natgw ;-)
677 _net=$(echo "$_i" | sed -e 's|@.*||')
679 # The interface for the private network isn't specified as
680 # part of the NATGW configuration and isn't part of the
681 # command to add the route. It is implicitly added by "ip
682 # route" but our stub doesn't do this and adds "ethXXX".
683 _t="${_t}${_t:+${_nl}}"
684 _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 "
686 _t=$(echo "$_t" | sort)
690 ######################################################################
692 # Samba/winbind fakery
700 if [ "$1" != "down" ] ; then
702 debug "Marking Samba services as up, listening and managed by CTDB"
703 # Get into known state.
704 eventscript_call ctdb_service_managed
706 # All possible service names for all known distros.
707 for i in "smb" "nmb" "samba" ; do
708 service "$i" force-started
711 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
712 export CTDB_MANAGED_SERVICES="foo samba bar"
714 export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
715 export FAKE_WBINFO_FAIL="no"
717 # Some things in 50.samba are backgrounded and waited for. If
718 # we don't sleep at all then timeouts can happen. This avoids
720 export FAKE_SLEEP_FORCE=0.1
722 debug "Marking Samba services as down, not listening and not managed by CTDB"
723 # Get into known state.
724 eventscript_call ctdb_service_unmanaged
726 # All possible service names for all known distros.
727 for i in "smb" "nmb" "samba" ; do
728 service "$i" force-stopped
731 export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
732 export CTDB_MANAGED_SERVICES="foo bar"
733 unset CTDB_MANAGES_SAMBA
735 export FAKE_TCP_LISTEN=""
736 export FAKE_WBINFO_FAIL="yes"
744 service_name="winbind"
746 if [ "$1" != "down" ] ; then
748 debug "Marking Winbind service as up and managed by CTDB"
749 # Get into known state.
750 eventscript_call ctdb_service_managed
752 service "winbind" force-started
754 export CTDB_MANAGED_SERVICES="foo winbind bar"
756 export FAKE_WBINFO_FAIL="no"
759 debug "Marking Winbind service as down and not managed by CTDB"
760 # Get into known state.
761 eventscript_call ctdb_service_unmanaged
763 service "winbind" force-stopped
765 export CTDB_MANAGED_SERVICES="foo bar"
766 unset CTDB_MANAGES_WINBIND
768 export FAKE_WBINFO_FAIL="yes"
774 debug "Making wbinfo commands fail"
775 FAKE_WBINFO_FAIL="yes"
778 ######################################################################
788 export FAKE_RPCINFO_SERVICES=""
790 export CTDB_NFS_SKIP_SHARE_CHECK="no"
792 export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
793 export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS
795 # Reset the failcounts for nfs services.
796 eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
798 if [ "$1" != "down" ] ; then
799 debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
801 eventscript_call ctdb_service_managed
802 service "nfs" force-started # might not be enough
804 export CTDB_MANAGED_SERVICES="foo nfs bar"
806 rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
808 debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
810 eventscript_call ctdb_service_unmanaged
811 service "nfs" force-stopped # might not be enough
812 eventscript_call startstop_nfs stop
814 export CTDB_MANAGED_SERVICES="foo bar"
815 unset CTDB_MANAGES_NFS
818 # This is really nasty. However, when we test NFS we don't
819 # actually test statd-callout. If we leave it there then left
820 # over, backgrounded instances of statd-callout will do horrible
821 # things with the "ctdb ip" stub and cause the actual
822 # statd-callout tests that follow to fail.
823 rm "${CTDB_BASE}/statd-callout"
829 export CTDB_NFS_SERVER_MODE="ganesha"
830 if [ "$1" != "down" ] ; then
831 export CTDB_MANAGES_NFS="yes"
834 # We do not support testing the Ganesha-nfsd-specific part of the
836 export CTDB_SKIP_GANESHA_NFSD_CHECK="yes"
837 export CTDB_NFS_SKIP_SHARE_CHECK="yes"
843 debug "Marking RPC service \"${_i}\" as unavailable"
844 FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
851 debug "Marking RPC service \"${_i}\" as available"
856 nlockmgr) _t="3:4" ;;
858 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
861 FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
868 _etc="$CTDB_ETCDIR" # shortcut for readability
869 for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
870 if [ -r "$_c" ] ; then
877 program_stack_traces ()
883 for _pid in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do
884 [ $_count -le $_max ] || break
887 Stack trace for ${_prog}[${_pid}]:
888 [<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
890 _count=$(($_count + 1))
903 nlockmgr) echo "lockd" ;;
904 status) echo "statd" ;;
910 # Set the required result for a particular RPC program having failed
911 # for a certain number of iterations. This is probably still a work
912 # in progress. Note that we could hook aggressively
913 # nfs_check_rpc_service() to try to implement this but we're better
914 # off testing nfs_check_rpc_service() using independent code... even
915 # if it is incomplete and hacky. So, if the 60.nfs eventscript
916 # changes and the tests start to fail then it may be due to this
917 # function being incomplete.
918 rpc_set_service_failure_response ()
921 # The number of failures defaults to the iteration number. This
922 # will be true when we fail from the 1st iteration... but we need
923 # the flexibility to set the number of failures.
924 _numfails="${2:-${iteration:-1}}"
926 _progname=$(convert_progname "$_rpc_service")
930 # A handy newline. :-)
937 _file=$(ls "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9]."${_progname}.check")
938 [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
940 while read _op _li _actions ; do
947 if [ "$_op" != "%" ] ; then
948 if [ $_numfails $_op $_li ] ; then
952 if [ $_numfails -gt 0 -a $(($_numfails $_op $_li)) -eq 0 ] ; then
959 for _action in $_actions ; do
963 case "$_rpc_service" in
968 ERROR: $_rpc_service failed RPC check:
969 rpcinfo: RPC: Program not registered
970 program $_rpc_service version $_ver is not available"
973 _p="rpc.${_progname}"
975 *:b) _bg=mark_background ;;
980 _t=$(program_stack_traces "nfsd" 5)
981 _t="${_t}${_t:+${_nl}}Starting nfslock: OK
983 _t=$(echo "$_t" | $_bg)
985 Trying to restart NFS service
989 _t=$(echo "Starting nfslock: OK" | $_bg)
990 _t="Trying to restart lock manager service${_t:+${_nl}}${_t}"
993 _t="Trying to restart $_progname [${_p}]"
994 _stacks=$(program_stack_traces "$_p" 5)
995 _t="${_t}${_stacks:+${_nl}}${_stacks}"
997 _out="${_out}${_out:+${_nl}}${_t}"
1003 required_result $_rc "$_out"
1009 ######################################################################
1015 service_name="vsftpd"
1017 if [ "$1" != "down" ] ; then
1018 die "setup_vsftpd up not implemented!!!"
1020 debug "Setting up VSFTPD environment: service down, not managed by CTDB"
1022 eventscript_call ctdb_service_unmanaged
1023 service vsftpd force-stopped
1025 export CTDB_MANAGED_SERVICES="foo"
1026 unset CTDB_MANAGES_VSFTPD
1030 ######################################################################
1036 if [ "$1" != "down" ] ; then
1037 die "setup_httpd up not implemented!!!"
1039 debug "Setting up HTTPD environment: service down, not managed by CTDB"
1041 for service_name in "apache2" "httpd" ; do
1042 eventscript_call ctdb_service_unmanaged
1043 service "$service_name" force-stopped
1046 export CTDB_MANAGED_SERVICES="foo"
1047 unset CTDB_MANAGES_HTTPD
1051 ######################################################################
1061 echo "Marking ${_t} as having no active paths"
1062 FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}"
1067 CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}"
1070 export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES
1071 export FAKE_SLEEP_FORCE=0.1
1074 ######################################################################
1076 # Result and test functions
1078 # Set some globals and print the summary.
1083 _f=$(basename "$0" ".sh")
1085 # Remaining format should be NN.service.event.NNN or NN.service.NNN:
1093 script_dir="${CTDB_BASE}/events.d"
1098 script_dir="${CTDB_BASE}/events.d"
1103 script_dir="${CTDB_BASE}"
1108 script_dir="${CTDB_BASE}"
1111 [ -r "${script_dir}/${script}" ] || \
1112 die "Internal error - unable to find script \"${script_dir}/${script}\""
1114 printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
1117 # Run an eventscript once. The test passes if the return code and
1118 # output match those required.
1120 # Any args are passed to the eventscript.
1124 [ -n "$event" ] || die 'simple_test: $event not set'
1130 echo "Running script \"$script $event${args:+ }$args\""
1137 ##################################################
1138 CTDB_BASE="$CTDB_BASE"
1139 CTDB_ETCDIR="$CTDB_ETCDIR"
1140 ctdb client is "$(which ctdb)"
1141 ip command is "$(which ip)"
1145 script_test "${script_dir}/${script}" "$event" "$@"
1151 simple_test_event ()
1153 # If something has previously failed then don't continue.
1155 $_passed || return 1
1158 echo "=================================================="
1162 simple_test_command ()
1167 # Run an eventscript iteratively.
1168 # - 1st argument is the number of iterations.
1169 # - 2nd argument is something to eval to do setup for every iteration.
1170 # The easiest thing to do here is to define a function and pass it
1172 # - Subsequent arguments come in pairs: an iteration number and
1173 # something to eval for that iteration. Each time an iteration
1174 # number is matched the associated argument is given to eval after
1175 # the default setup is done. The iteration numbers need to be given
1176 # in ascending order.
1178 # Some optional args can be given *before* these, surrounded by extra
1179 # "--" args. These args are passed to the eventscript. Quoting is
1182 # One use of the 2nd and further arguments is to call
1183 # required_result() to change what is expected of a particular
1187 [ -n "$event" ] || die 'simple_test: $event not set'
1190 if [ "$1" = "--" ] ; then
1192 while [ "$1" != "--" ] ; do
1193 args="${args}${args:+ }$1"
1203 echo "Running $_repeats iterations of \"$script $event\" $args"
1210 --------------------------------------------------
1211 CTDB_BASE="$CTDB_BASE"
1212 CTDB_ETCDIR="$CTDB_ETCDIR"
1213 ctdb client is "$(which ctdb)"
1214 ip command is "$(which ip)"
1215 --------------------------------------------------
1219 for iteration in $(seq 1 $_repeats) ; do
1220 # This is inefficient because the iteration-specific setup
1221 # might completely replace the default one. However, running
1222 # the default is good because it allows you to revert to a
1223 # particular result without needing to specify it explicitly.
1224 eval $_setup_default
1225 if [ $iteration = "$1" ] ; then
1234 ##################################################
1235 Iteration $iteration
1240 if $TEST_COMMAND_TRACE ; then
1245 _out=$($_shell "${script_dir}/${script}" "$event" $args 2>&1)
1248 _fout=$(echo "$_out" | result_filter)
1250 if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
1257 result_print "$_passed" "$_out" "$_rc"
1260 result_footer "$_result"
1263 # Run an NFS eventscript iteratively.
1265 # - 1st argument is the number of iterations.
1267 # - 2nd argument is the NFS/RPC service being tested
1269 # rpcinfo is used on each iteration to test the availability of the
1272 # - Subsequent arguments come in pairs: an iteration number and
1273 # something to eval before that iteration. Each time an iteration
1274 # number is matched the associated argument is given to eval after
1275 # the default setup is done. The iteration numbers need to be given
1276 # in ascending order.
1278 # These arguments can allow a service to be started or stopped
1279 # before a particular iteration.
1287 echo "Running $_repeats iterations of \"$script $event\" $args"
1289 _iterate_failcount=0
1290 for _iteration in $(seq 1 $_repeats) ; do
1291 # This is not a numerical comparison because $1 will often not
1293 if [ "$_iteration" = "$1" ] ; then
1294 debug "##################################################"
1296 debug "##################################################"
1299 if rpcinfo -T tcp localhost "$_rpc_service" >/dev/null 2>&1 ; then
1300 _iterate_failcount=0
1302 _iterate_failcount=$(($_iterate_failcount + 1))
1304 rpc_set_service_failure_response "$_rpc_service" $_iterate_failcount
1305 _out=$(simple_test 2>&1)
1307 if "$TEST_VERBOSE" || [ $_ret -ne 0 ] ; then
1308 echo "##################################################"
1309 echo "Iteration ${_iteration}:"
1312 if [ $_ret -ne 0 ] ; then