X-Git-Url: http://git.samba.org/?a=blobdiff_plain;f=ctdb%2Ftests%2Feventscripts%2Fscripts%2Flocal.sh;h=ea900ebcd0d8c99e32ef48af9915c9c46b843bb6;hb=72e415be4cdb39ba3443adf9a4a441b5474868fe;hp=e1ae84ceae13bb36a060427f3d28324e20aa3f4e;hpb=0e1f20fe03b9303c29deb1a82dba8b0d457e1677;p=samba.git diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index e1ae84ceae1..ea900ebcd0d 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -9,25 +9,36 @@ EVENTSCRIPTS_PATH="" if [ -d "${TEST_SUBDIR}/stubs" ] ; then EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs" + case "$EVENTSCRIPTS_PATH" in + /*) : ;; + *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;; + esac + export CTDB_HELPER_BINDIR="$EVENTSCRIPTS_PATH" fi export EVENTSCRIPTS_PATH PATH="${EVENTSCRIPTS_PATH}:${PATH}" +export CTDB="ctdb" + export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts" if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \ "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR" fi mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR" -export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb" +export CTDB_SCRIPT_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/script-state" + +export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb" +touch "${CTDB_LOGGING#file:}" || \ + die "Unable to setup logging for \"$CTDB_LOGGING\"" -if [ -d "${TEST_SUBDIR}/etc" ] ; then +if [ -d "${TEST_SUBDIR}/etc" ] ; then cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR" - export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc" + export CTDB_SYS_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc" else - die "Unable to setup \$CTDB_ETCDIR" + die "Unable to setup \$CTDB_SYS_ETCDIR" fi if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then @@ -71,17 +82,6 @@ else debug () { : ; } fi -eventscripts_tests_cleanup_hooks="" - -# This loses quoting! -eventscripts_test_add_cleanup () -{ - eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*" -} - -trap 'eval $eventscripts_tests_cleanup_hooks' 0 - - ###################################################################### # General setup fakery @@ -112,6 +112,20 @@ setup_generic () rm -f "$FAKE_IP_STATE"/*/* rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true + + + export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db" + export CTDB_DBDIR_PERSISTENT="${CTDB_DBDIR}/persistent" + export CTDB_DBDIR_STATE="${CTDB_DBDIR}/state" + mkdir -p "$CTDB_DBDIR_PERSISTENT" + mkdir -p "$CTDB_DBDIR_STATE" + + export FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + export FAKE_TDB_IS_OK + export FAKE_DATE_OUTPUT + + export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN + export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR") } tcp_port_down () @@ -186,11 +200,6 @@ ethtool_interfaces_up () done } -setup_nmap_output_filter () -{ - OUT_FILTER="-e 's@^(DEBUG: # Nmap 5.21 scan initiated) .+ (as:)@\1 DATE \2@' -e 's@^(DEBUG: # Nmap done at) .+ (--)@\1 DATE \2@'" -} - dump_routes () { echo "# ip rule show" @@ -250,16 +259,43 @@ eventscript_call () ) } -# Set output for ctdb command. Option 1st argument is return code. -ctdb_set_output () +# For now this creates the same public addresses each time. However, +# it could be made more flexible. +setup_public_addresses () { - _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out" - cat >"$_out" + if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \ + "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then + rm "$CTDB_PUBLIC_ADDRESSES" + fi - _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc" - echo "${1:-0}" >"$_rc" + export CTDB_PUBLIC_ADDRESSES=$(mktemp \ + --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \ + "public-addresses-XXXXXXXX") + + echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}" + cat >"$CTDB_PUBLIC_ADDRESSES" <>"$CTDB_PUBLIC_ADDRESSES" - done - eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" - fi + setup_public_addresses export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb" + export FAKE_CTDB_EXTRA_CONFIG="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-config.sh" + rm -f "$FAKE_CTDB_EXTRA_CONFIG" + export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down" mkdir -p "$FAKE_CTDB_IFACES_DOWN" rm -f "$FAKE_CTDB_IFACES_DOWN"/* @@ -292,12 +321,78 @@ setup_ctdb () export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus" mkdir -p "$FAKE_CTDB_SCRIPTSTATUS" rm -f "$FAKE_CTDB_SCRIPTSTATUS"/* + + export CTDB_PARTIALLY_ONLINE_INTERFACES + + export FAKE_CTDB_TUNABLES_OK="MonitorInterval TDBMutexEnabled DatabaseHashSize" + export FAKE_CTDB_TUNABLES_OBSOLETE="EventScriptUnhealthyOnTimeout" +} + +setup_config () +{ + cat >"$FAKE_CTDB_EXTRA_CONFIG" +} + +validate_percentage () +{ + case "$1" in + [0-9]|[0-9][0-9]|100) return 0 ;; + *) echo "WARNING: ${1} is an invalid percentage${2:+\" in }${2}${2:+\"}" + return 1 + esac +} + +setup_memcheck () +{ + _mem_usage="${1:-10}" # Default is 10% + _swap_usage="${2:-0}" # Default is 0% + + setup_ctdb + + _swap_total=5857276 + _swap_free=$(( (100 - $_swap_usage) * $_swap_total / 100 )) + + _mem_total=3940712 + _mem_free=225268 + _mem_buffers=146120 + _mem_cached=$(( $_mem_total * (100 - $_mem_usage) / 100 - $_mem_free - $_mem_buffers )) + + export FAKE_PROC_MEMINFO="\ +MemTotal: ${_mem_total} kB +MemFree: ${_mem_free} kB +Buffers: ${_mem_buffers} kB +Cached: ${_mem_cached} kB +SwapCached: 56016 kB +Active: 2422104 kB +Inactive: 1019928 kB +Active(anon): 1917580 kB +Inactive(anon): 523080 kB +Active(file): 504524 kB +Inactive(file): 496848 kB +Unevictable: 4844 kB +Mlocked: 4844 kB +SwapTotal: ${_swap_total} kB +SwapFree: ${_swap_free} kB +..." + + export CTDB_MONITOR_MEMORY_USAGE + export CTDB_MONITOR_SWAP_USAGE +} + +setup_fscheck () +{ + export FAKE_FS_USE="${1:-10}" # Default is 10% usage + + # Causes some variables to be exported + setup_ctdb + + export CTDB_MONITOR_FILESYSTEM_USAGE } ctdb_get_interfaces () { # The echo/subshell forces all the output onto 1 line. - echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') + echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}') } ctdb_get_1_interface () @@ -320,10 +415,10 @@ ctdb_get_all_public_addresses () # Each line is suitable for passing to takeip/releaseip ctdb_get_my_public_addresses () { - ctdb ip -v -Y | { + ctdb ip -v -X | { read _x # skip header line - while IFS=":" read _x _ip _x _iface _x ; do + while IFS="|" read _x _ip _x _iface _x ; do [ -n "$_iface" ] || continue while IFS="/$IFS" read _i _maskbits _x ; do if [ "$_ip" = "$_i" ] ; then @@ -339,7 +434,7 @@ ctdb_get_my_public_addresses () # This is suitable for passing to takeip/releaseip ctdb_get_1_public_address () { - ctdb_get_my_public_addresses | head -n 1 + ctdb_get_my_public_addresses | { head -n 1 ; cat >/dev/null ; } } ctdb_not_implemented () @@ -361,8 +456,12 @@ ctdb_fake_scriptstatus () echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script" } +###################################################################### + setup_ctdb_policy_routing () { + service_name="per_ip_routing" + export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing" export CTDB_PER_IP_ROUTING_RULE_PREF=100 export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000 @@ -372,28 +471,347 @@ setup_ctdb_policy_routing () rm -f "$CTDB_PER_IP_ROUTING_CONF" } +# Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF. +# $1 is the number of assigned IPs to use (, all), defaulting to +# 1. If $2 is "default" then a default route is also added. +create_policy_routing_config () +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ] ; then + cat + else + { head -n "$_num_ips" ; cat >/dev/null ; } + fi | + while read _dev _ip _bits ; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.254" # a dumb, calculated default + + echo "$_ip $_net" + + if [ "$_should_add_default" = "default" ] ; then + echo "$_ip 0.0.0.0/0 $_gw" + fi + done >"$CTDB_PER_IP_ROUTING_CONF" +} + +# Check the routes against those that are expected. $1 is the number +# of assigned IPs to use (, all), defaulting to 1. If $2 is +# "default" then expect default routes to have been added. +check_routes () +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + _policy_rules="" + _policy_routes="" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ] ; then + cat + else + { head -n "$_num_ips" ; cat >/dev/null ; } + fi | { + while read _dev _ip _bits ; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.254" # a dumb, calculated default + + _policy_rules="${_policy_rules} +${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip " + _policy_routes="${_policy_routes} +# ip route show table ctdb.$_ip +$_net dev $_dev scope link " + + if [ "$_should_add_default" = "default" ] ; then + _policy_routes="${_policy_routes} +default via $_gw dev $_dev " + fi + done + + ok <"$CTDB_LVS_NODES" +} + +check_ipvsadm () +{ + if [ "$1" = "NULL" ] ; then + required_result 0 < mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +EOF + else + required_result 0 < mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet ${CTDB_LVS_PUBLIC_IP}/32 scope ${_scope} lo + valid_lft forever preferred_lft forever +EOF + fi + + simple_test_command ip addr show dev lo +} + +###################################################################### + +ctdb_catdb_format_pairs () +{ + _count=0 + + while read _k _v ; do + _kn=$(echo -n "$_k" | wc -c) + _vn=$(echo -n "$_v" | wc -c) + cat <"$CTDB_NATGW_NODES" + + # Assume all of the nodes are on a /24 network and have IPv4 + # addresses: + read _ip <"$CTDB_NATGW_NODES" + export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24" + + # These are fixed. Probably don't use the same network for the + # private node IPs. To unset the default gateway just set it to + # "". :-) + export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24" + export CTDB_NATGW_PUBLIC_IFACE="eth1" + export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254" + export CTDB_NATGW_SLAVE_ONLY="" +} + +ok_natgw_master_ip_addr_show () +{ + _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@') + + # This is based on CTDB_NATGW_PUBLIC_IP + _brd="10.1.1.255" + +ok < mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether ${_mac} brd ff:ff:ff:ff:ff:ff + inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE} + valid_lft forever preferred_lft forever +EOF +} + +ok_natgw_slave_ip_addr_show () +{ + _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@') +ok < mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether ${_mac} brd ff:ff:ff:ff:ff:ff +EOF +} + +ok_natgw_master_static_routes () +{ + _nl=" +" + _t="" + for _i in $CTDB_NATGW_STATIC_ROUTES ; do + # This is intentionally different to the code in 11.natgw ;-) + case "$_i" in + *@*) + _net=$(echo "$_i" | sed -e 's|@.*||') + _gw=$(echo "$_i" | sed -e 's|.*@||') + ;; + *) + _net="$_i" + _gw="$CTDB_NATGW_DEFAULT_GATEWAY" + esac + + [ -n "$_gw" ] || continue + _t="${_t}${_t:+${_nl}}" + _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 " + done + _t=$(echo "$_t" | sort) + ok "$_t" +} + +ok_natgw_slave_static_routes () +{ + _nl=" +" + _t="" + for _i in $CTDB_NATGW_STATIC_ROUTES ; do + # This is intentionally different to the code in 11.natgw ;-) + _net=$(echo "$_i" | sed -e 's|@.*||') + + # The interface for the private network isn't specified as + # part of the NATGW configuration and isn't part of the + # command to add the route. It is implicitly added by "ip + # route" but our stub doesn't do this and adds "ethXXX". + _t="${_t}${_t:+${_nl}}" + _t="${_t}${_net} via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX metric 10 " + done + _t=$(echo "$_t" | sort) + ok "$_t" +} + +###################################################################### + +# Samba/winbind fakery setup_samba () { setup_ctdb + service_name="samba" + if [ "$1" != "down" ] ; then debug "Marking Samba services as up, listening and managed by CTDB" # Get into known state. - for i in "samba" "winbind" ; do - eventscript_call ctdb_service_managed "$i" - done + eventscript_call ctdb_service_managed + # All possible service names for all known distros. - for i in "smb" "nmb" "winbind" "samba" ; do + for i in "smb" "nmb" "samba" ; do service "$i" force-started done export CTDB_SAMBA_SKIP_SHARE_CHECK="no" - export CTDB_MANAGED_SERVICES="foo samba winbind bar" + export CTDB_MANAGED_SERVICES="foo samba bar" export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139" export FAKE_WBINFO_FAIL="no" @@ -405,26 +823,69 @@ setup_samba () else debug "Marking Samba services as down, not listening and not managed by CTDB" # Get into known state. - for i in "samba" "winbind" ; do - eventscript_call ctdb_service_unmanaged "$i" - done + eventscript_call ctdb_service_unmanaged + # All possible service names for all known distros. - for i in "smb" "nmb" "winbind" "samba" ; do + for i in "smb" "nmb" "samba" ; do service "$i" force-stopped done export CTDB_SAMBA_SKIP_SHARE_CHECK="no" export CTDB_MANAGED_SERVICES="foo bar" unset CTDB_MANAGES_SAMBA - unset CTDB_MANAGES_WINBIND export FAKE_TCP_LISTEN="" export FAKE_WBINFO_FAIL="yes" fi +} + +samba_setup_fake_threads () +{ + export FAKE_SMBD_THREAD_PIDS="$*" - # This is ugly but if this file isn't removed before each test - # then configuration changes between tests don't stick. - rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache" + _nl=" +" + _out="" + _count=0 + for _pid ; do + [ "$_count" -lt 5 ] || break + _t=$(program_stack_trace "smbd" $_pid) + _out="${_out:+${_out}${_nl}}${_t}" + _count=$((_count + 1)) + done + SAMBA_STACK_TRACES="$_out" +} + +setup_winbind () +{ + setup_ctdb + + service_name="winbind" + + if [ "$1" != "down" ] ; then + + debug "Marking Winbind service as up and managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_managed + + service "winbind" force-started + + export CTDB_MANAGED_SERVICES="foo winbind bar" + + export FAKE_WBINFO_FAIL="no" + + else + debug "Marking Winbind service as down and not managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_unmanaged + + service "winbind" force-stopped + + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_WINBIND + + export FAKE_WBINFO_FAIL="yes" + fi } wbinfo_down () @@ -441,45 +902,62 @@ setup_nfs () { setup_ctdb + service_name="nfs" + export FAKE_RPCINFO_SERVICES="" export CTDB_NFS_SKIP_SHARE_CHECK="no" - export CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no" + + export RPCNFSDCOUNT + + # This doesn't even need to exist + export CTDB_NFS_EXPORTS_FILE="$EVENTSCRIPTS_TESTS_VAR_DIR/etc-exports" # Reset the failcounts for nfs services. eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' - rpc_fail_limits_file="${EVENTSCRIPTS_TESTS_VAR_DIR}/rpc_fail_limits" - - # Force this file to exist so tests can be individually run. - if [ ! -f "$rpc_fail_limits_file" ] ; then - # This is gross... but is needed to fake through the nfs monitor event. - eventscript_call ctdb_service_managed "nfs" - service "nfs" force-started # might not be enough - CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \ - CTDB_MANAGES_NFS="yes" \ - "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file" - fi - if [ "$1" != "down" ] ; then debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB" - eventscript_call ctdb_service_managed "nfs" - service "nfs" force-started # might not be enough + eventscript_call ctdb_service_managed + service "nfs" force-started + service "nfslock" force-started export CTDB_MANAGED_SERVICES="foo nfs bar" - rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" + rpc_services_up \ + "portmapper" "nfs" "mountd" "rquotad" "nlockmgr" "status" + + nfs_setup_fake_threads "nfsd" + nfs_setup_fake_threads "rpc.foobar" # Just set the variable to empty else debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" - eventscript_call ctdb_service_unmanaged "nfs" - service "nfs" force-stopped # might not be enough - eventscript_call startstop_nfs stop + eventscript_call ctdb_service_unmanaged + service "nfs" force-stopped + service "nfslock" force-stopped export CTDB_MANAGED_SERVICES="foo bar" unset CTDB_MANAGES_NFS fi + + # This is really nasty. However, when we test NFS we don't + # actually test statd-callout. If we leave it there then left + # over, backgrounded instances of statd-callout will do horrible + # things with the "ctdb ip" stub and cause the actual + # statd-callout tests that follow to fail. + rm "${CTDB_BASE}/statd-callout" +} + +setup_nfs_ganesha () +{ + setup_nfs "$@" + export CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-ganesha-callout" + if [ "$1" != "down" ] ; then + export CTDB_MANAGES_NFS="yes" + fi + + export CTDB_NFS_SKIP_SHARE_CHECK="yes" } rpc_services_down () @@ -495,11 +973,12 @@ rpc_services_up () for _i ; do debug "Marking RPC service \"${_i}\" as available" case "$_i" in - nfs) _t="2:3" ;; - mountd) _t="1:3" ;; - rquotad) _t="1:2" ;; - nlockmgr) _t="3:4" ;; - status) _t="1:1" ;; + portmapper) _t="2:4" ;; + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; *) die "Internal error - unsupported RPC service \"${_i}\"" ;; esac @@ -507,6 +986,78 @@ rpc_services_up () done } + +nfs_load_config () +{ + _etc="$CTDB_SYS_ETCDIR" # shortcut for readability + for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do + if [ -r "$_c" ] ; then + . "$_c" + break + fi + done +} + +nfs_setup_fake_threads () +{ + _prog="$1" ; shift + + case "$_prog" in + nfsd) + export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR") + _threads="${PROCFS_PATH}/fs/nfsd/threads" + mkdir -p $(dirname "$_threads") + echo $# >"$_threads" + export FAKE_NFSD_THREAD_PIDS="$*" + ;; + *) + export FAKE_RPC_THREAD_PIDS="$*" + ;; + esac +} + +program_stack_trace () +{ + _prog="$1" + _pid="$2" + + cat <] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff +EOF +} + +program_stack_traces () +{ + _prog="$1" + _max="${2:-1}" + + _count=1 + for _pid in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do + [ $_count -le $_max ] || break + + program_stack_trace "$_prog" "$_pid" + _count=$(($_count + 1)) + done +} + +guess_output () +{ + case "$1" in + $CTDB_NFS_CALLOUT\ start\ nlockmgr) + echo "&Starting nfslock: OK" + ;; + $CTDB_NFS_CALLOUT\ start\ nfs) + cat <"$_rc_file" + echo "ERROR: ${_rpc_check_out}" >>"$_out" + else + _unhealthy=false + echo 0 >"$_rc_file" + fi + + if [ $restart_every -gt 0 ] && \ + [ $(($_numfails % $restart_every)) -eq 0 ] ; then + if ! $_unhealthy ; then + echo "WARNING: ${_rpc_check_out}" >>"$_out" fi - done - if [ "$_numfails" "$_op" "$_li" ] ; then - _out="" - _rc=0 - for _action in $_actions ; do - case "$_action" in - verbose) - _ver=1 - _pn="$_progname" - case "$_progname" in - knfsd) _ver=3 ; _pn="nfs" ;; - lockd) _ver=4 ; _pn="nlockmgr" ;; - statd) _pn="status" ;; - esac - _out="\ -ERROR: $_pn failed RPC check: -rpcinfo: RPC: Program not registered -program $_pn version $_ver is not available" - ;; - restart|restart:*) - _opts="" - _p="rpc.${_progname}" - case "$_progname" in - statd) - _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}" - ;; - esac - case "${_progname}${_action#restart}" in - knfsd) - _t="\ -Trying to restart NFS service -Starting nfslock: OK -Starting nfs: OK" - ;; - knfsd:bs) - _t="Trying to restart NFS service" - ;; - lockd|lockd:b) - _t="\ -Trying to restart lock manager service -Stopping nfslock: OK -Starting nfslock: OK" - ;; - lockd:*) - _t="Trying to restart lock manager service" - ;; - *) - _t="Trying to restart $_progname [${_p}${_opts}]" - esac - _out="${_out}${_out:+${_nl}}${_t}" - ;; - unhealthy) - _rc=1 - esac - done - required_result $_rc "$_out" - return + echo "Trying to restart service \"${_rpc_service}\"..." >>"$_out" + + if [ -n "$service_debug_cmd" ] ; then + $service_debug_cmd 2>&1 >>"$_out" + fi + + guess_output "$service_start_cmd" >>"$_out" fi - done + ) + + read _rc <"$_rc_file" + required_result $_rc <"$_out" + + rm -f "$_out" "$_rc_file" +} + +###################################################################### + +# Recovery lock fakery + +cleanup_reclock () +{ + _pattern="${script_dir}/${script}" + while pgrep -f "$_pattern" >/dev/null ; do + echo "Waiting for backgrounded ${script} to exit..." + (FAKE_SLEEP_REALLY=yes sleep 1) + done +} + +setup_reclock () +{ + CTDB_RECOVERY_LOCK=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR") + export CTDB_RECOVERY_LOCK + + test_cleanup cleanup_reclock } ###################################################################### @@ -630,12 +1180,14 @@ Starting nfslock: OK" setup_vsftpd () { + service_name="vsftpd" + if [ "$1" != "down" ] ; then die "setup_vsftpd up not implemented!!!" else debug "Setting up VSFTPD environment: service down, not managed by CTDB" - eventscript_call ctdb_service_unmanaged vsftpd + eventscript_call ctdb_service_unmanaged service vsftpd force-stopped export CTDB_MANAGED_SERVICES="foo" @@ -654,9 +1206,9 @@ setup_httpd () else debug "Setting up HTTPD environment: service down, not managed by CTDB" - for i in "apache2" "httpd" ; do - eventscript_call ctdb_service_unmanaged "$i" - service "$i" force-stopped + for service_name in "apache2" "httpd" ; do + eventscript_call ctdb_service_unmanaged + service "$service_name" force-stopped done export CTDB_MANAGED_SERVICES="foo" @@ -666,6 +1218,29 @@ setup_httpd () ###################################################################### +# multipathd fakery + +setup_multipathd () +{ + for i ; do + case "$i" in + \!*) + _t="${i#!}" + echo "Marking ${_t} as having no active paths" + FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}" + ;; + *) + _t="$i" + esac + CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}" + done + + export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES + export FAKE_SLEEP_FORCE=0.1 +} + +###################################################################### + # Result and test functions # Set some globals and print the summary. @@ -678,29 +1253,33 @@ define_test () # Remaining format should be NN.service.event.NNN or NN.service.NNN: _num="${_f##*.}" _f="${_f%.*}" + case "$_f" in - *.*.*) + [0-9][0-9].*.*) script="${_f%.*}" event="${_f##*.}" + script_dir="${CTDB_BASE}/events.d" ;; - *.*) + [0-9][0-9].*) script="$_f" unset event + script_dir="${CTDB_BASE}/events.d" + ;; + *.*) + script="${_f%.*}" + event="${_f##*.}" + script_dir="${CTDB_BASE}" ;; *) - die "Internal error - unknown testcase filename format" + script="${_f%.*}" + unset event + script_dir="${CTDB_BASE}" esac - printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" -} + [ -r "${script_dir}/${script}" ] || \ + die "Internal error - unable to find script \"${script_dir}/${script}\"" -_extra_header () -{ - cat <&1) + test_header () + { + echo "Running script \"$script $event${args:+ }$args\"" + } + + extra_header () + { + cat <&1) - - result_check + unit_test "$@" } -# Run an eventscript iteratively. +# Run an NFS eventscript iteratively. +# # - 1st argument is the number of iterations. -# - 2nd argument is something to eval to do setup for every iteration. -# The easiest thing to do here is to define a function and pass it -# here. +# +# - 2nd argument is the NFS/RPC service being tested +# +# rpcinfo (or $service_check_cmd) is used on each iteration to test +# the availability of the service +# +# If this is not set or null then no RPC service is checked and the +# required output is not reset on each iteration. This is useful in +# baseline tests to confirm that the eventscript and test +# infrastructure is working correctly. +# # - Subsequent arguments come in pairs: an iteration number and -# something to eval for that iteration. Each time an iteration +# something to eval before that iteration. Each time an iteration # number is matched the associated argument is given to eval after # the default setup is done. The iteration numbers need to be given # in ascending order. # -# Some optional args can be given *before* these, surrounded by extra -# "--" args. These args are passed to the eventscript. Quoting is -# lost. +# These arguments can allow a service to be started or stopped +# before a particular iteration. # -# One use of the 2nd and further arguments is to call -# required_result() to change what is expected of a particular -# iteration. -iterate_test () +nfs_iterate_test () { - [ -n "$event" ] || die 'simple_test: $event not set' - - args="" - if [ "$1" = "--" ] ; then - shift - while [ "$1" != "--" ] ; do - args="${args}${args:+ }$1" - shift - done + _repeats="$1" + _rpc_service="$2" + if [ -n "$2" ] ; then + shift 2 + else shift fi - _repeats="$1" - _setup_default="$2" - shift 2 - echo "Running $_repeats iterations of \"$script $event\" $args" - _result=true - - for iteration in $(seq 1 $_repeats) ; do - # This is inefficient because the iteration-specific setup - # might completely replace the default one. However, running - # the default is good because it allows you to revert to a - # particular result without needing to specify it explicitly. - eval $_setup_default - if [ $iteration = "$1" ] ; then - eval $2 + _iterate_failcount=0 + for _iteration in $(seq 1 $_repeats) ; do + # This is not a numerical comparison because $1 will often not + # be set. + if [ "$_iteration" = "$1" ] ; then + debug "##################################################" + eval "$2" + debug "##################################################" shift 2 fi + if [ -n "$_rpc_service" ] ; then + _ok=false + if [ -n "$service_check_cmd" ] ; then + if eval "$service_check_cmd" ; then + _ok=true + fi + else + if rpcinfo -T tcp localhost "$_rpc_service" >/dev/null 2>&1 ; then + _ok=true + fi + fi - _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) - _rc=$? - - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | eval sed -r $OUT_FILTER) - else - _fout="$_out" - fi - - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - _passed=true - else - _passed=false - _result=false + if $_ok ; then + _iterate_failcount=0 + else + _iterate_failcount=$(($_iterate_failcount + 1)) + fi + rpc_set_service_failure_response "$_rpc_service" $_iterate_failcount + fi + _out=$(simple_test 2>&1) + _ret=$? + if "$TEST_VERBOSE" || [ $_ret -ne 0 ] ; then + echo "##################################################" + echo "Iteration ${_iteration}:" + echo "$_out" + fi + if [ $_ret -ne 0 ] ; then + exit $_ret fi - - result_print "$_passed" "$_out" "$_rc" "Iteration $iteration" done - - result_footer "$_result" "$(_extra_header)" }