ctdb-scripts: Dump stack traces of smbd processes after shutdown
[samba.git] / ctdb / tests / eventscripts / scripts / local.sh
index ce1c2510bd0b729ad5a9f46023e6e7459774dc7e..ea900ebcd0d8c99e32ef48af9915c9c46b843bb6 100644 (file)
@@ -13,29 +13,32 @@ if [ -d "${TEST_SUBDIR}/stubs" ] ; then
        /*) : ;;
        *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;;
     esac
+    export CTDB_HELPER_BINDIR="$EVENTSCRIPTS_PATH"
 fi
 
 export EVENTSCRIPTS_PATH
 
 PATH="${EVENTSCRIPTS_PATH}:${PATH}"
 
+export CTDB="ctdb"
+
 export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts"
 if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \
     "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then
     rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR"
 fi
 mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR"
-export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb"
+export CTDB_SCRIPT_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/script-state"
 
 export CTDB_LOGGING="file:${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb"
 touch "${CTDB_LOGGING#file:}" || \
     die "Unable to setup logging for \"$CTDB_LOGGING\""
 
-if [ -d "${TEST_SUBDIR}/etc" ] ; then    
+if [ -d "${TEST_SUBDIR}/etc" ] ; then
     cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR"
-    export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
+    export CTDB_SYS_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc"
 else
-    die "Unable to setup \$CTDB_ETCDIR"
+    die "Unable to setup \$CTDB_SYS_ETCDIR"
 fi
 
 if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then
@@ -112,7 +115,10 @@ setup_generic ()
 
 
     export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
-    mkdir -p "${CTDB_DBDIR}/persistent"
+    export CTDB_DBDIR_PERSISTENT="${CTDB_DBDIR}/persistent"
+    export CTDB_DBDIR_STATE="${CTDB_DBDIR}/state"
+    mkdir -p "$CTDB_DBDIR_PERSISTENT"
+    mkdir -p "$CTDB_DBDIR_STATE"
 
     export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
     export FAKE_TDB_IS_OK
@@ -288,8 +294,8 @@ ctdb_set_pnn ()
     export FAKE_CTDB_PNN="$1"
     echo "Setting up PNN ${FAKE_CTDB_PNN}"
 
-    export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb/${FAKE_CTDB_PNN}"
-    mkdir -p "$CTDB_VARDIR"
+    export CTDB_SCRIPT_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/script-state/${FAKE_CTDB_PNN}"
+    mkdir -p "$CTDB_SCRIPT_VARDIR"
 }
 
 setup_ctdb ()
@@ -369,9 +375,8 @@ SwapTotal:       ${_swap_total} kB
 SwapFree:        ${_swap_free} kB
 ..."
 
-    export CTDB_MONITOR_FREE_MEMORY
-    export CTDB_MONITOR_FREE_MEMORY_WARN
-    export CTDB_CHECK_SWAP_IS_NOT_USED
+    export CTDB_MONITOR_MEMORY_USAGE
+    export CTDB_MONITOR_SWAP_USAGE
 }
 
 setup_fscheck ()
@@ -538,6 +543,83 @@ EOF
 
 ######################################################################
 
+setup_ctdb_lvs ()
+{
+       lvs_state_dir="${EVENTSCRIPTS_TESTS_VAR_DIR}/lvs"
+       mkdir -p "$lvs_state_dir"
+
+       export FAKE_LVS_STATE_DIR="${lvs_state_dir}/state"
+       mkdir "$FAKE_LVS_STATE_DIR"
+
+       lvs_header=$(ipvsadm -l -n)
+
+       export CTDB_LVS_PUBLIC_IP="$1"
+       export CTDB_LVS_PUBLIC_IFACE="$2"
+
+       [ -n "$CTDB_LVS_PUBLIC_IP" ] || return 0
+       [ -n "$CTDB_LVS_PUBLIC_IFACE" ] || return 0
+
+       export CTDB_LVS_NODES=$(mktemp --tmpdir="$lvs_state_dir")
+       export FAKE_CTDB_LVS_MASTER=""
+
+       # Read from stdin
+       _pnn=0
+       while read _ip _opts ; do
+               case "$_opts" in
+               master)
+                       FAKE_CTDB_LVS_MASTER="$_pnn"
+                       echo "$_ip"
+                       ;;
+               slave-only)
+                       printf "%s\tslave-only\n" "$_ip"
+                       ;;
+               *)
+                       echo "$_ip"
+                       ;;
+               esac
+               _pnn=$(($_pnn + 1))
+       done >"$CTDB_LVS_NODES"
+}
+
+check_ipvsadm ()
+{
+       if [ "$1" = "NULL" ] ; then
+               required_result 0 <<EOF
+$lvs_header
+EOF
+       else
+               required_result 0 <<EOF
+$lvs_header
+$(cat)
+EOF
+       fi
+
+       simple_test_command ipvsadm -l -n
+}
+
+check_lvs_ip ()
+{
+       _scope="$1"
+
+       if [ "$_scope" = "NULL" ] ; then
+               required_result 0 <<EOF
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+EOF
+       else
+               required_result 0 <<EOF
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+    inet ${CTDB_LVS_PUBLIC_IP}/32 scope ${_scope} lo
+       valid_lft forever preferred_lft forever
+EOF
+       fi
+
+       simple_test_command ip addr show dev lo
+}
+
+######################################################################
+
 ctdb_catdb_format_pairs ()
 {
     _count=0
@@ -599,35 +681,43 @@ EOF
 
 setup_ctdb_natgw ()
 {
-    debug "Setting up NAT gateway"
-
-    natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
-    mkdir -p "$natgw_config_dir"
-
-    # These will accumulate, 1 per test... but will be cleaned up at
-    # the end.
-    export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
-
-    # Read from stdin
-    while read _ip _master _dev ; do
-       echo "$_ip"
-       if [ "$_master" = "master" ] ; then
-           export FAKE_CTDB_NATGW_MASTER="$_ip"
-       fi
-    done >"$CTDB_NATGW_NODES"
-
-    # Assume all of the nodes are on a /24 network and have IPv4
-    # addresses:
-    read _ip <"$CTDB_NATGW_NODES"
-    export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
-
-    # These are fixed.  Probably don't use the same network for the
-    # private node IPs.  To unset the default gateway just set it to
-    # "".  :-)
-    export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
-    export CTDB_NATGW_PUBLIC_IFACE="eth1"
-    export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
-    export CTDB_NATGW_SLAVE_ONLY=""
+       debug "Setting up NAT gateway"
+
+       natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
+       mkdir -p "$natgw_config_dir"
+
+       # These will accumulate, 1 per test... but will be cleaned up at
+       # the end.
+       export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
+
+       # Read from stdin
+       while read _ip _opts ; do
+               case "$_opts" in
+               master)
+                       export FAKE_CTDB_NATGW_MASTER="$_ip"
+                       echo "$_ip"
+                       ;;
+               slave-only)
+                       printf "%s\tslave-only\n" "$_ip"
+                       ;;
+               *)
+                       echo "$_ip"
+                       ;;
+               esac
+       done >"$CTDB_NATGW_NODES"
+
+       # Assume all of the nodes are on a /24 network and have IPv4
+       # addresses:
+       read _ip <"$CTDB_NATGW_NODES"
+       export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
+
+       # These are fixed.  Probably don't use the same network for the
+       # private node IPs.  To unset the default gateway just set it to
+       # "".  :-)
+       export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
+       export CTDB_NATGW_PUBLIC_IFACE="eth1"
+       export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
+       export CTDB_NATGW_SLAVE_ONLY=""
 }
 
 ok_natgw_master_ip_addr_show ()
@@ -749,6 +839,23 @@ setup_samba ()
     fi
 }
 
+samba_setup_fake_threads ()
+{
+       export FAKE_SMBD_THREAD_PIDS="$*"
+
+       _nl="
+"
+       _out=""
+       _count=0
+       for _pid ; do
+               [ "$_count" -lt 5 ] || break
+               _t=$(program_stack_trace "smbd" $_pid)
+               _out="${_out:+${_out}${_nl}}${_t}"
+               _count=$((_count + 1))
+       done
+       SAMBA_STACK_TRACES="$_out"
+}
+
 setup_winbind ()
 {
     setup_ctdb
@@ -803,6 +910,9 @@ setup_nfs ()
 
     export RPCNFSDCOUNT
 
+    # This doesn't even need to exist
+    export CTDB_NFS_EXPORTS_FILE="$EVENTSCRIPTS_TESTS_VAR_DIR/etc-exports"
+
     # Reset the failcounts for nfs services.
     eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
 
@@ -879,7 +989,7 @@ rpc_services_up ()
 
 nfs_load_config ()
 {
-    _etc="$CTDB_ETCDIR" # shortcut for readability
+    _etc="$CTDB_SYS_ETCDIR" # shortcut for readability
     for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
        if [ -r "$_c" ] ; then
            . "$_c"
@@ -906,6 +1016,17 @@ nfs_setup_fake_threads ()
     esac
 }
 
+program_stack_trace ()
+{
+       _prog="$1"
+       _pid="$2"
+
+       cat <<EOF
+Stack trace for ${_prog}[${_pid}]:
+[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
+EOF
+}
+
 program_stack_traces ()
 {
     _prog="$1"
@@ -915,10 +1036,7 @@ program_stack_traces ()
     for _pid in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do
        [ $_count -le $_max ] || break
 
-       cat <<EOF
-Stack trace for ${_prog}[${_pid}]:
-[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
-EOF
+       program_stack_trace "$_prog" "$_pid"
        _count=$(($_count + 1))
     done
 }
@@ -1013,7 +1131,8 @@ program $_rpc_service${_ver:+ version }${_ver} is not available"
            echo 0 >"$_rc_file"
        fi
 
-       if [ $restart_every -gt 0 -a $(($_numfails % $restart_every)) -eq 0 ] ; then
+       if [ $restart_every -gt 0 ] && \
+                  [ $(($_numfails % $restart_every)) -eq 0 ] ; then
            if ! $_unhealthy ; then
                echo "WARNING: ${_rpc_check_out}" >>"$_out"
            fi
@@ -1036,6 +1155,27 @@ program $_rpc_service${_ver:+ version }${_ver} is not available"
 
 ######################################################################
 
+# Recovery lock fakery
+
+cleanup_reclock ()
+{
+       _pattern="${script_dir}/${script}"
+       while pgrep -f "$_pattern" >/dev/null ; do
+               echo "Waiting for backgrounded ${script} to exit..."
+               (FAKE_SLEEP_REALLY=yes sleep 1)
+       done
+}
+
+setup_reclock ()
+{
+       CTDB_RECOVERY_LOCK=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
+       export CTDB_RECOVERY_LOCK
+
+       test_cleanup cleanup_reclock
+}
+
+######################################################################
+
 # VSFTPD fakery
 
 setup_vsftpd ()
@@ -1164,7 +1304,7 @@ simple_test ()
 
 ##################################################
 CTDB_BASE="$CTDB_BASE"
-CTDB_ETCDIR="$CTDB_ETCDIR"
+CTDB_SYS_ETCDIR="$CTDB_SYS_ETCDIR"
 ctdb client is "$(which ctdb)"
 ip command is "$(which ip)"
 EOF