tests: Improve tickle tests

It is hard to diagnose failures in the NFS tickle test because there's
no way of telling if the test node doesn't have the tickle or if it
didn't get propagated.

Factor out check_tickles() into local.bash and give it some
parameters.

Have the NFS test call it first to ensure the tickle has been
registered.  Then use new function check_tickles_all() to ensure the
tickle has been propagated to all nodes.  Give this a bit of extra
time (double the timeout) just in case we're racing with the update.

Add a useful comment to the CIFS test so that I stop asking myself how
the test could ever have worked reliably.  :-)

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
Pair-programmed-with: Amitay Isaacs <amitay@gmail.com>

(Imported from commit bafb9151ccb5722df36f9ba168716f4f4fa01cdc)

diff --git a/tests/complex/31_nfs_tickle.sh b/tests/complex/31_nfs_tickle.sh
index b5ed47e7e8c8b6bcb2fe5d0c658c2afc180e52eb..5aeb870d3c65a1015300a292899bf05bb8a31124 100755 (executable)
--- a/tests/complex/31_nfs_tickle.sh
+++ b/tests/complex/31_nfs_tickle.sh
@@ -34,8 +34,8 @@ Steps:
 
 Expected results:
 
-* CTDB should correctly record the socket in the nfs-tickles directory
-  and should send a reset packet when the node is disabled.
+* CTDB should correctly record the socket and should send a reset
+  packet when the node is disabled.
 EOF
 }
 
@@ -58,6 +58,8 @@ monitor_interval="${out#*= }"
 #echo "Monitor interval on node $test_node is $monitor_interval seconds."
 
 select_test_node_and_ips
+try_command_on_node $test_node "$CTDB listnodes | wc -l"
+numnodes="$out"
 
 test_port=2049
 
@@ -73,18 +75,16 @@ echo "Source socket is $src_socket"
 
 wait_for_monitor_event $test_node
 
-echo "Sleeping until tickles are synchronised across nodes..."
-try_command_on_node $test_node $CTDB getvar TickleUpdateInterval
-sleep_for "${out#*= }"
+echo "Wait until NFS connection is tracked by CTDB on test node ..."
+wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
 
-try_command_on_node -v any "ctdb -Y gettickles $test_ip $test_port"
+echo "Getting TicklesUpdateInterval..."
+try_command_on_node $test_node $CTDB getvar TickleUpdateInterval
+update_interval="$out"
 
-if [ "${out/${src_socket}/}" != "$out" ] ; then
-    echo "GOOD: NFS connection tracked OK."
-else
-    echo "BAD: Socket not tracked in NFS tickles."
-    testfailures=1
-fi
+echo "Wait until NFS connection is tracked by CTDB on all nodes..."
+wait_until $(($update_interval * 2)) \
+    check_tickles_all $numnodes  $test_ip $test_port $src_socket
 
 tcptickle_sniff_start $src_socket "${test_ip}:${test_port}"
 

diff --git a/tests/complex/32_cifs_tickle.sh b/tests/complex/32_cifs_tickle.sh
index 93634e72bd6dc908d63667709b945d2ffcef1b1e..70ce93ff4b91e8351cd2bf8b5f1e56dc4a050f0d 100755 (executable)
--- a/tests/complex/32_cifs_tickle.sh
+++ b/tests/complex/32_cifs_tickle.sh
@@ -72,15 +72,8 @@ echo "Source socket is $src_socket"
 
 # This should happen as soon as connection is up... but unless we wait
 # we sometimes beat the registration.
-check_tickles ()
-{
-    try_command_on_node 0 ctdb gettickles $test_ip -n $test_node
-    # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    [ "${out/SRC: ${src_socket} /}" != "$out" ]
-}
-
 echo "Checking if CIFS connection is tracked by CTDB..."
-wait_until 10 check_tickles
+wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
 echo "$out"
 
 if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
@@ -92,6 +85,9 @@ fi
 
 tcptickle_sniff_start $src_socket "${test_ip}:${test_port}"
 
+# The test node is only being disabled so the tickling is done from
+# the test node.  We don't need to wait until the tickles are
+# transferred to another node.
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node
 wait_until_node_has_status $test_node disabled

diff --git a/tests/complex/scripts/local.bash b/tests/complex/scripts/local.bash
index eb4c41c8b614cc3bf7ad3578a91d527f8c20e5de..6fbc1aea719d9d5b2b86faac452946c5d1f40963 100644 (file)
--- a/tests/complex/scripts/local.bash
+++ b/tests/complex/scripts/local.bash
@@ -27,6 +27,35 @@ wait_until_get_src_socket ()
     wait_until 5 get_src_socket "$@"
 }
 
+#######################################
+
+check_tickles ()
+{
+    local node="$1"
+    local test_ip="$2"
+    local test_port="$3"
+    local src_socket="$4"
+    try_command_on_node $node ctdb gettickles $test_ip $test_port
+    # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
+    [ "${out/SRC: ${src_socket} /}" != "$out" ]
+}
+
+check_tickles_all ()
+{
+    local numnodes="$1"
+    local test_ip="$2"
+    local test_port="$3"
+    local src_socket="$4"
+
+    try_command_on_node all ctdb gettickles $test_ip $test_port
+    # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
+    local t="${src_socket//./\\.}"
+    local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
+    [ $count -eq $numnodes ]
+}
+
+
+
 #######################################
 
 # filename will be in $tcpdump_filename, pid in $tcpdump_pid