tests: Try harder to avoid failures due to repeated recoveries

About a year ago a check was added to _cluster_is_healthy() to make
sure that node 0 isn't in recovery.  This was to avoid unexpected
recoveries causing tests to fail.  However, it was misguided because
each test initially calls cluster_is_healthy() and will now fail if an
unexpected recovery occurs.

Instead, have cluster_is_healthy() warn if the cluster is in recovery.

Also:

* Rename wait_until_healthy() to wait_until_ready() because it waits
  until both healthy and out of recovery.

* Change the post-recovery sleep in restart_ctdb() to 2 seconds and
  add a loop to wait (for 2 seconds at a time) if the cluster is back
  in recovery.  The logic here is that the re-recovery timeout has
  been set to 1 second, so sleeping for just 1 second might race
  against the next recovery.

* Use reverse logic in node_has_status() so that it works for "all".

* Tweak wait_until() so that it can handle timeouts with a
  recheck-interval specified.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(Imported from commit 6a552f1a12ebe43f946bbbee2a3846b5a640ae4f)

diff --git a/tests/complex/34_nfs_tickle_restart.sh b/tests/complex/34_nfs_tickle_restart.sh
index 93587e2f316750c8ed252f60c3d3df3d682122e6..b7eea4ca21ceca8193f3905fbc2b2750f7f26e0e 100755 (executable)
--- a/tests/complex/34_nfs_tickle_restart.sh
+++ b/tests/complex/34_nfs_tickle_restart.sh
@@ -79,7 +79,7 @@ try_command_on_node $rn $CTDB_TEST_WRAPPER restart_ctdb_1
 echo "Setting NoIPTakeover on node ${rn}"
 try_command_on_node $rn $CTDB setvar NoIPTakeover 1
 
-wait_until_healthy
+wait_until_ready
 
 echo "Getting TickleUpdateInterval..."
 try_command_on_node $test_node $CTDB getvar TickleUpdateInterval

diff --git a/tests/scripts/integration.bash b/tests/scripts/integration.bash
index 4a1f0910d4749414ab93d21084a97555b3d8cd81..60f72b6f86ee59135782865abb7eea09891c769a 100644 (file)
--- a/tests/scripts/integration.bash
+++ b/tests/scripts/integration.bash
@@ -258,11 +258,19 @@ select_test_node_and_ips ()
 #######################################
 
 # Wait until either timeout expires or command succeeds.  The command
-# will be tried once per second.
+# will be tried once per second, unless timeout has format T/I, where
+# I is the recheck interval.
 wait_until ()
 {
     local timeout="$1" ; shift # "$@" is the command...
 
+    local interval=1
+    case "$timeout" in
+       */*)
+           interval="${timeout#*/}"
+           timeout="${timeout%/*}"
+    esac
+
     local negate=false
     if [ "$1" = "!" ] ; then
        negate=true
@@ -280,9 +288,12 @@ wait_until ()
            echo "OK"
            return 0
        fi
-       echo -n .
-       t=$(($t - 1))
-       sleep 1
+       local i
+       for i in $(seq 1 $interval) ; do
+           echo -n .
+       done
+       t=$(($t - $interval))
+       sleep $interval
     done
 
     echo "*TIMEOUT*"
@@ -302,14 +313,26 @@ sleep_for ()
 
 _cluster_is_healthy ()
 {
-    $CTDB nodestatus all >/dev/null && \
-       node_has_status 0 recovered
+    $CTDB nodestatus all >/dev/null
+}
+
+_cluster_is_recovered ()
+{
+    node_has_status all recovered
+}
+
+_cluster_is_ready ()
+{
+    _cluster_is_healthy && _cluster_is_recovered
 }
 
 cluster_is_healthy ()
 {
     if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then
        echo "Cluster is HEALTHY"
+       if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_recovered ; then
+         echo "WARNING: cluster in recovery mode!"
+       fi
        return 0
     else
        echo "Cluster is UNHEALTHY"
@@ -325,13 +348,13 @@ cluster_is_healthy ()
     fi
 }
 
-wait_until_healthy ()
+wait_until_ready ()
 {
     local timeout="${1:-120}"
 
-    echo "Waiting for cluster to become healthy..."
+    echo "Waiting for cluster to become ready..."
 
-    wait_until $timeout onnode -q any $CTDB_TEST_WRAPPER _cluster_is_healthy
+    wait_until $timeout onnode -q any $CTDB_TEST_WRAPPER _cluster_is_ready
 }
 
 # This function is becoming nicely overloaded.  Soon it will collapse!  :-)
@@ -356,7 +379,7 @@ node_has_status ()
        (unfrozen)     fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
        (monon)        mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
        (monoff)       mpat='^Monitoring mode:DISABLED \(1\)$' ;;
-       (recovered)    rpat='^Recovery mode:NORMAL \(0\)$' ;;
+       (recovered)    rpat='^Recovery mode:RECOVERY \(1\)$' ;;
        *)
            echo "node_has_status: unknown status \"$status\""
            return 1
@@ -382,7 +405,7 @@ node_has_status ()
     elif [ -n "$mpat" ] ; then
        $CTDB getmonmode -n "$pnn" | egrep -q "$mpat"
     elif [ -n "$rpat" ] ; then
-        $CTDB status -n "$pnn" | egrep -q "$rpat"
+        ! $CTDB status -n "$pnn" | egrep -q "$rpat"
     else
        echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
        return 1
@@ -532,8 +555,8 @@ restart_ctdb ()
            continue
        }
 
-       wait_until_healthy || {
-           echo "Cluster didn't become healthy.  Restarting..."
+       wait_until_ready || {
+           echo "Cluster didn't become ready.  Restarting..."
            continue
        }
 
@@ -545,7 +568,13 @@ restart_ctdb ()
        # help the cluster to stabilise before a subsequent test.
        echo "Forcing a recovery..."
        onnode -q 0 $CTDB recover
-       sleep_for 1
+       sleep_for 2
+
+       if ! onnode -q any $CTDB_TEST_WRAPPER _cluster_is_recovered ; then
+           echo "Cluster has gone into recovery again, waiting..."
+           wait_until 30/2 onnode -q any $CTDB_TEST_WRAPPER _cluster_is_recovered
+       fi
+
 
        # Cluster is still healthy.  Good, we're done!
        if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then