tests: Run transaction tests with externally imposed timeout

This works around cases where ctdb_transaction gets stuck - this still
needs to be debugged.  However, this change will at least cause
individual tests to fail rather than having whole test runs time out.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(Imported from commit f4871b8736f22941b227c19656319033c0c812e8)

diff --git a/tests/simple/53_ctdb_transaction.sh b/tests/simple/53_ctdb_transaction.sh
index be4a98e4296a5516dfcf33ee29c34f492c232a14..aa9202fc90a0babe159f248eee9435da05d45b72 100755 (executable)
--- a/tests/simple/53_ctdb_transaction.sh
+++ b/tests/simple/53_ctdb_transaction.sh
@@ -33,11 +33,15 @@ cluster_is_healthy
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-if test "x${CTDB_TEST_TIMELIMIT}" == "x" ; then
-       CTDB_TEST_TIMELIMIT=30
+if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
+    CTDB_TEST_TIMELIMIT=30
 fi
 
-t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}"
+# Add a timeout command to ensure this test completes if
+# ctdb_transaction gets stuck
+timeout_cmd="timeout $((CTDB_TEST_TIMELIMIT * 2))"
+
+t="$CTDB_TEST_WRAPPER $VALGRIND $timeout_cmd ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}"
 
 echo "Running ctdb_transaction on all $num_nodes nodes."
 try_command_on_node -v -p all "$t & $t"

diff --git a/tests/simple/54_ctdb_transaction_recovery.sh b/tests/simple/54_ctdb_transaction_recovery.sh
index 5bb56ecd3304b9c41bd2cdb8a9f0c86b51e96e86..ec31aa0e20e411c781a14efbff9c88a150592e84 100755 (executable)
--- a/tests/simple/54_ctdb_transaction_recovery.sh
+++ b/tests/simple/54_ctdb_transaction_recovery.sh
@@ -52,11 +52,17 @@ cluster_is_healthy
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-if test "x${CTDB_TEST_TIMELIMIT}" == "x" ; then
-       CTDB_TEST_TIMELIMIT=30
+if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
+    CTDB_TEST_TIMELIMIT=30
 fi
 
-t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}"
+# Add a timeout command to ensure this test completes if
+# ctdb_transaction gets stuck.  This one can get more "stuck" than the
+# previous test because a recovery can stop it committing a
+# transaction.
+timeout_cmd="timeout 600"
+
+t="$CTDB_TEST_WRAPPER $VALGRIND $timeout_cmd ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}"
 
 echo "Starting recovery loop"
 recovery_loop_start