ctdb-scripts: NFS RPC checks should be simple and consistent

Change status, nlockmgr, mountd, rquotad to be unhealthy after 6
rpcinfo check failures and do a verbose restart after every 2
failures.  Change 60.ganesha for consistency, since 60.ganesha tests
are broken and depend on the consistency.

Apart from the consistency aspect, the check infrastructure will soon
be simplified so that it only allows the equivalent of "unhealthy" and
"verbose restart:b" actions.

Update tests to have a corresponding numbers of iterations.  Run 1
extra iteration in most tests to check there are no unexpected
behaviour changes after the designated number of iterations completes.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/ctdb/config/events.d/60.ganesha b/ctdb/config/events.d/60.ganesha
index 662d84c2151c0077f35a466d60bad4860cd9c18c..43c70df1c6d0214f3c95256f2e8ce8fc5f416447 100755 (executable)
--- a/ctdb/config/events.d/60.ganesha
+++ b/ctdb/config/events.d/60.ganesha
@@ -230,19 +230,17 @@ case "$1" in
        p="rpc.statd"
        type $p >/dev/null 2>/dev/null && \
            nfs_check_rpc_service "statd" \
-               %  10 "verbose restart:b unhealthy" \
-               -ge 6 "verbose unhealthy" \
-               -eq 4 "verbose restart" \
-               -eq 2 "restart:b"
+               -ge 6 "verbose restart:b unhealthy" \
+               %   2 "verbose restart:b"
 
        if [ "$CTDB_SKIP_GANESHA_NFSD_CHECK" != "yes" ] ; then
            monitor_ganesha_nfsd
        fi
 
        # rquotad is sometimes not started correctly on RHEL5
-       # not a critical service so we dont flag the node as unhealthy
        nfs_check_rpc_service "rquotad" \
-           -gt 0 "verbose restart:b"
+           -ge 6 "verbose restart:b unhealthy" \
+           %   2 "verbose restart:b"
        ;;
 
      *)

diff --git a/ctdb/config/nfs-rpc-checks.d/10.statd.check b/ctdb/config/nfs-rpc-checks.d/10.statd.check
index 526e238bcccd1a14feb7847a502bd1fbb3730239..fd7a864aa3a6735606959cd5c47b1fda80ca4f98 100644 (file)
--- a/ctdb/config/nfs-rpc-checks.d/10.statd.check
+++ b/ctdb/config/nfs-rpc-checks.d/10.statd.check
@@ -1,4 +1,2 @@
-%  10 verbose restart:b unhealthy
--ge 6 verbose unhealthy
--eq 4 verbose restart
--eq 2 restart:b
+-ge 6 verbose restart:b unhealthy
+%   2 verbose restart:b

diff --git a/ctdb/config/nfs-rpc-checks.d/30.lockd.check b/ctdb/config/nfs-rpc-checks.d/30.lockd.check
index 95ae7b34db2772f71bb9e0e4517c52c35c68884c..fd7a864aa3a6735606959cd5c47b1fda80ca4f98 100644 (file)
--- a/ctdb/config/nfs-rpc-checks.d/30.lockd.check
+++ b/ctdb/config/nfs-rpc-checks.d/30.lockd.check
@@ -1,2 +1,2 @@
--ge 15 verbose restart:b unhealthy
--eq 10 restart:b
+-ge 6 verbose restart:b unhealthy
+%   2 verbose restart:b

diff --git a/ctdb/config/nfs-rpc-checks.d/40.mountd.check b/ctdb/config/nfs-rpc-checks.d/40.mountd.check
index 6b4f80191305e6be16cc15307c0fe9da10c9bc57..fd7a864aa3a6735606959cd5c47b1fda80ca4f98 100644 (file)
--- a/ctdb/config/nfs-rpc-checks.d/40.mountd.check
+++ b/ctdb/config/nfs-rpc-checks.d/40.mountd.check
@@ -1,2 +1,2 @@
--ge 10 verbose restart:b unhealthy
--eq 5 restart:b
+-ge 6 verbose restart:b unhealthy
+%   2 verbose restart:b

diff --git a/ctdb/config/nfs-rpc-checks.d/50.rquotad.check b/ctdb/config/nfs-rpc-checks.d/50.rquotad.check
index 1ebb8289140bcab755ec48daa882408546c258af..fd7a864aa3a6735606959cd5c47b1fda80ca4f98 100644 (file)
--- a/ctdb/config/nfs-rpc-checks.d/50.rquotad.check
+++ b/ctdb/config/nfs-rpc-checks.d/50.rquotad.check
@@ -1 +1,2 @@
--gt 0 verbose restart:b
+-ge 6 verbose restart:b unhealthy
+%   2 verbose restart:b

diff --git a/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh b/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh
index 429e60fbd4aa02ab03f6aeb2bf16dc532ff1fc0b..4fda6f076ee436b83ab9c0404742c3aa1fc4d169 100755 (executable)
--- a/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh
+++ b/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh
@@ -2,10 +2,9 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "rquotad down"
+define_test "rquotad down, 2 iterations"
 
 setup_nfs_ganesha
 rpc_services_down "rquotad"
 
-rpc_set_service_failure_response "rquotad"
-simple_test
+nfs_iterate_test 2 "rquotad"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.121.sh b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh
index cb8f0b28a66699cbd1eff62eb3e557bc7b0d8852..de69498721a0d5908374a75c82591b70b1ba8a15 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.121.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh
@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "lockd down, 15 iterations"
+define_test "lockd down, 7 iterations"
 
 # This simulates an ongoing failure in the eventscript's automated
 # attempts to restart the service.  That is, the eventscript is unable
@@ -11,4 +11,4 @@ define_test "lockd down, 15 iterations"
 setup_nfs
 rpc_services_down "nlockmgr"
 
-nfs_iterate_test 15 "nlockmgr"
+nfs_iterate_test 7 "nlockmgr"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.122.sh b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh
index cd67b443c37f6d5b03747853c25b8e43d53fc243..a0e0656c412af988959b68a94c9fb1e99f52cbae 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.122.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh
@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "lockd down, 15 iterations, back up after 10"
+define_test "lockd down, 7 iterations, back up after 2"
 
 # This simulates a success the eventscript's automated attempts to
 # restart the service.
@@ -10,8 +10,8 @@ define_test "lockd down, 15 iterations, back up after 10"
 setup_nfs
 rpc_services_down "nlockmgr"
 
-# Iteration 10 should try to restart rpc.lockd.  However, our test
+# Iteration 2 should try to restart rpc.lockd.  However, our test
 # stub rpc.lockd does nothing, so we have to explicitly flag it as up.
 
-nfs_iterate_test 15 "nlockmgr" \
-    11 "rpc_services_up nlockmgr"
+nfs_iterate_test 7 "nlockmgr" \
+    3 "rpc_services_up nlockmgr"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.131.sh b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh
index 09fd1e5907c959c3fcb71024bdbed17f842f57a8..72a1917f6f1b81c8e2d5fa350c1764eb3a4727da 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.131.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh
@@ -2,9 +2,9 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "rquotad down, 5 iterations"
+define_test "rquotad down, 7 iterations"
 
 setup_nfs
 rpc_services_down "rquotad"
 
-nfs_iterate_test 5 "rquotad"
+nfs_iterate_test 7 "rquotad"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.132.sh b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh
index 8f06ed2c13efb774c54ccf6824b329e6ca79d6b1..e34bb494b1a8c3a588a661d3b478405782a03d6f 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.132.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh
@@ -2,13 +2,13 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "rquotad down, 5 iterations, back up after 1"
+define_test "rquotad down, 7 iterations, back up after 2"
 
-# rquotad fails once but then comes back of its own accord after 1
+# rquotad fails once but then comes back after restart after 2nd
 # failure.
 
 setup_nfs
 rpc_services_down "rquotad"
 
-nfs_iterate_test 5 "rquotad" \
-    2 'rpc_services_up "rquotad"'
+nfs_iterate_test 7 "rquotad" \
+    3 'rpc_services_up "rquotad"'

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.141.sh b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh
index 56690a9140ad2d2df296220b752b35ea31ccf51f..96159338a0e4e15ea3b5913afa7fdaa2d406abea 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.141.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh
@@ -2,11 +2,11 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "statd down, 6 iterations"
+define_test "statd down, 7 iterations"
 
 # statd fails and attempts to restart it fail.
 
 setup_nfs
 rpc_services_down "status"
 
-nfs_iterate_test 6 "status"
+nfs_iterate_test 7 "status"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.142.sh b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh
index 3ed1c895e0ed234378918aaa1c28229df4245f8d..ec88bdcff0b6883ae09973668f78ab0559bc8253 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.142.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh
@@ -2,12 +2,12 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "statd down, 8 iterations, back up after 2"
+define_test "statd down, 7 iterations, back up after 2"
 
 # statd fails and the first attempt to restart it succeeds.
 
 setup_nfs
 rpc_services_down "status"
 
-nfs_iterate_test 8 "status" \
+nfs_iterate_test 7 "status" \
     3 'rpc_services_up "status"'

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.144.sh b/ctdb/tests/eventscripts/60.nfs.monitor.144.sh
index 5f409173ca2e1bd30d6d496e3b6e73d44a070ccb..96159338a0e4e15ea3b5913afa7fdaa2d406abea 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.144.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.144.sh
@@ -2,11 +2,11 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "statd down, 10 iterations"
+define_test "statd down, 7 iterations"
 
 # statd fails and attempts to restart it fail.
 
 setup_nfs
 rpc_services_down "status"
 
-nfs_iterate_test 10 "status"
+nfs_iterate_test 7 "status"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.152.sh b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh
index 9ba2f1b6b11f00b4d491a6605c435658337cdd6f..514369849d4cd631ce1b53d859435b6388b30a14 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.152.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh
@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "mountd down, 10 iterations"
+define_test "mountd down, 7 iterations"
 
 # This simulates an ongoing failure in the eventscript's automated
 # attempts to restart the service.  That is, the eventscript is unable
@@ -11,4 +11,4 @@ define_test "mountd down, 10 iterations"
 setup_nfs
 rpc_services_down "mountd"
 
-nfs_iterate_test 10 "mountd"
+nfs_iterate_test 7 "mountd"

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.153.sh b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh
index 6da10bfde19ef5aa57ae0ddb2441e6a0cf5d6033..5dce3c47c5d4feaafb91969dd6652135d7d41fe4 100755 (executable)
--- a/ctdb/tests/eventscripts/60.nfs.monitor.153.sh
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh
@@ -2,13 +2,13 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "mountd down, 10 iterations, back up after 5"
+define_test "mountd down, 7 iterations, back up after 2"
 
 setup_nfs
 rpc_services_down "mountd"
 
-# Iteration 5 should try to restart rpc.mountd.  However, our test
+# Iteration 2 should try to restart rpc.mountd.  However, our test
 # stub rpc.mountd does nothing, so we have to explicitly flag it as
 # up.
-nfs_iterate_test 10 "mountd" \
-    6 "rpc_services_up mountd"
+nfs_iterate_test 7 "mountd" \
+    3 "rpc_services_up mountd"