Eventscripts: lower the fail/restart limits for nfsd.

[sahlberg/ctdb.git] / config / events.d / 60.nfs

diff --git a/config/events.d/60.nfs b/config/events.d/60.nfs
index 038adbb09409b34010f80219b1a211ec5b92e37e..79a071bab913f6ac71c74fdd42062b7c5c3bd0d2 100755 (executable)
--- a/config/events.d/60.nfs
+++ b/config/events.d/60.nfs
@@ -14,11 +14,16 @@ start_nfs() {
 service_name="nfs"
 service_start="start_nfs"
 service_stop="startstop_nfs stop"
+service_reconfigure="startstop_nfs restart"
 
 loadconfig
 
+[ "$NFS_SERVER_MODE" != "GANESHA" ] || exit 0
+
 ctdb_start_stop_service
 
+is_ctdb_managed_service || exit 0
+
 case "$1" in 
      init)
        # read statd from persistent database
@@ -51,24 +56,29 @@ case "$1" in
 
        # check that statd responds to rpc requests
        # if statd is not running we try to restart it
-       if ctdb_check_rpc "STATD" 100024 1 >/dev/null ; then
-               (service_name="nfs_statd"; ctdb_counter_init)
-       else
-               p="rpc.statd" ; cmd="$p"
-               cmd="${cmd}${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}"
-               cmd="${cmd}${STATD_PORT:+ -p }${STATD_PORT}"
-               cmd="${cmd}${STATD_OUTGOING_PORT:+ -o }${STATD_OUTGOING_PORT}"
-               (
-                       service_name="nfs_statd"
-                       ctdb_counter_incr
-                       ctdb_check_counter_limit 10 quiet >/dev/null
-               ) || {
-                       echo "$ctdb_check_rpc_out"
-                       echo "Trying to restart STATD [$cmd]"
-               }
-               $cmd
-       fi
-
+       # we only do this IF we have a rpc.statd command.
+       # For platforms where rpc.statd does not exist, we skip
+       # the check completely
+       p="rpc.statd"
+       which $p >/dev/null 2>/dev/null && {
+               if ctdb_check_rpc "STATD" 100024 1 >/dev/null ; then
+                       (service_name="nfs_statd"; ctdb_counter_init)
+               else
+                       cmd="$p"
+                       cmd="${cmd}${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}"
+                       cmd="${cmd}${STATD_PORT:+ -p }${STATD_PORT}"
+                       cmd="${cmd}${STATD_OUTGOING_PORT:+ -o }${STATD_OUTGOING_PORT}"
+                       (
+                               service_name="nfs_statd"
+                               ctdb_counter_incr
+                               ctdb_check_counter_limit 10 quiet >/dev/null
+                       ) || {
+                               echo "$ctdb_check_rpc_out"
+                               echo "Trying to restart STATD [$cmd]"
+                               $cmd
+                       }
+               fi
+       }
 
        # check that NFS responds to rpc requests
        [ "$CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK" = "yes" ] || {
@@ -78,15 +88,20 @@ case "$1" in
                (
                        service_name="nfs_knfsd"
                        ctdb_counter_incr
-                       ctdb_check_counter_limit 10 quiet >/dev/null
+
+                       ctdb_check_counter_equal 2 || {
+                               echo "Trying to restart NFS service"
+                               startstop_nfs restart >/dev/null 2>&1 &
+                               exit 0
+                       }
+
+                       ctdb_check_counter_limit 5 quiet >/dev/null
                ) || {
                        echo "$ctdb_check_rpc_out"
                        echo "Trying to restart NFS service"
                        startstop_nfs restart
                        exit 1
                }
-               # we haven't hit the failure limit so restart quietly
-               startstop_nfs restart >/dev/null 2>&1 &
            fi
        }
 
@@ -98,16 +113,51 @@ case "$1" in
        } || exit $?
 
        # check that lockd responds to rpc requests
-       ctdb_check_rpc "LOCKD" 100021 1 || {
-               echo "Trying to restart lock manager service"
-               startstop_nfs restart
-               startstop_nfslock restart
-               exit 1
-       }
+       if ctdb_check_rpc "LOCKD" 100021 1 >/dev/null ; then
+               (service_name="lockd"; ctdb_counter_init)
+       else
+               (
+                       service_name="lockd"
+                       ctdb_counter_incr
+
+                       ctdb_check_counter_equal 10 || {
+                               echo "Trying to restart NFS lock service"
+                               startstop_nfs restart >/dev/null 2>&1 &
+                               startstop_nfslock restart  >/dev/null 2>&1 &
+                               exit 0
+                       }
+
+                       ctdb_check_counter_limit 15 quiet >/dev/null
+       ) || {
+                       echo "$ctdb_check_rpc_out"
+                       echo "Trying to restart NFS lock service"
+                       startstop_nfs restart
+                       startstop_nfslock restart
+                       exit 1
+               }
+       fi
 
        # mount needs special handling since it is sometimes not started
        # correctly on RHEL5
-       ctdb_check_rpc "MOUNTD" 100005 1 || {
+       if ctdb_check_rpc "MOUNTD" 100005 1 >/dev/null ; then
+               (service_name="nfs_mountd"; ctdb_counter_init)
+       else
+       (
+               service_name="nfs_mountd"
+               ctdb_counter_incr
+
+               ctdb_check_counter_equal 5 || {
+                       p="rpc.mountd"
+                       cmd="${p}${MOUNTD_PORT:+ -p }${MOUNTD_PORT}"
+                       echo "Trying to restart MOUNTD [${cmd}]"
+                       killall -q -9 $p
+                       $cmd &
+                       exit 0
+               }
+
+               ctdb_check_counter_limit 10 quiet >/dev/null
+       ) || {
+               echo "$ctdb_check_rpc_out"
                p="rpc.mountd"
                cmd="${p}${MOUNTD_PORT:+ -p }${MOUNTD_PORT}"
                echo "Trying to restart MOUNTD [${cmd}]"
@@ -115,6 +165,9 @@ case "$1" in
                $cmd &
                exit 1
        }
+       fi
+
+
        # rquotad needs special handling since it is sometimes not started
        # correctly on RHEL5
        # this is not a critical service so we dont flag the node as unhealthy
@@ -128,7 +181,7 @@ case "$1" in
 
        # once every 60 seconds, update the statd state database for which
        # clients need notifications
-       LAST_UPDATE=`stat --printf="%Y" $CTDB_VARDIR/state/statd/update-trigger`
+       LAST_UPDATE=`stat --printf="%Y" $CTDB_VARDIR/state/statd/update-trigger 2>/dev/null`
        CURRENT_TIME=`date +"%s"`
        [ $CURRENT_TIME -ge $(($LAST_UPDATE + 60)) ] && {
            mkdir -p $CTDB_VARDIR/state/statd