initscript: wait until we can ping ctdbd before setting tunables.

Currently we do a "sleep 1" after starting and before running
set_ctdb_variables to set the tunables.  This is too arbitrary and
might fail if the system is heavily loaded.  This, for example, could
result in some nodes running with DeterministicIPs and some without,
in which case a different IP allocation algorithm would run depending
on who is the recmaster!

This makes the start function wait until "ctdb ping" succeeds (with 10
second timeout) before trying to run set_ctdb_variables.  If a timeout
occurs then the start function attempts to kill ctdbd before exiting
with a failure.

It also cleans up the status reporting code for Red Hat and SUSE so
that the final status code is reported.  Currently there are cases
where a correct status is prematurely reported before a failure
occurs.

Signed-off-by: Martin Schwenke <martin@meltin.net>

diff --git a/config/ctdb.init b/config/ctdb.init
index 7dfdd26639c70cb4fe29a30cf5a28a05cb630d70..fc66ab2fbd32636d7a8f87f5e05f02bce05cf480 100755 (executable)
--- a/config/ctdb.init
+++ b/config/ctdb.init
@@ -164,6 +164,19 @@ set_retval() {
     return $1
 }
 
+wait_until_ready () {
+    _timeout="${1:-10}" # default is 10 seconds
+
+    _count=0
+    while ! ctdb ping >/dev/null 2>&1 ; do
+       if [ $_count -ge $_timeout ] ; then
+           return 1
+       fi
+       sleep 1
+       _count=$(($_count + 1))
+    done
+}
+
 ctdbd=${CTDBD:-/usr/sbin/ctdbd}
 
 start() {
@@ -193,14 +206,11 @@ start() {
            ;;
        suse)
            eval startproc $ctdbd "$CTDB_OPTIONS"
-           rc_status -v
            RETVAL=$?
            ;;
        redhat)
            eval $ctdbd "$CTDB_OPTIONS"
            RETVAL=$?
-           [ $RETVAL -eq 0 ] && success || failure
-           echo
            [ $RETVAL -eq 0 ] && touch /var/lock/subsys/ctdb || RETVAL=1
            ;;
        debian)
@@ -210,9 +220,25 @@ start() {
            ;;
     esac
 
-    sleep 1
+    if [ $RETVAL -eq 0 ] ; then
+       if wait_until_ready ; then
+           set_ctdb_variables
+       else
+           RETVAL=1
+           pkill -9 -f $ctdbd >/dev/null 2>&1
+       fi
+    fi
 
-    set_ctdb_variables
+    case $init_style in
+       suse)
+           set_retval $RETVAL
+           rc_status -v
+           ;;
+       redhat)
+           [ $RETVAL -eq 0 ] && success || failure
+           echo
+           ;;
+    esac
 
     return $RETVAL
 }