1 # utility functions for ctdb event scripts
3 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
5 #######################################
6 # pull in a system config file, if any
9 if [ "$1" != "ctdb" ] ; then
14 foo="${service_config:-${service_name}}"
15 if [ -n "$foo" ] ; then
20 if [ -f /etc/sysconfig/$1 ]; then
22 elif [ -f /etc/default/$1 ]; then
24 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
25 . $CTDB_BASE/sysconfig/$1
29 ##############################################################
30 # determine on what type of system (init style) we are running
32 # only do detection if not already set:
33 test "x$CTDB_INIT_STYLE" != "x" && return
35 if [ -x /sbin/startproc ]; then
36 CTDB_INIT_STYLE="suse"
37 elif [ -x /sbin/start-stop-daemon ]; then
38 CTDB_INIT_STYLE="debian"
40 CTDB_INIT_STYLE="redhat"
44 ######################################################
45 # simulate /sbin/service on platforms that don't have it
50 # do nothing, when no service was specified
51 [ -z "$_service_name" ] && return
53 if [ -x /sbin/service ]; then
54 /sbin/service "$_service_name" "$_op"
55 elif [ -x /etc/init.d/$_service_name ]; then
56 /etc/init.d/$_service_name "$_op"
57 elif [ -x /etc/rc.d/init.d/$_service_name ]; then
58 /etc/rc.d/init.d/$_service_name "$_op"
62 ######################################################
63 # simulate /sbin/service (niced) on platforms that don't have it
65 # do nothing, when no service was specified
71 ######################################################
72 # wait for a command to return a zero exit status
73 # usage: ctdb_wait_command SERVICE_NAME <command>
74 ######################################################
78 [ -z "$wait_cmd" ] && return;
80 echo "Waiting for service $service_name to start"
81 while [ $all_ok -eq 0 ]; do
82 $wait_cmd > /dev/null 2>&1 && all_ok=1
83 ctdb status > /dev/null 2>&1 || {
84 echo "ctdb daemon has died. Exiting wait for $service_name"
87 [ $all_ok -eq 1 ] || sleep 1
89 echo "Local service $service_name is up"
93 ######################################################
94 # wait for a set of tcp ports
95 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
96 ######################################################
97 ctdb_wait_tcp_ports() {
101 [ -z "$wait_ports" ] && return;
103 echo "Waiting for tcp service $service_name to start"
104 while [ $all_ok -eq 0 ]; do
106 for p in $wait_ports; do
107 if [ -x /usr/bin/netcat ]; then
108 /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
109 elif [ -x /usr/bin/nc ]; then
110 /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
111 elif [ -x /usr/bin/netstat ]; then
112 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
113 elif [ -x /bin/netstat ]; then
114 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
116 echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
120 [ $all_ok -eq 1 ] || sleep 1
121 ctdb status > /dev/null 2>&1 || {
122 echo "ctdb daemon has died. Exiting tcp wait $service_name"
126 echo "Local tcp services for $service_name are up"
130 ######################################################
131 # check that a rpc server is registered with portmap
132 # and responding to requests
133 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
134 ######################################################
139 rpcinfo -u localhost $prognum $version > /dev/null || {
140 echo "ERROR: $progname not responding to rpc requests"
145 ######################################################
146 # check a set of directories is available
147 # return 1 on a missing directory
148 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
149 ######################################################
150 ctdb_check_directories_probe() {
151 while IFS="" read d ; do
157 [ -d "$d" ] || return 1
162 ######################################################
163 # check a set of directories is available
164 # usage: ctdb_check_directories SERVICE_NAME <directories...>
165 ######################################################
166 ctdb_check_directories() {
167 n="${1:-${service_name}}"
168 ctdb_check_directories_probe || {
169 echo "ERROR: $n directory \"$d\" not available"
174 ######################################################
175 # check a set of tcp ports
176 # usage: ctdb_check_tcp_ports <ports...>
177 ######################################################
178 ctdb_check_tcp_ports() {
181 if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
182 if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
183 echo "ERROR: $service_name tcp port $p is not responding"
190 ######################################################
191 # check a unix socket
192 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
193 ######################################################
194 ctdb_check_unix_socket() {
196 [ -z "$socket_path" ] && return
198 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
199 echo "ERROR: $service_name socket $socket_path not found"
204 ######################################################
205 # check a command returns zero status
206 # usage: ctdb_check_command SERVICE_NAME <command>
207 ######################################################
208 ctdb_check_command() {
211 [ -z "$wait_cmd" ] && return;
212 $wait_cmd > /dev/null 2>&1 || {
213 echo "ERROR: $service_name - $wait_cmd returned error"
218 ################################################
219 # kill off any TCP connections with the given IP
220 ################################################
221 kill_tcp_connections() {
226 connfile="$CTDB_BASE/state/connections.$_IP"
227 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
228 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
230 while read dest src; do
231 srcip=`echo $src | sed -e "s/:[^:]*$//"`
232 srcport=`echo $src | sed -e "s/^.*://"`
233 destip=`echo $dest | sed -e "s/:[^:]*$//"`
234 destport=`echo $dest | sed -e "s/^.*://"`
235 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
236 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
238 # we only do one-way killtcp for CIFS
240 # for all others we do 2-way
242 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
245 _killcount=`expr $_killcount + 1`
249 [ $_failed = 0 ] || {
250 echo "Failed to send killtcp control"
253 [ $_killcount -gt 0 ] || {
257 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
259 _count=`expr $_count + 1`
260 [ $_count -gt 3 ] && {
261 echo "Timed out killing tcp connections for IP $_IP"
265 echo "killed $_killcount TCP connections to released IP $_IP"
268 ##################################################################
269 # kill off the local end for any TCP connections with the given IP
270 ##################################################################
271 kill_tcp_connections_local_only() {
276 connfile="$CTDB_BASE/state/connections.$_IP"
277 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
278 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
280 while read dest src; do
281 srcip=`echo $src | sed -e "s/:[^:]*$//"`
282 srcport=`echo $src | sed -e "s/^.*://"`
283 destip=`echo $dest | sed -e "s/:[^:]*$//"`
284 destport=`echo $dest | sed -e "s/^.*://"`
285 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
286 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
287 _killcount=`expr $_killcount + 1`
291 [ $_failed = 0 ] || {
292 echo "Failed to send killtcp control"
295 [ $_killcount -gt 0 ] || {
299 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
301 _count=`expr $_count + 1`
302 [ $_count -gt 3 ] && {
303 echo "Timed out killing tcp connections for IP $_IP"
307 echo "killed $_killcount TCP connections to released IP $_IP"
310 ##################################################################
311 # tickle any TCP connections with the given IP
312 ##################################################################
313 tickle_tcp_connections() {
318 connfile="$CTDB_BASE/state/connections.$_IP"
319 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
320 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
322 while read dest src; do
323 srcip=`echo $src | sed -e "s/:[^:]*$//"`
324 srcport=`echo $src | sed -e "s/^.*://"`
325 destip=`echo $dest | sed -e "s/:[^:]*$//"`
326 destport=`echo $dest | sed -e "s/^.*://"`
327 echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
328 ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
329 echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
330 ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
334 [ $_failed = 0 ] || {
335 echo "Failed to send tickle control"
340 ########################################################
341 # start/stop the nfs service on different platforms
342 ########################################################
345 [ -x /etc/init.d/nfsserver ] && {
348 [ -x /etc/init.d/nfslock ] && {
356 service nfsserver start
359 service nfsserver stop > /dev/null 2>&1
366 service nfslock start
370 service nfs stop > /dev/null 2>&1
371 service nfslock stop > /dev/null 2>&1
376 echo "Unknown platform. NFS is not supported with ctdb"
382 ########################################################
383 # start/stop the nfs lockmanager service on different platforms
384 ########################################################
385 startstop_nfslock() {
387 [ -x /etc/init.d/nfsserver ] && {
390 [ -x /etc/init.d/nfslock ] && {
396 # for sles there is no service for lockmanager
397 # so we instead just shutdown/restart nfs
400 service nfsserver start
403 service nfsserver stop > /dev/null 2>&1
410 service nfslock start
413 service nfslock stop > /dev/null 2>&1
418 echo "Unknown platform. NFS locking is not supported with ctdb"
424 # better use delete_ip_from_iface() together with add_ip_to_iface
426 # the ip tool will delete all secondary IPs if this is the primary.
427 # To work around this _very_ annoying behaviour we have to keep a
428 # record of the secondaries and re-add them afterwards. yuck
430 if ip addr list dev $2 primary | grep -q "inet $1 " ; then
431 secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
433 ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
434 [ -z "$secondaries" ] || {
435 for i in $secondaries; do
436 if ip addr list dev $2 | grep -q "inet $i" ; then
437 echo "kept secondary $i on dev $2"
439 echo "re-adding secondary address $i to dev $2"
440 ip addr add $i dev $2 || failed=1
451 local _lockfile="$CTDB_BASE/state/interface_modify.$_iface.flock"
453 test -f $_lockfile || {
457 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh add "$_iface" "$_ip" "$_maskbits"
461 delete_ip_from_iface()
466 local _lockfile="$CTDB_BASE/state/interface_modify.$_iface.flock"
468 test -f $_lockfile || {
472 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh delete "$_iface" "$_ip" "$_maskbits"
476 ########################################################
477 # some simple logic for counting events - per eventscript
478 # usage: ctdb_counter_init
480 # ctdb_check_counter_limit <limit>
481 # ctdb_check_counter_limit succeeds when count >= <limit>
482 ########################################################
483 _ctdb_counter_common () {
484 _counter_file="$ctdb_fail_dir/$service_name"
485 mkdir -p "${_counter_file%/*}" # dirname
487 ctdb_counter_init () {
492 ctdb_counter_incr () {
496 echo -n 1 >> "$_counter_file"
498 ctdb_check_counter_limit () {
501 _limit="${1:-${service_fail_limit}}"
505 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
506 if [ $_size -ge $_limit ] ; then
507 echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
509 elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
510 echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
513 ########################################################
515 ctdb_spool_dir="/var/spool/ctdb"
516 ctdb_status_dir="$ctdb_spool_dir/status"
517 ctdb_fail_dir="$ctdb_spool_dir/failcount"
518 ctdb_active_dir="$ctdb_spool_dir/active"
522 echo "node is \"$1\", problem with \"${script_name}\": $(cat $2)"
527 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
528 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
530 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
531 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
540 d="$ctdb_status_dir/$script_name"
547 for i in "banned" "unhealthy" ; do
554 ctdb_service_needs_reconfigure ()
556 [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
559 ctdb_service_set_reconfigure ()
561 d="$ctdb_status_dir/$service_name"
566 ctdb_service_unset_reconfigure ()
568 rm -f "$ctdb_status_dir/$service_name/reconfigure"
571 ctdb_service_reconfigure ()
573 if [ -n "$service_reconfigure" ] ; then
574 eval $service_reconfigure
576 service "$service_name" restart
578 ctdb_service_unset_reconfigure
582 ctdb_compat_managed_service ()
584 if [ "$1" = "yes" ] ; then
589 is_ctdb_managed_service ()
591 t=" $CTDB_MANAGED_SERVICES "
593 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
594 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
595 ctdb_compat_managed_service "$CTDB_MANAGES_SCP" "scp"
596 ctdb_compat_managed_service "$CTDB_MANAGES_WINDBIND" "windbind"
597 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
598 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
599 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
600 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
602 # Returns 0 if "<space>$service_name<space>" appears in $t
603 [ "${t#* ${service_name} }" != "${t}" ]
606 ctdb_start_stop_service ()
608 _active="$ctdb_active_dir/$service_name"
610 if is_ctdb_managed_service ; then
611 if ! [ -e "$_active" ] ; then
612 echo "Starting service $service_name"
613 ctdb_service_start || exit $?
614 mkdir -p "$ctdb_active_dir"
618 elif ! is_ctdb_managed_service ; then
619 if [ -e "$_active" ] ; then
620 echo "Stopping service $service_name"
621 ctdb_service_stop || exit $?
628 ctdb_service_start ()
630 if [ -n "$service_start" ] ; then
633 service "$service_name" start
640 if [ -n "$service_stop" ] ; then
643 service "$service_name" stop
647 ctdb_standard_event_handler ()
661 ipv4_host_addr_to_net_addr()
666 local HOST0=$(echo $HOST | awk -F . '{print $4}')
667 local HOST1=$(echo $HOST | awk -F . '{print $3}')
668 local HOST2=$(echo $HOST | awk -F . '{print $2}')
669 local HOST3=$(echo $HOST | awk -F . '{print $1}')
671 local HOST_NUM=$(( $HOST0 + $HOST1 * 256 + $HOST2 * (256 ** 2) + $HOST3 * (256 ** 3) ))
673 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
675 local NET_NUM=$(( $HOST_NUM & $MASK_NUM))
677 local NET0=$(( $NET_NUM & 255 ))
678 local NET1=$(( ($NET_NUM & (255 * 256)) / 256 ))
679 local NET2=$(( ($NET_NUM & (255 * 256**2)) / 256**2 ))
680 local NET3=$(( ($NET_NUM & (255 * 256**3)) / 256**3 ))
682 echo "$NET3.$NET2.$NET1.$NET0"
685 ipv4_maskbits_to_net_mask()
689 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
691 local MASK0=$(( $MASK_NUM & 255 ))
692 local MASK1=$(( ($MASK_NUM & (255 * 256)) / 256 ))
693 local MASK2=$(( ($MASK_NUM & (255 * 256**2)) / 256**2 ))
694 local MASK3=$(( ($MASK_NUM & (255 * 256**3)) / 256**3 ))
696 echo "$MASK3.$MASK2.$MASK1.$MASK0"
704 local N=`echo $ADDR | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//'`
705 test -n "$N" && fail=1
707 local ADDR0=$(echo $ADDR | awk -F . '{print $4}')
708 local ADDR1=$(echo $ADDR | awk -F . '{print $3}')
709 local ADDR2=$(echo $ADDR | awk -F . '{print $2}')
710 local ADDR3=$(echo $ADDR | awk -F . '{print $1}')
712 test "$ADDR0" -gt 255 && fail=1
713 test "$ADDR1" -gt 255 && fail=1
714 test "$ADDR2" -gt 255 && fail=1
715 test "$ADDR3" -gt 255 && fail=1
717 test x"$fail" != x"0" && {
718 #echo "IPv4: '$ADDR' is not a valid address"
725 ########################################################
726 # load a site local config file
727 ########################################################
729 [ -x $CTDB_BASE/rc.local ] && {
730 . $CTDB_BASE/rc.local
733 [ -d $CTDB_BASE/rc.local.d ] && {
734 for i in $CTDB_BASE/rc.local.d/* ; do
735 [ -x "$i" ] && . "$i"
739 script_name="${0##*/}" # basename
740 service_name="$script_name" # default is just the script name