1 # utility functions for ctdb event scripts
3 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
5 #######################################
6 # pull in a system config file, if any
10 foo="${service_config:-${service_name}}"
11 if [ -n "$foo" ] ; then
14 elif [ "$1" != "ctdb" ] ; then
19 if [ -f /etc/sysconfig/$1 ]; then
21 elif [ -f /etc/default/$1 ]; then
23 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
24 . $CTDB_BASE/sysconfig/$1
32 ##############################################################
33 # determine on what type of system (init style) we are running
35 # only do detection if not already set:
36 test "x$CTDB_INIT_STYLE" != "x" && return
38 if [ -x /sbin/startproc ]; then
39 CTDB_INIT_STYLE="suse"
40 elif [ -x /sbin/start-stop-daemon ]; then
41 CTDB_INIT_STYLE="debian"
43 CTDB_INIT_STYLE="redhat"
47 ######################################################
48 # simulate /sbin/service on platforms that don't have it
53 # do nothing, when no service was specified
54 [ -z "$_service_name" ] && return
56 if [ -x /sbin/service ]; then
57 /sbin/service "$_service_name" "$_op"
58 elif [ -x /etc/init.d/$_service_name ]; then
59 /etc/init.d/$_service_name "$_op"
60 elif [ -x /etc/rc.d/init.d/$_service_name ]; then
61 /etc/rc.d/init.d/$_service_name "$_op"
65 ######################################################
66 # simulate /sbin/service (niced) on platforms that don't have it
71 # do nothing, when no service was specified
72 [ -z "$_service_name" ] && return
74 if [ -x /sbin/service ]; then
75 nice /sbin/service "$_service_name" "$_op"
76 elif [ -x /etc/init.d/$_service_name ]; then
77 nice /etc/init.d/$_service_name "$_op"
78 elif [ -x /etc/rc.d/init.d/$_service_name ]; then
79 nice /etc/rc.d/init.d/$_service_name "$_op"
83 ######################################################
84 # wait for a command to return a zero exit status
85 # usage: ctdb_wait_command SERVICE_NAME <command>
86 ######################################################
90 [ -z "$wait_cmd" ] && return;
92 echo "Waiting for service $service_name to start"
93 while [ $all_ok -eq 0 ]; do
94 $wait_cmd > /dev/null 2>&1 && all_ok=1
95 ctdb status > /dev/null 2>&1 || {
96 echo "ctdb daemon has died. Exiting wait for $service_name"
99 [ $all_ok -eq 1 ] || sleep 1
101 echo "Local service $service_name is up"
105 ######################################################
106 # wait for a set of tcp ports
107 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
108 ######################################################
109 ctdb_wait_tcp_ports() {
113 [ -z "$wait_ports" ] && return;
115 echo "Waiting for tcp service $service_name to start"
116 while [ $all_ok -eq 0 ]; do
118 for p in $wait_ports; do
119 if [ -x /usr/bin/netcat ]; then
120 /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
121 elif [ -x /usr/bin/nc ]; then
122 /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
123 elif [ -x /usr/bin/netstat ]; then
124 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
125 elif [ -x /bin/netstat ]; then
126 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
128 echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
132 [ $all_ok -eq 1 ] || sleep 1
133 ctdb status > /dev/null 2>&1 || {
134 echo "ctdb daemon has died. Exiting tcp wait $service_name"
138 echo "Local tcp services for $service_name are up"
142 ######################################################
143 # check that a rpc server is registered with portmap
144 # and responding to requests
145 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
146 ######################################################
152 ctdb_check_rpc_out=$(rpcinfo -u localhost $prognum $version 2>&1)
153 if [ $? -ne 0 ] ; then
154 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
156 echo "$ctdb_check_rpc_out"
161 ######################################################
162 # check a set of directories is available
163 # return 1 on a missing directory
164 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
165 ######################################################
166 ctdb_check_directories_probe() {
167 while IFS="" read d ; do
173 [ -d "${d}/." ] || return 1
178 ######################################################
179 # check a set of directories is available
180 # usage: ctdb_check_directories SERVICE_NAME <directories...>
181 ######################################################
182 ctdb_check_directories() {
183 n="${1:-${service_name}}"
184 ctdb_check_directories_probe || {
185 echo "ERROR: $n directory \"$d\" not available"
190 ######################################################
191 # check a set of tcp ports
192 # usage: ctdb_check_tcp_ports <ports...>
193 ######################################################
194 ctdb_check_tcp_ports() {
197 if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
198 if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
199 echo "ERROR: $service_name tcp port $p is not responding"
206 ######################################################
207 # check a unix socket
208 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
209 ######################################################
210 ctdb_check_unix_socket() {
212 [ -z "$socket_path" ] && return
214 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
215 echo "ERROR: $service_name socket $socket_path not found"
220 ######################################################
221 # check a command returns zero status
222 # usage: ctdb_check_command SERVICE_NAME <command>
223 ######################################################
224 ctdb_check_command() {
227 [ -z "$wait_cmd" ] && return;
228 $wait_cmd > /dev/null 2>&1 || {
229 echo "ERROR: $service_name - $wait_cmd returned error"
234 ################################################
235 # kill off any TCP connections with the given IP
236 ################################################
237 kill_tcp_connections() {
242 connfile="$CTDB_VARDIR/state/connections.$_IP"
243 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
244 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
246 while read dest src; do
247 srcip=`echo $src | sed -e "s/:[^:]*$//"`
248 srcport=`echo $src | sed -e "s/^.*://"`
249 destip=`echo $dest | sed -e "s/:[^:]*$//"`
250 destport=`echo $dest | sed -e "s/^.*://"`
251 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
252 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
254 # we only do one-way killtcp for CIFS
256 # for all others we do 2-way
258 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
261 _killcount=`expr $_killcount + 1`
265 [ $_failed = 0 ] || {
266 echo "Failed to send killtcp control"
269 [ $_killcount -gt 0 ] || {
273 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
275 _count=`expr $_count + 1`
276 [ $_count -gt 3 ] && {
277 echo "Timed out killing tcp connections for IP $_IP"
281 echo "killed $_killcount TCP connections to released IP $_IP"
284 ##################################################################
285 # kill off the local end for any TCP connections with the given IP
286 ##################################################################
287 kill_tcp_connections_local_only() {
292 connfile="$CTDB_VARDIR/state/connections.$_IP"
293 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
294 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
296 while read dest src; do
297 srcip=`echo $src | sed -e "s/:[^:]*$//"`
298 srcport=`echo $src | sed -e "s/^.*://"`
299 destip=`echo $dest | sed -e "s/:[^:]*$//"`
300 destport=`echo $dest | sed -e "s/^.*://"`
301 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
302 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
303 _killcount=`expr $_killcount + 1`
307 [ $_failed = 0 ] || {
308 echo "Failed to send killtcp control"
311 [ $_killcount -gt 0 ] || {
315 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
317 _count=`expr $_count + 1`
318 [ $_count -gt 3 ] && {
319 echo "Timed out killing tcp connections for IP $_IP"
323 echo "killed $_killcount TCP connections to released IP $_IP"
326 ##################################################################
327 # tickle any TCP connections with the given IP
328 ##################################################################
329 tickle_tcp_connections() {
334 connfile="$CTDB_VARDIR/state/connections.$_IP"
335 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
336 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
338 while read dest src; do
339 srcip=`echo $src | sed -e "s/:[^:]*$//"`
340 srcport=`echo $src | sed -e "s/^.*://"`
341 destip=`echo $dest | sed -e "s/:[^:]*$//"`
342 destport=`echo $dest | sed -e "s/^.*://"`
343 echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
344 ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
345 echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
346 ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
350 [ $_failed = 0 ] || {
351 echo "Failed to send tickle control"
356 ########################################################
357 # start/stop the nfs service on different platforms
358 ########################################################
361 [ -x /etc/init.d/nfsserver ] && {
364 [ -x /etc/init.d/nfslock ] && {
372 service nfsserver start
375 service nfsserver stop > /dev/null 2>&1
378 echo 0 >/proc/fs/nfsd/threads
379 service nfsserver stop > /dev/null 2>&1
381 service nfsserver start
388 service nfslock start
392 service nfs stop > /dev/null 2>&1
393 service nfslock stop > /dev/null 2>&1
396 echo 0 >/proc/fs/nfsd/threads
397 service nfs stop > /dev/null 2>&1
398 service nfslock stop > /dev/null 2>&1
400 service nfslock start
406 echo "Unknown platform. NFS is not supported with ctdb"
412 ########################################################
413 # start/stop the nfs lockmanager service on different platforms
414 ########################################################
415 startstop_nfslock() {
417 [ -x /etc/init.d/nfsserver ] && {
420 [ -x /etc/init.d/nfslock ] && {
426 # for sles there is no service for lockmanager
427 # so we instead just shutdown/restart nfs
430 service nfsserver start
433 service nfsserver stop > /dev/null 2>&1
436 service nfsserver stop
437 service nfsserver start
444 service nfslock start
447 service nfslock stop > /dev/null 2>&1
451 service nfslock start
456 echo "Unknown platform. NFS locking is not supported with ctdb"
462 # better use delete_ip_from_iface() together with add_ip_to_iface
463 # remove_ip should be removed in future
465 local _ip_maskbits=$1
467 local _ip=`echo "$_ip_maskbits" | cut -d '/' -f1`
468 local _maskbits=`echo "$_ip_maskbits" | cut -d '/' -f2`
470 delete_ip_from_iface "$_iface" "$_ip" "$_maskbits"
479 local _state_dir="$CTDB_VARDIR/state/interface_modify"
480 local _lockfile="$_state_dir/$_iface.flock"
481 local _readd_base="$_state_dir/$_iface.readd.d"
483 mkdir -p $_state_dir || {
485 echo "Failed to mkdir -p $_state_dir - $ret"
489 test -f $_lockfile || {
493 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh add "$_iface" "$_ip" "$_maskbits" "$_readd_base"
497 delete_ip_from_iface()
502 local _state_dir="$CTDB_VARDIR/state/interface_modify"
503 local _lockfile="$_state_dir/$_iface.flock"
504 local _readd_base="$_state_dir/$_iface.readd.d"
506 mkdir -p $_state_dir || {
508 echo "Failed to mkdir -p $_state_dir - $ret"
512 test -f $_lockfile || {
516 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh delete "$_iface" "$_ip" "$_maskbits" "$_readd_base"
520 setup_iface_ip_readd_script()
525 local _readd_script=$4
526 local _state_dir="$CTDB_VARDIR/state/interface_modify"
527 local _lockfile="$_state_dir/$_iface.flock"
528 local _readd_base="$_state_dir/$_iface.readd.d"
530 mkdir -p $_state_dir || {
532 echo "Failed to mkdir -p $_state_dir - $ret"
536 test -f $_lockfile || {
540 flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh readd_script "$_iface" "$_ip" "$_maskbits" "$_readd_base" "$_readd_script"
544 ########################################################
545 # some simple logic for counting events - per eventscript
546 # usage: ctdb_counter_init
548 # ctdb_check_counter_limit <limit>
549 # ctdb_check_counter_limit succeeds when count >= <limit>
550 ########################################################
551 _ctdb_counter_common () {
552 _counter_file="$ctdb_fail_dir/$service_name"
553 mkdir -p "${_counter_file%/*}" # dirname
555 ctdb_counter_init () {
560 ctdb_counter_incr () {
564 echo -n 1 >> "$_counter_file"
566 ctdb_check_counter_limit () {
569 _limit="${1:-${service_fail_limit}}"
573 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
574 if [ $_size -ge $_limit ] ; then
575 echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
577 elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
578 echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
581 ctdb_check_counter_equal () {
587 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
588 if [ $_size -eq $_limit ] ; then
594 ########################################################
596 ctdb_spool_dir="/var/spool/ctdb"
597 ctdb_status_dir="$ctdb_spool_dir/status"
598 ctdb_fail_dir="$ctdb_spool_dir/failcount"
599 ctdb_active_dir="$ctdb_spool_dir/active"
603 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
608 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
609 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
611 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
612 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
621 d="$ctdb_status_dir/$script_name"
628 for i in "banned" "unhealthy" ; do
635 ctdb_service_needs_reconfigure ()
637 [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
640 ctdb_service_set_reconfigure ()
642 d="$ctdb_status_dir/$service_name"
647 ctdb_service_unset_reconfigure ()
649 rm -f "$ctdb_status_dir/$service_name/reconfigure"
652 ctdb_service_reconfigure ()
654 echo "Reconfiguring service \"$service_name\"..."
655 if [ -n "$service_reconfigure" ] ; then
656 eval $service_reconfigure
658 service "$service_name" restart
660 ctdb_service_unset_reconfigure
664 ctdb_compat_managed_service ()
666 if [ "$1" = "yes" ] ; then
671 is_ctdb_managed_service ()
673 _service_name="${1:-${service_name}}"
675 t=" $CTDB_MANAGED_SERVICES "
677 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
678 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
679 ctdb_compat_managed_service "$CTDB_MANAGES_SCP" "scp"
680 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
681 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
682 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
683 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
684 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
685 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
687 # Returns 0 if "<space>$_service_name<space>" appears in $t
688 [ "${t#* ${_service_name} }" != "${t}" ]
691 ctdb_start_stop_service ()
693 _service_name="${1:-${service_name}}"
695 [ "$event_name" = "monitor" ] || return 0
697 _active="$ctdb_active_dir/$_service_name"
698 if is_ctdb_managed_service "$_service_name"; then
699 if ! [ -e "$_active" ] ; then
700 echo "Starting service $_service_name"
701 ctdb_service_start || exit $?
702 mkdir -p "$ctdb_active_dir"
707 if [ -e "$_active" ] ; then
708 echo "Stopping service $_service_name"
709 ctdb_service_stop || exit $?
716 ctdb_service_start ()
718 if [ -n "$service_start" ] ; then
719 eval $service_start || return $?
721 service "$service_name" start || return $?
728 if [ -n "$service_stop" ] ; then
731 service "$service_name" stop
735 ctdb_standard_event_handler ()
750 ipv4_host_addr_to_net_addr()
755 local HOST0=$(echo $HOST | awk -F . '{print $4}')
756 local HOST1=$(echo $HOST | awk -F . '{print $3}')
757 local HOST2=$(echo $HOST | awk -F . '{print $2}')
758 local HOST3=$(echo $HOST | awk -F . '{print $1}')
760 local HOST_NUM=$(( $HOST0 + $HOST1 * 256 + $HOST2 * (256 ** 2) + $HOST3 * (256 ** 3) ))
762 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
764 local NET_NUM=$(( $HOST_NUM & $MASK_NUM))
766 local NET0=$(( $NET_NUM & 255 ))
767 local NET1=$(( ($NET_NUM & (255 * 256)) / 256 ))
768 local NET2=$(( ($NET_NUM & (255 * 256**2)) / 256**2 ))
769 local NET3=$(( ($NET_NUM & (255 * 256**3)) / 256**3 ))
771 echo "$NET3.$NET2.$NET1.$NET0"
774 ipv4_maskbits_to_net_mask()
778 local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
780 local MASK0=$(( $MASK_NUM & 255 ))
781 local MASK1=$(( ($MASK_NUM & (255 * 256)) / 256 ))
782 local MASK2=$(( ($MASK_NUM & (255 * 256**2)) / 256**2 ))
783 local MASK3=$(( ($MASK_NUM & (255 * 256**3)) / 256**3 ))
785 echo "$MASK3.$MASK2.$MASK1.$MASK0"
793 local N=`echo $ADDR | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//'`
794 test -n "$N" && fail=1
796 local ADDR0=$(echo $ADDR | awk -F . '{print $4}')
797 local ADDR1=$(echo $ADDR | awk -F . '{print $3}')
798 local ADDR2=$(echo $ADDR | awk -F . '{print $2}')
799 local ADDR3=$(echo $ADDR | awk -F . '{print $1}')
801 test "$ADDR0" -gt 255 && fail=1
802 test "$ADDR1" -gt 255 && fail=1
803 test "$ADDR2" -gt 255 && fail=1
804 test "$ADDR3" -gt 255 && fail=1
806 test x"$fail" != x"0" && {
807 #echo "IPv4: '$ADDR' is not a valid address"
814 # iptables doesn't like being re-entered, so flock-wrap it.
817 flock -w 30 /var/ctdb/iptables-ctdb.flock /sbin/iptables "$@"
820 ########################################################
822 ########################################################
824 # Temporary directory for tickles.
825 tickledir="$CTDB_VARDIR/state/tickles"
826 mkdir -p "$tickledir"
832 mkdir -p "$tickledir" # Just in case
835 _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
837 # What public IPs do I hold?
838 _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
840 # IPs as a regexp choice
841 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
843 # Record connections to our public IPs in a temporary file
844 _my_connections="${tickledir}/${_port}.connections"
845 rm -f "$_my_connections"
847 awk -v destpat="^${_ipschoice}:${_port}\$" \
848 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
849 sort >"$_my_connections"
851 # Record our current tickles in a temporary file
852 _my_tickles="${tickledir}/${_port}.tickles"
855 ctdb -Y gettickles $_i $_port |
856 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
860 # Add tickles for connections that we haven't already got tickles for
861 comm -23 "$_my_connections" "$_my_tickles" |
862 while read _src _dst ; do
863 ctdb addtickle $_src $_dst
866 # Remove tickles for connections that are no longer there
867 comm -13 "$_my_connections" "$_my_tickles" |
868 while read _src _dst ; do
869 ctdb deltickle $_src $_dst
872 rm -f "$_my_connections" "$_my_tickles"
875 ########################################################
876 # load a site local config file
877 ########################################################
879 [ -x $CTDB_BASE/rc.local ] && {
880 . $CTDB_BASE/rc.local
883 [ -d $CTDB_BASE/rc.local.d ] && {
884 for i in $CTDB_BASE/rc.local.d/* ; do
885 [ -x "$i" ] && . "$i"
889 script_name="${0##*/}" # basename
890 service_name="$script_name" # default is just the script name