1 # utility functions for ctdb event scripts
3 #######################################
4 # pull in a system config file, if any
7 if [ -f /etc/sysconfig/$name ]; then
9 elif [ -f /etc/default/$name ]; then
11 elif [ -f $CTDB_BASE/sysconfig/$name ]; then
12 . $CTDB_BASE/sysconfig/$name
16 ##############################################################
17 # determine on what type of system (init style) we are running
19 # only do detection if not already set:
20 test "x$CTDB_INIT_STYLE" != "x" && return
22 if [ -x /sbin/startproc ]; then
23 CTDB_INIT_STYLE="suse"
24 elif [ -x /sbin/start-stop-daemon ]; then
25 CTDB_INIT_STYLE="debian"
27 CTDB_INIT_STYLE="redhat"
31 ######################################################
32 # simulate /sbin/service on platforms that don't have it
37 # do nothing, when no service was specified
38 test "x$service_name" = "x" && return
40 if [ -x /sbin/service ]; then
41 /sbin/service "$service_name" "$op"
42 elif [ -x /etc/init.d/$service_name ]; then
43 /etc/init.d/$service_name "$op"
44 elif [ -x /etc/rc.d/init.d/$service_name ]; then
45 /etc/rc.d/init.d/$service_name "$op"
49 ######################################################
50 # simulate /sbin/service (niced) on platforms that don't have it
55 # do nothing, when no service was specified
56 test "x$service_name" = "x" && return
58 if [ -x /sbin/service ]; then
59 nice /sbin/service "$service_name" "$op"
60 elif [ -x /etc/init.d/$service_name ]; then
61 nice /etc/init.d/$service_name "$op"
62 elif [ -x /etc/rc.d/init.d/$service_name ]; then
63 nice /etc/rc.d/init.d/$service_name "$op"
67 ######################################################
68 # wait for a command to return a zero exit status
69 # usage: ctdb_wait_command SERVICE_NAME <command>
70 ######################################################
74 [ -z "$wait_cmd" ] && return;
76 echo "Waiting for service $service_name to start"
77 while [ $all_ok -eq 0 ]; do
78 $wait_cmd > /dev/null 2>&1 && all_ok=1
79 ctdb status > /dev/null 2>&1 || {
80 echo "ctdb daemon has died. Exiting wait for $service_name"
83 [ $all_ok -eq 1 ] || sleep 1
85 echo "Local service $service_name is up"
89 ######################################################
90 # wait for a set of tcp ports
91 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
92 ######################################################
93 ctdb_wait_tcp_ports() {
97 [ -z "$wait_ports" ] && return;
99 echo "Waiting for tcp service $service_name to start"
100 while [ $all_ok -eq 0 ]; do
102 for p in $wait_ports; do
103 if [ -x /usr/bin/netcat ]; then
104 /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
105 elif [ -x /usr/bin/nc ]; then
106 /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
107 elif [ -x /usr/bin/netstat ]; then
108 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
109 elif [ -x /bin/netstat ]; then
110 (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
112 echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
116 [ $all_ok -eq 1 ] || sleep 1
117 ctdb status > /dev/null 2>&1 || {
118 echo "ctdb daemon has died. Exiting tcp wait $service_name"
122 echo "Local tcp services for $service_name are up"
127 ######################################################
128 # wait for a set of directories
129 # usage: ctdb_wait_directories SERVICE_NAME <directories...>
130 ######################################################
131 ctdb_wait_directories() {
135 [ -z "$wait_dirs" ] && return;
137 echo "Waiting for local directories for $service_name"
138 while [ $all_ok -eq 0 ]; do
140 for d in $wait_dirs; do
141 [ -d $d ] || all_ok=0
143 [ $all_ok -eq 1 ] || sleep 1
144 ctdb status > /dev/null 2>&1 || {
145 echo "ctdb daemon has died. Exiting directory wait for $service_name"
149 echo "Local directories for $service_name are available"
153 ######################################################
154 # check that a rpc server is registered with portmap
155 # and responding to requests
156 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
157 ######################################################
162 rpcinfo -u localhost $prognum $version > /dev/null || {
163 echo "ERROR: $service_name not responding to rpc requests"
168 ######################################################
169 # check a set of directories is available
170 # return 1 on a missing directory
171 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
172 ######################################################
173 ctdb_check_directories_probe() {
182 [ -d "$d" ] || return 1
188 ######################################################
189 # check a set of directories is available
190 # usage: ctdb_check_directories SERVICE_NAME <directories...>
191 ######################################################
192 ctdb_check_directories() {
193 # Note: ctdb_check_directories_probe sets both $service_name and $d.
194 ctdb_check_directories_probe "$@" || {
195 echo "ERROR: $service_name directory $d not available"
200 ######################################################
201 # check a set of tcp ports
202 # usage: ctdb_check_tcp_ports SERVICE_NAME <ports...>
203 ######################################################
204 ctdb_check_tcp_ports() {
208 [ -z "$wait_ports" ] && return;
210 # check availability of netcat or netstat first
213 if [ -x /usr/bin/netstat ]; then
214 NETSTAT=/usr/bin/netstat
215 elif [ -x /bin/netstat ]; then
217 elif [ -x /usr/bin/netcat ]; then
218 NETCAT=/usr/bin/netcat
219 elif [ -x /bin/netcat ]; then
221 elif [ -x /usr/bin/nc ]; then
223 elif [ -x /bin/nc ]; then
227 for p in $wait_ports; do
230 if [ "x${NETCAT}" != "x" ]; then
231 ${NETCAT} -z 127.0.0.1 $p > /dev/null || all_ok=0
232 elif [ "x${NETSTAT}" != "x" ]; then
233 if ! ${NETSTAT} -a -n | egrep "0.0.0.0:$p .*LISTEN" > /dev/null ; then
234 if ! ${NETSTAT} -a -n | egrep ":::$p .*LISTEN" > /dev/null ; then
239 echo "ERROR: neither netcat (or nc) nor netstat found!"
240 echo "ERROR: can't monitor ${service_name} tcp port ${p}"
244 [ $all_ok -eq 1 ] || {
245 echo "ERROR: $service_name tcp port $p is not responding"
251 ######################################################
252 # check a unix socket
253 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
254 ######################################################
255 ctdb_check_unix_socket() {
258 [ -z "$socket_path" ] && return;
260 # check availability of netstat first
262 if [ -x $(type -p netstat) ]; then
263 NETSTAT=$(type -p netstat)
264 elif [ -x /usr/bin/netstat ]; then
265 NETSTAT=/usr/bin/netstat
266 elif [ -x /bin/netstat ]; then
271 if [ "x$NETSTAT" != "x" ]; then
272 if $NETSTAT -l -a -n | grep -qE "^unix.*LISTEN.*${socket_path}$"; then
278 [ -S ${socket_path} ] && all_ok=1 || all_ok=0
281 [ $all_ok -eq 1 ] || {
282 echo "ERROR: $service_name socket $socket_path not found"
287 ######################################################
288 # check a command returns zero status
289 # usage: ctdb_check_command SERVICE_NAME <command>
290 ######################################################
291 ctdb_check_command() {
294 [ -z "$wait_cmd" ] && return;
295 $wait_cmd > /dev/null 2>&1 || {
296 echo "ERROR: $service_name - $wait_cmd returned error"
301 ################################################
302 # kill off any TCP connections with the given IP
303 ################################################
304 kill_tcp_connections() {
309 connfile="$CTDB_BASE/state/connections.$_IP"
310 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
311 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
313 while read dest src; do
314 srcip=`echo $src | sed -e "s/:[^:]*$//"`
315 srcport=`echo $src | sed -e "s/^.*://"`
316 destip=`echo $dest | sed -e "s/:[^:]*$//"`
317 destport=`echo $dest | sed -e "s/^.*://"`
318 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
319 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
321 # we only do one-way killtcp for NFS and CIFS
323 # for all others we do 2-way
325 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
328 _killcount=`expr $_killcount + 1`
332 [ $_failed = 0 ] || {
333 echo "Failed to send killtcp control"
336 [ $_killcount -gt 0 ] || {
340 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
342 _count=`expr $_count + 1`
343 [ $_count -gt 3 ] && {
344 echo "Timed out killing tcp connections for IP $_IP"
348 echo "killed $_killcount TCP connections to released IP $_IP"
351 ##################################################################
352 # kill off the local end for any TCP connections with the given IP
353 ##################################################################
354 kill_tcp_connections_local_only() {
359 connfile="$CTDB_BASE/state/connections.$_IP"
360 netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
361 netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
363 while read dest src; do
364 srcip=`echo $src | sed -e "s/:[^:]*$//"`
365 srcport=`echo $src | sed -e "s/^.*://"`
366 destip=`echo $dest | sed -e "s/:[^:]*$//"`
367 destport=`echo $dest | sed -e "s/^.*://"`
368 echo "Killing TCP connection $srcip:$srcport $destip:$destport"
369 ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
370 _killcount=`expr $_killcount + 1`
374 [ $_failed = 0 ] || {
375 echo "Failed to send killtcp control"
378 [ $_killcount -gt 0 ] || {
382 while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
384 _count=`expr $_count + 1`
385 [ $_count -gt 3 ] && {
386 echo "Timed out killing tcp connections for IP $_IP"
390 echo "killed $_killcount TCP connections to released IP $_IP"
393 ########################################################
394 # start/stop the nfs service on different platforms
395 ########################################################
398 [ -x /etc/init.d/nfsserver ] && {
401 [ -x /etc/init.d/nfslock ] && {
409 service nfsserver start
412 service nfsserver stop > /dev/null 2>&1
419 service nfslock start
423 service nfs stop > /dev/null 2>&1
424 service nfslock stop > /dev/null 2>&1
429 echo "Unknown platform. NFS is not supported with ctdb"
435 ########################################################
436 # start/stop the nfs lockmanager service on different platforms
437 ########################################################
438 startstop_nfslock() {
440 [ -x /etc/init.d/nfsserver ] && {
443 [ -x /etc/init.d/nfslock ] && {
449 # for sles there is no service for lockmanager
450 # so we instead just shutdown/restart nfs
453 service nfsserver start
456 service nfsserver stop > /dev/null 2>&1
463 service nfslock start
466 service nfslock stop > /dev/null 2>&1
471 echo "Unknown platform. NFS locking is not supported with ctdb"
477 ########################################################
478 # remove an ip address from an interface
479 ########################################################
481 # the ip tool will delete all secondary IPs if this is the primary.
482 # To work around this _very_ annoying behaviour we have to keep a
483 # record of the secondaries and re-add them afterwards. yuck
485 if ip addr list dev $2 primary | grep -q "inet $1 " ; then
486 secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
488 ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
489 [ -z "$secondaries" ] || {
490 for i in $secondaries; do
491 if ip addr list dev $2 | grep -q "inet $i" ; then
492 echo "kept secondary $i on dev $2"
494 echo "re-adding secondary address $i to dev $2"
495 ip addr add $i dev $2 || failed=1
501 ########################################################
502 # some simple logic for counting events - per eventscript
503 # usage: ctdb_counter_init <tag>
504 # ctdb_counter_incr <tag>
505 # ctdb_counter_limit <tag> <limit>
506 # e.g. <tag> = "fail-count"
507 # ctdb_counter_limit succeeds when count >= <limit>
508 ########################################################
509 _ctdb_counter_common () {
511 _eventscript="${0##*/}" # basename
513 _counter_file="$CTDB_BASE/state/${_eventscript}-${_tag}"
514 mkdir -p "${_counter_file%/*}" # dirname
516 ctdb_counter_init () {
517 _ctdb_counter_common "$1"
519 echo -n > "$_counter_file"
521 ctdb_counter_incr () {
522 _ctdb_counter_common "$1"
525 echo -n 1 >> "$_counter_file"
527 ctdb_counter_limit () {
528 _ctdb_counter_common "$1"
532 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
533 [ $_size -ge $_limit ]
535 ########################################################
536 # load a site local config file
537 ########################################################
539 [ -x $CTDB_BASE/rc.local ] && {
540 . $CTDB_BASE/rc.local