config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
   6
   7 [ -z "$CTDB_VARDIR" ] && {
   8     if [ -d "/var/lib/ctdb" ] ; then
   9         export CTDB_VARDIR="/var/lib/ctdb"
  10     else
  11         export CTDB_VARDIR="/var/ctdb"
  12     fi
  13 }
  14 [ -z "$CTDB_ETCDIR" ] && {
  15     export CTDB_ETCDIR="/etc"
  16 }
  17
  18 #######################################
  19 # pull in a system config file, if any
  20 _loadconfig() {
  21
  22     if [ -z "$1" ] ; then
  23         foo="${service_config:-${service_name}}"
  24         if [ -n "$foo" ] ; then
  25             loadconfig "$foo"
  26         fi
  27     elif [ "$1" != "ctdb" ] ; then
  28         loadconfig "ctdb"
  29     fi
  30
  31     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  32         . $CTDB_ETCDIR/sysconfig/$1
  33     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  34         . $CTDB_ETCDIR/default/$1
  35     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  36         . $CTDB_BASE/sysconfig/$1
  37     fi
  38 }
  39
  40 loadconfig () {
  41     _loadconfig "$@"
  42 }
  43
  44 ##############################################################
  45
  46 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  47 # configuration file.
  48 debug ()
  49 {
  50     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  51         # If there are arguments then echo them.  Otherwise expect to
  52         # use stdin, which allows us to pass lots of debug using a
  53         # here document.
  54         if [ -n "$1" ] ; then
  55             echo "DEBUG: $*"
  56         elif ! tty -s ; then
  57             sed -e 's@^@DEBUG: @'
  58         fi
  59     fi
  60 }
  61
  62 die ()
  63 {
  64     _msg="$1"
  65     _rc="${2:-1}"
  66
  67     echo "$_msg"
  68     exit $_rc
  69 }
  70
  71 # Log given message or stdin to either syslog or a CTDB log file
  72 # $1 is the tag passed to logger if syslog is in use.
  73 script_log ()
  74 {
  75     _tag="$1" ; shift
  76
  77     _using_syslog=false
  78     if [ "$CTDB_SYSLOG" = "yes" -o -z "$CTDB_LOGFILE" ] ; then
  79         _using_syslog=true
  80     fi
  81     case "$CTDB_OPTIONS" in
  82         *--syslog*) _using_syslog=true ;;
  83     esac
  84
  85     if $_using_syslog ; then
  86         logger -t "ctdbd: ${_tag}" $*
  87     else
  88         {
  89             if [ -n "$*" ] ; then
  90                 echo "$*"
  91             else
  92                 cat
  93             fi
  94         } >>"${CTDB_LOGFILE:-/var/log/log.ctdb}"
  95     fi
  96 }
  97
  98 # When things are run in the background in an eventscript then logging
  99 # output might get lost.  This is the "solution".  :-)
 100 background_with_logging ()
 101 {
 102     (
 103         "$@" 2>&1 </dev/null |
 104         script_log "${script_name}&"
 105     )&
 106
 107     return 0
 108 }
 109
 110 ##############################################################
 111 # check number of args for different events
 112 ctdb_check_args ()
 113 {
 114     case "$1" in
 115         takeip|releaseip)
 116             if [ $# != 4 ]; then
 117                 echo "ERROR: must supply interface, IP and maskbits"
 118                 exit 1
 119             fi
 120             ;;
 121         updateip)
 122             if [ $# != 5 ]; then
 123                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 124                 exit 1
 125             fi
 126             ;;
 127     esac
 128 }
 129
 130 ##############################################################
 131 # determine on what type of system (init style) we are running
 132 detect_init_style() {
 133     # only do detection if not already set:
 134     test "x$CTDB_INIT_STYLE" != "x" && return
 135
 136     if [ -x /sbin/startproc ]; then
 137         CTDB_INIT_STYLE="suse"
 138     elif [ -x /sbin/start-stop-daemon ]; then
 139         CTDB_INIT_STYLE="debian"
 140     else
 141         CTDB_INIT_STYLE="redhat"
 142     fi
 143 }
 144
 145 ######################################################
 146 # simulate /sbin/service on platforms that don't have it
 147 # _service() makes it easier to hook the service() function for
 148 # testing.
 149 _service ()
 150 {
 151   _service_name="$1"
 152   _op="$2"
 153
 154   # do nothing, when no service was specified
 155   [ -z "$_service_name" ] && return
 156
 157   if [ -x /sbin/service ]; then
 158       $_nice /sbin/service "$_service_name" "$_op"
 159   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 160       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 161   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 162       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 163   fi
 164 }
 165
 166 service()
 167 {
 168     _nice=""
 169     _service "$@"
 170 }
 171
 172 ######################################################
 173 # simulate /sbin/service (niced) on platforms that don't have it
 174 nice_service()
 175 {
 176     _nice="nice"
 177     _service "$@"
 178 }
 179
 180 ######################################################
 181 # wrapper around /proc/ settings to allow them to be hooked
 182 # for testing
 183 # 1st arg is relative path under /proc/, 2nd arg is value to set
 184 set_proc ()
 185 {
 186     echo "$2" >"/proc/$1"
 187 }
 188
 189 ######################################################
 190 # wrapper around getting file contents from /proc/ to allow
 191 # this to be hooked for testing
 192 # 1st arg is relative path under /proc/
 193 get_proc ()
 194 {
 195     cat "/proc/$1"
 196 }
 197
 198 ######################################################
 199 # Check that an RPC service is healthy -
 200 # this includes allowing a certain number of failures
 201 # before marking the NFS service unhealthy.
 202 #
 203 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 204 #
 205 # each triple is a set of 3 arguments: an operator, a
 206 # fail count limit and an action string.
 207 #
 208 # For example:
 209 #
 210 #       nfs_check_rpc_service "lockd" \
 211 #           -ge 15 "verbose restart unhealthy" \
 212 #           -eq 10 "restart:bs"
 213 #
 214 # says that if lockd is down for 15 iterations then do
 215 # a verbose restart of lockd and mark the node unhealthy.
 216 # Before this, after 10 iterations of failure, the
 217 # service is restarted silently in the background.
 218 # Order is important: the number of failures need to be
 219 # specified in reverse order because processing stops
 220 # after the first condition that is true.
 221 ######################################################
 222 nfs_check_rpc_service ()
 223 {
 224     _prog_name="$1" ; shift
 225
 226     _v=""
 227     case "$1" in
 228         -*) : ;;
 229         *) _v="$1" ; shift ;;
 230     esac
 231
 232     _version=${_v:-1}
 233     _rpc_prog="$_prog_name"
 234     _restart=""
 235     _opts=""
 236     case "$_prog_name" in
 237         knfsd)
 238             _rpc_prog=nfs
 239             _version=${_v:-3}
 240             _restart="echo 'Trying to restart NFS service'"
 241             _restart="${_restart}; startstop_nfs restart"
 242             ;;
 243         ganesha)
 244             _rpc_prog=nfs
 245             _version=${_v:-3}
 246             _restart="echo 'Trying to restart Ganesha NFS service'"
 247             _restart="${_restart}; startstop_ganesha restart"
 248             ;;
 249         mountd)
 250             _opts="${MOUNTD_PORT:+ -p }${MOUNTD_PORT}"
 251             ;;
 252         rquotad)
 253             _opts="${RQUOTAD_PORT:+ -p }${RQUOTAD_PORT}"
 254             ;;
 255         lockd)
 256             _rpc_prog=nlockmgr
 257             _version=${_v:-4}
 258             _restart="echo 'Trying to restart lock manager service'"
 259             _restart="${_restart}; startstop_nfslock restart"
 260             ;;
 261         statd)
 262             _rpc_prog=status
 263             _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}"
 264             _opts="${_opts}${STATD_PORT:+ -p }${STATD_PORT}"
 265             _opts="${_opts}${STATD_OUTGOING_PORT:+ -o }${STATD_OUTGOING_PORT}"
 266             ;;
 267         *)
 268             echo "Internal error: unknown RPC program \"$_prog_name\"."
 269             exit 1
 270     esac
 271
 272     _service_name="nfs_${_prog_name}"
 273
 274     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 275         ctdb_counter_init "$_service_name"
 276         return 0
 277     fi
 278
 279     ctdb_counter_incr "$_service_name"
 280
 281     while [ -n "$3" ] ; do
 282         ctdb_check_counter "quiet" "$1" "$2" "$_service_name" || {
 283             for _action in $3 ; do
 284                 case "$_action" in
 285                     verbose)
 286                         echo "$ctdb_check_rpc_out"
 287                         ;;
 288                     restart|restart:*)
 289                         # No explicit command specified, construct rpc command.
 290                         if [ -z "$_restart" ] ; then
 291                             _p="rpc.${_prog_name}"
 292                             _restart="echo 'Trying to restart $_prog_name [${_p}${_opts}]'"
 293                             _restart="${_restart}; killall -q -9 $_p"
 294                             _restart="${_restart}; $_p $_opts"
 295                         fi
 296
 297                         # Process restart flags...
 298                         _flags="${_action#restart:}"
 299                         # There may not have been a colon...
 300                         [ "$_flags" != "$_action" ] || _flags=""
 301                         # q=quiet - everything to /dev/null
 302                         if [ "${_flags#*q}" != "$_flags" ] ; then
 303                             _restart="{ ${_restart} ; } >/dev/null 2>&1"
 304                         fi
 305                         # s=stealthy - last command to /dev/null
 306                         if [ "${_flags#*s}" != "$_flags" ] ; then
 307                             _restart="${_restart} >/dev/null 2>&1"
 308                         fi
 309                         # b=background - the whole thing, easy and reliable
 310                         if [ "${_flags#*b}" != "$_flags" ] ; then
 311                             _restart="{ ${_restart} ; } &"
 312                         fi
 313
 314                         # Do it!
 315                         eval "${_restart}"
 316                         ;;
 317                     unhealthy)
 318                         exit 1
 319                         ;;
 320                     *)
 321                         echo "Internal error: unknown action \"$_action\"."
 322                         exit 1
 323                 esac
 324             done
 325
 326             # Only process the first action group.
 327             break
 328         }
 329         shift 3
 330     done
 331 }
 332
 333 ######################################################
 334 # check that a rpc server is registered with portmap
 335 # and responding to requests
 336 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 337 ######################################################
 338 ctdb_check_rpc ()
 339 {
 340     progname="$1"
 341     version="$2"
 342
 343     if ! ctdb_check_rpc_out=$(rpcinfo -u localhost $progname $version 2>&1) ; then
 344         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 345 $ctdb_check_rpc_out"
 346         echo "$ctdb_check_rpc_out"
 347         return 1
 348     fi
 349 }
 350
 351 ######################################################
 352 # check a set of directories is available
 353 # return 1 on a missing directory
 354 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
 355 ######################################################
 356 ctdb_check_directories_probe() {
 357     while IFS="" read d ; do
 358         case "$d" in
 359             *%*)
 360                 continue
 361                 ;;
 362             *)
 363                 [ -d "${d}/." ] || return 1
 364         esac
 365     done
 366 }
 367
 368 ######################################################
 369 # check a set of directories is available
 370 # usage: ctdb_check_directories SERVICE_NAME <directories...>
 371 ######################################################
 372 ctdb_check_directories() {
 373     n="${1:-${service_name}}"
 374     ctdb_check_directories_probe || {
 375         echo "ERROR: $n directory \"$d\" not available"
 376         exit 1
 377     }
 378 }
 379
 380 ######################################################
 381 # check a set of tcp ports
 382 # usage: ctdb_check_tcp_ports <ports...>
 383 ######################################################
 384
 385 # This flag file is created when a service is initially started.  It
 386 # is deleted the first time TCP port checks for that service succeed.
 387 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 388 # message if a port check fails.
 389 _ctdb_check_tcp_common ()
 390 {
 391     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 392 }
 393
 394 ctdb_check_tcp_init ()
 395 {
 396     _ctdb_check_tcp_common
 397     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 398     touch "$_ctdb_service_started_file"
 399 }
 400
 401 ctdb_check_tcp_ports()
 402 {
 403     if [ -z "$1" ] ; then
 404         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 405         exit 1
 406     fi
 407
 408     # Set default value for CTDB_TCP_PORT_CHECKS if unset.
 409     # If any of these defaults are unsupported then this variable can
 410     # be overridden in /etc/sysconfig/ctdb or via a file in
 411     # /etc/ctdb/rc.local.d/.
 412     : ${CTDB_TCP_PORT_CHECKERS:=ctdb nmap netstat}
 413
 414     for _c in $CTDB_TCP_PORT_CHECKERS ; do
 415         ctdb_check_tcp_ports_$_c "$@"
 416         case "$?" in
 417             0)
 418                 _ctdb_check_tcp_common
 419                 rm -f "$_ctdb_service_started_file"
 420                 return 0
 421                 ;;
 422             1)
 423                 _ctdb_check_tcp_common
 424                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 425                     echo "ERROR: $service_name tcp port $_p is not responding"
 426                     debug <<EOF
 427 $ctdb_check_tcp_ports_debug
 428 EOF
 429                 else
 430                     echo "INFO: $service_name tcp port $_p is not responding"
 431                 fi
 432
 433                 return 1
 434                 ;;
 435             127)
 436                 debug <<EOF
 437 ctdb_check_ports - checker $_c not implemented
 438 output from checker was:
 439 $ctdb_check_tcp_ports_debug
 440 EOF
 441                 ;;
 442             *)
 443
 444         esac
 445     done
 446
 447     echo "INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS=\"$CTDB_TCP_PORT_CHECKERS\""
 448
 449     return 127
 450 }
 451
 452 ctdb_check_tcp_ports_netstat ()
 453 {
 454     _cmd='netstat -l -t -n'
 455     _ns=$($_cmd 2>&1)
 456     if [ $? -eq 127 ] ; then
 457         # netstat probably not installed - unlikely?
 458         ctdb_check_tcp_ports_debug="$_ns"
 459         return 127
 460     fi
 461
 462     for _p ; do  # process each function argument (port)
 463         for _a in '0\.0\.0\.0' '::' ; do
 464             _pat="[[:space:]]${_a}:${_p}[[:space:]]+[^[:space:]]+[[:space:]]+LISTEN"
 465             if echo "$_ns" | grep -E -q "$_pat" ; then
 466                 # We matched the port, so process next port
 467                 continue 2
 468             fi
 469         done
 470
 471         # We didn't match the port, so flag an error.
 472         ctdb_check_tcp_ports_debug="$_cmd shows this output:
 473 $_ns"
 474         return 1
 475     done
 476
 477     return 0
 478 }
 479
 480 ctdb_check_tcp_ports_nmap ()
 481 {
 482     # nmap wants a comma-separated list of ports
 483     _ports=""
 484     for _p ; do
 485         _ports="${_ports}${_ports:+,}${_p}"
 486     done
 487
 488     _cmd="nmap -n -oG - -PS 127.0.0.1 -p $_ports"
 489
 490     _nmap_out=$($_cmd 2>&1)
 491     if [ $? -eq 127 ] ; then
 492         # nmap probably not installed
 493         ctdb_check_tcp_ports_debug="$_nmap_out"
 494         return 127
 495     fi
 496
 497     # get the port-related output
 498     _port_info=$(echo "$_nmap_out" | sed -n -r -e 's@^.*Ports:[[:space:]]@@p')
 499
 500     for _p ; do
 501         # looking for something like this:
 502         #  445/open/tcp//microsoft-ds///
 503         # possibly followed by a comma
 504         _t="$_p/open/tcp//"
 505         case "$_port_info" in
 506             # The info we're after must be either at the beginning of
 507             # the string or it must follow a space.
 508             $_t*|*\ $_t*) : ;;
 509             *)
 510                 # Nope, flag an error...
 511                 ctdb_check_tcp_ports_debug="$_cmd shows this output:
 512 $_nmap_out"
 513                 return 1
 514         esac
 515     done
 516
 517     return 0
 518 }
 519
 520 # Use the new "ctdb checktcpport" command to check the port.
 521 # This is very cheap.
 522 ctdb_check_tcp_ports_ctdb ()
 523 {
 524     for _p ; do  # process each function argument (port)
 525         _cmd="ctdb checktcpport $_p"
 526         _out=$($_cmd 2>&1)
 527         _ret=$?
 528         case "$_ret" in
 529             0)
 530                 ctdb_check_tcp_ports_debug="\"$_cmd\" was able to bind to port"
 531                 return 1
 532                 ;;
 533             98)
 534                 # Couldn't bind, something already listening, next port...
 535                 continue
 536                 ;;
 537             *)
 538                 ctdb_check_tcp_ports_debug="$_cmd (exited with $_ret) with output:
 539 $_out"
 540                 # assume not implemented
 541                 return 127
 542         esac
 543     done
 544
 545     return 0
 546 }
 547
 548 ######################################################
 549 # check a unix socket
 550 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 551 ######################################################
 552 ctdb_check_unix_socket() {
 553     socket_path="$1"
 554     [ -z "$socket_path" ] && return
 555
 556     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 557         echo "ERROR: $service_name socket $socket_path not found"
 558         return 1
 559     fi
 560 }
 561
 562 ######################################################
 563 # check a command returns zero status
 564 # usage: ctdb_check_command SERVICE_NAME <command>
 565 ######################################################
 566 ctdb_check_command() {
 567   service_name="$1"
 568   wait_cmd="$2"
 569   [ -z "$wait_cmd" ] && return;
 570   $wait_cmd > /dev/null 2>&1 || {
 571       echo "ERROR: $service_name - $wait_cmd returned error"
 572       exit 1
 573   }
 574 }
 575
 576 ################################################
 577 # kill off any TCP connections with the given IP
 578 ################################################
 579 kill_tcp_connections() {
 580     _IP="$1"
 581     _failed=0
 582
 583     _killcount=0
 584     connfile="$CTDB_VARDIR/state/connections.$_IP"
 585     mkdir -p "${connfile%/*}" # dirname
 586     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 587     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 588
 589     while read dest src; do
 590         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 591         srcport=`echo $src | sed -e "s/^.*://"`
 592         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 593         destport=`echo $dest | sed -e "s/^.*://"`
 594         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 595         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 596         case $destport in
 597           # we only do one-way killtcp for CIFS
 598           139|445) : ;;
 599           # for all others we do 2-way
 600           *)
 601                 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 602                 ;;
 603         esac
 604         _killcount=`expr $_killcount + 1`
 605      done < $connfile
 606     rm -f $connfile
 607
 608     [ $_failed = 0 ] || {
 609         echo "Failed to send killtcp control"
 610         return;
 611     }
 612     [ $_killcount -gt 0 ] || {
 613         return;
 614     }
 615     _count=0
 616     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 617         sleep 1
 618         _count=`expr $_count + 1`
 619         [ $_count -gt 3 ] && {
 620             echo "Timed out killing tcp connections for IP $_IP"
 621             return;
 622         }
 623     done
 624     echo "killed $_killcount TCP connections to released IP $_IP"
 625 }
 626
 627 ##################################################################
 628 # kill off the local end for any TCP connections with the given IP
 629 ##################################################################
 630 kill_tcp_connections_local_only() {
 631     _IP="$1"
 632     _failed=0
 633
 634     _killcount=0
 635     connfile="$CTDB_VARDIR/state/connections.$_IP"
 636     mkdir -p "${connfile%/*}" # dirname
 637     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 638     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 639
 640     while read dest src; do
 641         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 642         srcport=`echo $src | sed -e "s/^.*://"`
 643         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 644         destport=`echo $dest | sed -e "s/^.*://"`
 645         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 646         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 647         _killcount=`expr $_killcount + 1`
 648      done < $connfile
 649     rm -f $connfile
 650
 651     [ $_failed = 0 ] || {
 652         echo "Failed to send killtcp control"
 653         return;
 654     }
 655     [ $_killcount -gt 0 ] || {
 656         return;
 657     }
 658     _count=0
 659     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 660         sleep 1
 661         _count=`expr $_count + 1`
 662         [ $_count -gt 3 ] && {
 663             echo "Timed out killing tcp connections for IP $_IP"
 664             return;
 665         }
 666     done
 667     echo "killed $_killcount TCP connections to released IP $_IP"
 668 }
 669
 670 ##################################################################
 671 # tickle any TCP connections with the given IP
 672 ##################################################################
 673 tickle_tcp_connections() {
 674     _IP="$1"
 675     _failed=0
 676
 677     _killcount=0
 678     connfile="$CTDB_VARDIR/state/connections.$_IP"
 679     mkdir -p "${connfile%/*}" # dirname
 680     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 681     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 682
 683     while read dest src; do
 684         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 685         srcport=`echo $src | sed -e "s/^.*://"`
 686         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 687         destport=`echo $dest | sed -e "s/^.*://"`
 688         echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
 689         ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 690         echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
 691         ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 692      done < $connfile
 693     rm -f $connfile
 694
 695     [ $_failed = 0 ] || {
 696         echo "Failed to send tickle control"
 697         return;
 698     }
 699 }
 700
 701 ########################################################
 702 # start/stop the Ganesha nfs service
 703 ########################################################
 704 startstop_ganesha()
 705 {
 706     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 707     case "$1" in
 708         start)
 709             service "$_service_name" start
 710             ;;
 711         stop)
 712             service "$_service_name" stop
 713             ;;
 714         restart)
 715             service "$_service_name" restart
 716             ;;
 717     esac
 718 }
 719
 720 ########################################################
 721 # start/stop the nfs service on different platforms
 722 ########################################################
 723 startstop_nfs() {
 724         PLATFORM="unknown"
 725         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 726                 PLATFORM="sles"
 727         }
 728         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
 729                 PLATFORM="rhel"
 730         }
 731
 732         case $PLATFORM in
 733         sles)
 734                 case $1 in
 735                 start)
 736                         service nfsserver start
 737                         ;;
 738                 stop)
 739                         service nfsserver stop > /dev/null 2>&1
 740                         ;;
 741                 restart)
 742                         set_proc "fs/nfsd/threads" 0
 743                         service nfsserver stop > /dev/null 2>&1
 744                         pkill -9 nfsd
 745                         service nfsserver start
 746                         ;;
 747                 esac
 748                 ;;
 749         rhel)
 750                 case $1 in
 751                 start)
 752                         service nfslock start
 753                         service nfs start
 754                         ;;
 755                 stop)
 756                         service nfs stop
 757                         service nfslock stop
 758                         ;;
 759                 restart)
 760                         set_proc "fs/nfsd/threads" 0
 761                         service nfs stop > /dev/null 2>&1
 762                         service nfslock stop > /dev/null 2>&1
 763                         pkill -9 nfsd
 764                         service nfslock start
 765                         service nfs start
 766                         ;;
 767                 esac
 768                 ;;
 769         *)
 770                 echo "Unknown platform. NFS is not supported with ctdb"
 771                 exit 1
 772                 ;;
 773         esac
 774 }
 775
 776 ########################################################
 777 # start/stop the nfs lockmanager service on different platforms
 778 ########################################################
 779 startstop_nfslock() {
 780         PLATFORM="unknown"
 781         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 782                 PLATFORM="sles"
 783         }
 784         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
 785                 PLATFORM="rhel"
 786         }
 787
 788         case $PLATFORM in
 789         sles)
 790                 # for sles there is no service for lockmanager
 791                 # so we instead just shutdown/restart nfs
 792                 case $1 in
 793                 start)
 794                         service nfsserver start
 795                         ;;
 796                 stop)
 797                         service nfsserver stop > /dev/null 2>&1
 798                         ;;
 799                 restart)
 800                         service nfsserver stop
 801                         service nfsserver start
 802                         ;;
 803                 esac
 804                 ;;
 805         rhel)
 806                 case $1 in
 807                 start)
 808                         service nfslock start
 809                         ;;
 810                 stop)
 811                         service nfslock stop > /dev/null 2>&1
 812                         ;;
 813                 restart)
 814                         service nfslock stop
 815                         service nfslock start
 816                         ;;
 817                 esac
 818                 ;;
 819         *)
 820                 echo "Unknown platform. NFS locking is not supported with ctdb"
 821                 exit 1
 822                 ;;
 823         esac
 824 }
 825
 826 add_ip_to_iface()
 827 {
 828     _iface=$1
 829     _ip=$2
 830     _maskbits=$3
 831
 832     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
 833     mkdir -p "${_lockfile%/*}" # dirname
 834     [ -f "$_lockfile" ] || touch "$_lockfile"
 835
 836     (
 837         # Note: use of return/exit/die() below only gets us out of the
 838         # sub-shell, which is actually what we want.  That is, the
 839         # function should just return non-zero.
 840
 841         flock --timeout 30 0 || \
 842             die "add_ip_to_iface: unable to get lock for ${_iface}"
 843
 844         # Ensure interface is up
 845         ip link set "$_iface" up || \
 846             die "Failed to bringup interface $_iface"
 847
 848         ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || \
 849             die "Failed to add $_ip/$_maskbits on dev $_iface"
 850     ) <"$_lockfile"
 851
 852     # Do nothing here - return above only gets us out of the subshell
 853     # and doing anything here will affect the return code.
 854 }
 855
 856 delete_ip_from_iface()
 857 {
 858     _iface=$1
 859     _ip=$2
 860     _maskbits=$3
 861
 862     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
 863     mkdir -p "${_lockfile%/*}" # dirname
 864     [ -f "$_lockfile" ] || touch "$_lockfile"
 865
 866     (
 867         # Note: use of return/exit/die() below only gets us out of the
 868         # sub-shell, which is actually what we want.  That is, the
 869         # function should just return non-zero.
 870
 871         flock --timeout 30 0 || \
 872             die "delete_ip_from_iface: unable to get lock for ${_iface}"
 873
 874         _im="$_ip/$_maskbits"  # shorthand for readability
 875
 876         # "ip addr del" will delete all secondary IPs if this is the
 877         # primary.  To work around this _very_ annoying behaviour we
 878         # have to keep a record of the secondaries and re-add them
 879         # afterwards.  Yuck!
 880
 881         _secondaries=""
 882         if ip addr list dev "$_iface" primary | grep -Fq "inet $_im " ; then
 883             _secondaries=$(ip addr list dev "$_iface" secondary | \
 884                 awk '$1 == "inet" { print $2 }')
 885         fi
 886
 887         local _rc=0
 888         ip addr del "$_im" dev "$_iface" || {
 889             echo "Failed to del $_ip on dev $_iface"
 890             _rc=1
 891         }
 892
 893         if [ -n "$_secondaries" ] ; then
 894             for _i in $_secondaries; do
 895                 if ip addr list dev "$_iface" | grep -Fq "inet $_i" ; then
 896                     echo "Kept secondary $_i on dev $_iface"
 897                 else
 898                     echo "Re-adding secondary address $_i to dev $_iface"
 899                     ip addr add $_i brd + dev $_iface || {
 900                         echo "Failed to re-add address $_i to dev $_iface"
 901                         _rc=1
 902                     }
 903                 fi
 904             done
 905         fi
 906
 907         return $_rc
 908     ) <"$_lockfile"
 909
 910     # Do nothing here - return above only gets us out of the subshell
 911     # and doing anything here will affect the return code.
 912 }
 913
 914 # If the given IP is hosted then print 2 items: maskbits and iface
 915 ip_maskbits_iface ()
 916 {
 917     _addr="$1"
 918
 919     ip addr show to "${_addr}/32" 2>/dev/null | \
 920         awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
 921 }
 922
 923 drop_ip ()
 924 {
 925     _addr="${1%/*}"  # Remove optional maskbits
 926     _log_tag="$2"
 927
 928     set -- $(ip_maskbits_iface $_addr)
 929     if [ -n "$1" ] ; then
 930         _maskbits="$1"
 931         _iface="$2"
 932         if [ -n "$_log_tag" ] ; then
 933             script_log "$_log_tag" \
 934                 "Removing public address $_addr/$_maskbits from device $_iface"
 935         fi
 936         ip addr del $_addr/$_maskbits dev $_iface >/dev/null 2>&1
 937     fi
 938 }
 939
 940 drop_all_public_ips ()
 941 {
 942     _log_tag="$1"
 943
 944     while read _ip _x ; do
 945         drop_ip "$_ip" "$_log_tag"
 946     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 947 }
 948
 949 ########################################################
 950 # some simple logic for counting events - per eventscript
 951 # usage: ctdb_counter_init
 952 #        ctdb_counter_incr
 953 #        ctdb_check_counter_limit <limit>
 954 # ctdb_check_counter_limit fails when count >= <limit>
 955 ########################################################
 956 _ctdb_counter_common () {
 957     _service_name="${1:-${service_name}}"
 958     _counter_file="$ctdb_fail_dir/$_service_name"
 959     mkdir -p "${_counter_file%/*}" # dirname
 960 }
 961 ctdb_counter_init () {
 962     _ctdb_counter_common "$1"
 963
 964     >"$_counter_file"
 965 }
 966 ctdb_counter_incr () {
 967     _ctdb_counter_common "$1"
 968
 969     # unary counting!
 970     echo -n 1 >> "$_counter_file"
 971 }
 972 ctdb_check_counter_limit () {
 973     _ctdb_counter_common
 974
 975     _limit="${1:-${service_fail_limit}}"
 976     _quiet="$2"
 977
 978     # unary counting!
 979     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 980     if [ $_size -ge $_limit ] ; then
 981         echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
 982         exit 1
 983     elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
 984         echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
 985     fi
 986 }
 987 ctdb_check_counter () {
 988     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 989     _op="${2:--ge}"  # an integer operator supported by test
 990     _limit="${3:-${service_fail_limit}}"
 991     shift 3
 992     _ctdb_counter_common "$1"
 993
 994     # unary counting!
 995     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 996     if [ $_size $_op $_limit ] ; then
 997         if [ "$_msg" = "error" ] ; then
 998             echo "ERROR: $_limit consecutive failures for $_service_name, marking node unhealthy"
 999             exit 1
1000         else
1001             return 1
1002         fi
1003     fi
1004 }
1005
1006 ########################################################
1007
1008 ctdb_status_dir="$CTDB_VARDIR/status"
1009 ctdb_fail_dir="$CTDB_VARDIR/failcount"
1010
1011 ctdb_setup_service_state_dir ()
1012 {
1013     service_state_dir="$CTDB_VARDIR/state/${1:-${service_name}}"
1014     mkdir -p "$service_state_dir" || {
1015         echo "Error creating state dir \"$service_state_dir\""
1016         exit 1
1017     }
1018 }
1019
1020 ########################################################
1021 # Managed status history, for auto-start/stop
1022
1023 ctdb_managed_dir="$CTDB_VARDIR/managed_history"
1024
1025 _ctdb_managed_common ()
1026 {
1027     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1028 }
1029
1030 ctdb_service_managed ()
1031 {
1032     _ctdb_managed_common
1033     mkdir -p "$ctdb_managed_dir"
1034     touch "$_ctdb_managed_file"
1035 }
1036
1037 ctdb_service_unmanaged ()
1038 {
1039     _ctdb_managed_common
1040     rm -f "$_ctdb_managed_file"
1041 }
1042
1043 is_ctdb_previously_managed_service ()
1044 {
1045     _ctdb_managed_common
1046     [ -f "$_ctdb_managed_file" ]
1047 }
1048
1049 ########################################################
1050 # Check and set status
1051
1052 log_status_cat ()
1053 {
1054     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1055 }
1056
1057 ctdb_checkstatus ()
1058 {
1059     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1060         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1061         return 1
1062     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1063         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1064         return 2
1065     else
1066         return 0
1067     fi
1068 }
1069
1070 ctdb_setstatus ()
1071 {
1072     d="$ctdb_status_dir/$script_name"
1073     case "$1" in
1074         unhealthy|banned)
1075             mkdir -p "$d"
1076             cat "$2" >"$d/$1"
1077             ;;
1078         *)
1079             for i in "banned" "unhealthy" ; do
1080                 rm -f "$d/$i"
1081             done
1082             ;;
1083     esac
1084 }
1085
1086 ##################################################################
1087 # Reconfigure a service on demand
1088
1089 _ctdb_service_reconfigure_common ()
1090 {
1091     _d="$ctdb_status_dir/${service_name}"
1092     mkdir -p "$_d"
1093     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1094 }
1095
1096 ctdb_service_needs_reconfigure ()
1097 {
1098     _ctdb_service_reconfigure_common
1099     [ -e "$_ctdb_service_reconfigure_flag" ]
1100 }
1101
1102 ctdb_service_set_reconfigure ()
1103 {
1104     _ctdb_service_reconfigure_common
1105     >"$_ctdb_service_reconfigure_flag"
1106 }
1107
1108 ctdb_service_unset_reconfigure ()
1109 {
1110     _ctdb_service_reconfigure_common
1111     rm -f "$_ctdb_service_reconfigure_flag"
1112 }
1113
1114 ctdb_service_reconfigure ()
1115 {
1116     echo "Reconfiguring service \"${service_name}\"..."
1117     ctdb_service_unset_reconfigure
1118     service_reconfigure || return $?
1119     ctdb_counter_init
1120 }
1121
1122 # Default service_reconfigure() function does nothing.
1123 service_reconfigure ()
1124 {
1125     :
1126 }
1127
1128 ctdb_reconfigure_try_lock ()
1129 {
1130     _ctdb_service_reconfigure_common
1131     _lock="${_d}/reconfigure_lock"
1132     mkdir -p "${_lock%/*}" # dirname
1133     touch "$_lock"
1134
1135     (
1136         flock 0
1137         # This is overkill but will work if we need to extend this to
1138         # allow certain events to run multiple times in parallel
1139         # (e.g. takeip) and write multiple PIDs to the file.
1140         read _locker_event
1141         if [ -n "$_locker_event" ] ; then
1142             while read _pid ; do
1143                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1144                     kill -0 "$_pid" 2>/dev/null ; then
1145                     exit 1
1146                 fi
1147             done
1148         fi
1149
1150         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1151         exit 0
1152     ) <"$_lock"
1153 }
1154
1155 ctdb_replay_monitor_status ()
1156 {
1157     echo "Replaying previous status for this script due to reconfigure..."
1158     # Leading colon (':') is missing in some versions...
1159     _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:")
1160     # Output looks like this:
1161     # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar:
1162     # This is the cheapest way of getting fields in the middle.
1163     set -- $(IFS=":" ; echo $_out)
1164     _code="$3"
1165     _status="$4"
1166     # The error output field can include colons so we'll try to
1167     # preserve them.  The weak checking at the beginning tries to make
1168     # this work for both broken (no leading ':') and fixed output.
1169     _out="${_out%:}"
1170     _err_out="${_out#*monitor:${script_name}:*:*:*:*:}"
1171     case "$_status" in
1172         OK) : ;;  # Do nothing special.
1173         TIMEDOUT)
1174             # Recast this as an error, since we can't exit with the
1175             # correct negative number.
1176             _code=1
1177             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1178             ;;
1179         DISABLED)
1180             # Recast this as an OK, since we can't exit with the
1181             # correct negative number.
1182             _code=0
1183             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1184             ;;
1185         *) : ;;  # Must be ERROR, do nothing special.
1186     esac
1187     echo "$_err_out"
1188     exit $_code
1189 }
1190
1191 ctdb_service_check_reconfigure ()
1192 {
1193     # We only care about some events in this function.  For others we
1194     # return now.
1195     case "$event_name" in
1196         monitor|ipreallocated|reconfigure) : ;;
1197         *) return 0 ;;
1198     esac
1199
1200     if ctdb_reconfigure_try_lock ; then
1201         # No events covered by this function are running, so proceed
1202         # with gay abandon.
1203         case "$event_name" in
1204             reconfigure)
1205                 (ctdb_service_reconfigure)
1206                 exit $?
1207                 ;;
1208             ipreallocated)
1209                 if ctdb_service_needs_reconfigure ; then
1210                     ctdb_service_reconfigure
1211                 fi
1212                 ;;
1213             monitor)
1214                 if ctdb_service_needs_reconfigure ; then
1215                     ctdb_service_reconfigure
1216                     # Given that the reconfigure might not have
1217                     # resulted in the service being stable yet, we
1218                     # replay the previous status since that's the best
1219                     # information we have.
1220                     ctdb_replay_monitor_status
1221                 fi
1222                 ;;
1223         esac
1224     else
1225         # Somebody else is running an event we don't want to collide
1226         # with.  We proceed with caution.
1227         case "$event_name" in
1228             reconfigure)
1229                 # Tell whoever called us to retry.
1230                 exit 2
1231                 ;;
1232             ipreallocated)
1233                 # Defer any scheduled reconfigure and just run the
1234                 # rest of the ipreallocated event, as per the
1235                 # eventscript.  There's an assumption here that the
1236                 # event doesn't depend on any scheduled reconfigure.
1237                 # This is true in the current code.
1238                 return 0
1239                 ;;
1240             monitor)
1241                 # There is most likely a reconfigure in progress so
1242                 # the service is possibly unstable.  As above, we
1243                 # defer any scheduled reconfigured.  We also replay
1244                 # the previous monitor status since that's the best
1245                 # information we have.
1246                 ctdb_replay_monitor_status
1247                 ;;
1248         esac
1249     fi
1250 }
1251
1252 ##################################################################
1253 # Does CTDB manage this service? - and associated auto-start/stop
1254
1255 ctdb_compat_managed_service ()
1256 {
1257     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1258         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1259     fi
1260 }
1261
1262 is_ctdb_managed_service ()
1263 {
1264     # $t is used just for readability and to allow better accurate
1265     # matching via leading/trailing spaces
1266     t=" $CTDB_MANAGED_SERVICES "
1267
1268     # Return 0 if "<space>$service_name<space>" appears in $t
1269     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1270         return 0
1271     fi
1272
1273     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1274     # backward compatibility and try again.
1275     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1276     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1277     ctdb_compat_managed_service "$CTDB_MANAGES_SCP"      "scp"
1278     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1279     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1280     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1281     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1282     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1283     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1284     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1285
1286     t=" $CTDB_MANAGED_SERVICES "
1287
1288     # Return 0 if "<space>$service_name<space>" appears in $t
1289     [ "${t#* ${service_name} }" != "${t}" ]
1290 }
1291
1292 ctdb_start_stop_service ()
1293 {
1294     # Allow service-start/service-stop pseudo-events to start/stop
1295     # services when we're not auto-starting/stopping and we're not
1296     # monitoring.
1297     case "$event_name" in
1298         service-start)
1299             if is_ctdb_managed_service ; then
1300                 die 'service-start event not permitted when service is managed'
1301             fi
1302             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1303                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1304             fi
1305             ctdb_service_start
1306             exit $?
1307             ;;
1308         service-stop)
1309             if is_ctdb_managed_service ; then
1310                 die 'service-stop event not permitted when service is managed'
1311             fi
1312             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1313                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1314             fi
1315             ctdb_service_stop
1316             exit $?
1317             ;;
1318     esac
1319
1320     # Do nothing unless configured to...
1321     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1322
1323     [ "$event_name" = "monitor" ] || return 0
1324
1325     if is_ctdb_managed_service ; then
1326         if ! is_ctdb_previously_managed_service ; then
1327             echo "Starting service \"$service_name\" - now managed"
1328             background_with_logging ctdb_service_start
1329             exit $?
1330         fi
1331     else
1332         if is_ctdb_previously_managed_service ; then
1333             echo "Stopping service \"$service_name\" - no longer managed"
1334             background_with_logging ctdb_service_stop
1335             exit $?
1336         fi
1337     fi
1338 }
1339
1340 ctdb_service_start ()
1341 {
1342     # The service is marked managed if we've ever tried to start it.
1343     ctdb_service_managed
1344
1345     service_start || return $?
1346
1347     ctdb_counter_init
1348     ctdb_check_tcp_init
1349 }
1350
1351 ctdb_service_stop ()
1352 {
1353     ctdb_service_unmanaged
1354     service_stop
1355 }
1356
1357 # Default service_start() and service_stop() functions.
1358
1359 # These may be overridden in an eventscript.  When overriding, the
1360 # following convention must be followed.  If these functions are
1361 # called with no arguments then they may use internal logic to
1362 # determine whether the service is managed and, therefore, whether
1363 # they should take any action.  However, if the service name is
1364 # specified as an argument then an attempt must be made to start or
1365 # stop the service.  This is because the auto-start/stop code calls
1366 # them with the service name as an argument.
1367 service_start ()
1368 {
1369     service "$service_name" start
1370 }
1371
1372 service_stop ()
1373 {
1374     service "$service_name" stop
1375 }
1376
1377 ##################################################################
1378
1379 ctdb_standard_event_handler ()
1380 {
1381     case "$1" in
1382         status)
1383             ctdb_checkstatus
1384             exit
1385             ;;
1386         setstatus)
1387             shift
1388             ctdb_setstatus "$@"
1389             exit
1390             ;;
1391     esac
1392 }
1393
1394 # iptables doesn't like being re-entered, so flock-wrap it.
1395 iptables()
1396 {
1397         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1398 }
1399
1400 ########################################################
1401 # tickle handling
1402 ########################################################
1403
1404 update_tickles ()
1405 {
1406         _port="$1"
1407
1408         tickledir="$CTDB_VARDIR/state/tickles"
1409         mkdir -p "$tickledir"
1410
1411         # Who am I?
1412         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1413
1414         # What public IPs do I hold?
1415         _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
1416
1417         # IPs as a regexp choice
1418         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1419
1420         # Record connections to our public IPs in a temporary file
1421         _my_connections="${tickledir}/${_port}.connections"
1422         rm -f "$_my_connections"
1423         netstat -tn |
1424         awk -v destpat="^${_ipschoice}:${_port}\$" \
1425           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1426         sort >"$_my_connections"
1427
1428         # Record our current tickles in a temporary file
1429         _my_tickles="${tickledir}/${_port}.tickles"
1430         rm -f "$_my_tickles"
1431         for _i in $_ips ; do
1432                 ctdb -Y gettickles $_i $_port |
1433                 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1434         done |
1435         sort >"$_my_tickles"
1436
1437         # Add tickles for connections that we haven't already got tickles for
1438         comm -23 "$_my_connections" "$_my_tickles" |
1439         while read _src _dst ; do
1440                 ctdb addtickle $_src $_dst
1441         done
1442
1443         # Remove tickles for connections that are no longer there
1444         comm -13 "$_my_connections" "$_my_tickles" |
1445         while read _src _dst ; do
1446                 ctdb deltickle $_src $_dst
1447         done
1448
1449         rm -f "$_my_connections" "$_my_tickles"
1450 }
1451
1452 ########################################################
1453 # load a site local config file
1454 ########################################################
1455
1456 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1457         . "$CTDB_RC_LOCAL"
1458 }
1459
1460 [ -x $CTDB_BASE/rc.local ] && {
1461         . $CTDB_BASE/rc.local
1462 }
1463
1464 [ -d $CTDB_BASE/rc.local.d ] && {
1465         for i in $CTDB_BASE/rc.local.d/* ; do
1466                 [ -x "$i" ] && . "$i"
1467         done
1468 }
1469
1470 script_name="${0##*/}"       # basename
1471 service_fail_limit=1
1472 event_name="$1"