ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x /usr/sbin/service ]; then
 175       $_nice /usr/sbin/service "$_service_name" "$_op"
 176   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 178   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 179       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 180   fi
 181 }
 182
 183 service()
 184 {
 185     _nice=""
 186     _service "$@"
 187 }
 188
 189 ######################################################
 190 # simulate /sbin/service (niced) on platforms that don't have it
 191 nice_service()
 192 {
 193     _nice="nice"
 194     _service "$@"
 195 }
 196
 197 ######################################################
 198 # wrapper around /proc/ settings to allow them to be hooked
 199 # for testing
 200 # 1st arg is relative path under /proc/, 2nd arg is value to set
 201 set_proc ()
 202 {
 203     echo "$2" >"/proc/$1"
 204 }
 205
 206 ######################################################
 207 # wrapper around getting file contents from /proc/ to allow
 208 # this to be hooked for testing
 209 # 1st arg is relative path under /proc/
 210 get_proc ()
 211 {
 212     cat "/proc/$1"
 213 }
 214
 215 ######################################################
 216 # Print up to $_max kernel stack traces for processes named $_program
 217 program_stack_traces ()
 218 {
 219     _prog="$1"
 220     _max="${2:-1}"
 221
 222     _count=1
 223     for _pid in $(pidof "$_prog") ; do
 224         [ $_count -le $_max ] || break
 225
 226         # Do this first to avoid racing with process exit
 227         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 228         if [ -n "$_stack" ] ; then
 229             echo "Stack trace for ${_prog}[${_pid}]:"
 230             echo "$_stack"
 231             _count=$(($_count + 1))
 232         fi
 233     done
 234 }
 235
 236 ######################################################
 237 # Check that an RPC service is healthy -
 238 # this includes allowing a certain number of failures
 239 # before marking the NFS service unhealthy.
 240 #
 241 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 242 #
 243 # each triple is a set of 3 arguments: an operator, a
 244 # fail count limit and an action string.
 245 #
 246 # For example:
 247 #
 248 #       nfs_check_rpc_service "lockd" \
 249 #           -ge 15 "verbose restart unhealthy" \
 250 #           -eq 10 "restart:bs"
 251 #
 252 # says that if lockd is down for 15 iterations then do
 253 # a verbose restart of lockd and mark the node unhealthy.
 254 # Before this, after 10 iterations of failure, the
 255 # service is restarted silently in the background.
 256 # Order is important: the number of failures need to be
 257 # specified in reverse order because processing stops
 258 # after the first condition that is true.
 259 ######################################################
 260 nfs_check_rpc_service ()
 261 {
 262     _prog_name="$1" ; shift
 263
 264     if _nfs_check_rpc_common "$_prog_name" ; then
 265         return
 266     fi
 267
 268     while [ -n "$3" ] ; do
 269         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 270             break
 271         fi
 272         shift 3
 273     done
 274 }
 275
 276 # The new way of doing things...
 277 nfs_check_rpc_services ()
 278 {
 279     # Files must end with .check - avoids editor backups, RPM fu, ...
 280     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 281         _t="${_f%.check}"
 282         _prog_name="${_t##*/[0-9][0-9].}"
 283
 284         if _nfs_check_rpc_common "$_prog_name" ; then
 285             # This RPC service is up, check next service...
 286             continue
 287         fi
 288
 289         # Check each line in the file in turn until one of the limit
 290         # checks is hit...
 291         while read _cmp _lim _rest ; do
 292             # Skip comments
 293             case "$_cmp" in
 294                 \#*) continue ;;
 295             esac
 296
 297             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 298                 # Limit was hit on this line, no further checking...
 299                 break
 300             fi
 301         done <"$_f"
 302     done
 303 }
 304
 305 _nfs_check_rpc_common ()
 306 {
 307     _prog_name="$1"
 308
 309     # Some platforms don't have separate programs for all services.
 310     case "$_prog_name" in
 311         statd)
 312             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 313     esac
 314
 315     case "$_prog_name" in
 316         nfsd)
 317             _rpc_prog=nfs
 318             _version=3
 319             ;;
 320         mountd)
 321             _rpc_prog=mountd
 322             _version=1
 323             ;;
 324         rquotad)
 325             _rpc_prog=rquotad
 326             _version=1
 327             ;;
 328         lockd)
 329             _rpc_prog=nlockmgr
 330             _version=4
 331             ;;
 332         statd)
 333             _rpc_prog=status
 334             _version=1
 335             ;;
 336         *)
 337             echo "Internal error: unknown RPC program \"$_prog_name\"."
 338             exit 1
 339     esac
 340
 341     _service_name="nfs_${_prog_name}"
 342
 343     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 344         ctdb_counter_init "$_service_name"
 345         return 0
 346     fi
 347
 348     ctdb_counter_incr "$_service_name"
 349
 350     return 1
 351 }
 352
 353 _nfs_check_rpc_action ()
 354 {
 355     _cmp="$1"
 356     _limit="$2"
 357     _actions="$3"
 358
 359     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 360         return 1
 361     fi
 362
 363     for _action in $_actions ; do
 364         case "$_action" in
 365             verbose)
 366                 echo "$ctdb_check_rpc_out"
 367                 ;;
 368             restart)
 369                 _nfs_restart_rpc_service "$_prog_name"
 370                 ;;
 371             restart:b)
 372                 _nfs_restart_rpc_service "$_prog_name" true
 373                 ;;
 374             unhealthy)
 375                 exit 1
 376                 ;;
 377             *)
 378                 echo "Internal error: unknown action \"$_action\"."
 379                 exit 1
 380         esac
 381     done
 382
 383     return 0
 384 }
 385
 386 _nfs_restart_rpc_service ()
 387 {
 388     _prog_name="$1"
 389     _background="${2:-false}"
 390
 391     if $_background ; then
 392         _maybe_background="background_with_logging"
 393     else
 394         _maybe_background=""
 395     fi
 396
 397     _p="rpc.${_prog_name}"
 398
 399     case "$_prog_name" in
 400         nfsd)
 401             echo "Trying to restart NFS service"
 402             $_maybe_background startstop_nfs restart
 403             ;;
 404         mountd)
 405             echo "Trying to restart $_prog_name [${_p}]"
 406             killall -q -9 "$_p"
 407             nfs_dump_some_threads "$_p"
 408             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 409             ;;
 410         rquotad)
 411             echo "Trying to restart $_prog_name [${_p}]"
 412             killall -q -9 "$_p"
 413             nfs_dump_some_threads "$_p"
 414             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 415             ;;
 416         lockd)
 417             echo "Trying to restart lock manager service"
 418             $_maybe_background startstop_nfslock restart
 419             ;;
 420         statd)
 421             echo "Trying to restart $_prog_name [${_p}]"
 422             killall -q -9 "$_p"
 423             nfs_dump_some_threads "$_p"
 424             $_maybe_background $_p \
 425                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 426                 ${STATD_PORT:+-p} $STATD_PORT \
 427                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 428             ;;
 429         *)
 430             echo "Internal error: unknown RPC program \"$_prog_name\"."
 431             exit 1
 432     esac
 433 }
 434
 435 ######################################################
 436 # check that a rpc server is registered with portmap
 437 # and responding to requests
 438 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 439 ######################################################
 440 ctdb_check_rpc ()
 441 {
 442     progname="$1"
 443     version="$2"
 444
 445     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 446
 447     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 448         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 449 $ctdb_check_rpc_out"
 450         echo "$ctdb_check_rpc_out"
 451         return 1
 452     fi
 453 }
 454
 455 ######################################################
 456 # Ensure $service_name is set
 457 assert_service_name ()
 458 {
 459     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 460 }
 461
 462 ######################################################
 463 # check a set of directories is available
 464 # return 1 on a missing directory
 465 # directories are read from stdin
 466 ######################################################
 467 ctdb_check_directories_probe()
 468 {
 469     while IFS="" read d ; do
 470         case "$d" in
 471             *%*)
 472                 continue
 473                 ;;
 474             *)
 475                 [ -d "${d}/." ] || return 1
 476         esac
 477     done
 478 }
 479
 480 ######################################################
 481 # check a set of directories is available
 482 # directories are read from stdin
 483 ######################################################
 484 ctdb_check_directories()
 485 {
 486     ctdb_check_directories_probe || {
 487         echo "ERROR: $service_name directory \"$d\" not available"
 488         exit 1
 489     }
 490 }
 491
 492 ######################################################
 493 # check a set of tcp ports
 494 # usage: ctdb_check_tcp_ports <ports...>
 495 ######################################################
 496
 497 # This flag file is created when a service is initially started.  It
 498 # is deleted the first time TCP port checks for that service succeed.
 499 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 500 # message if a port check fails.
 501 _ctdb_check_tcp_common ()
 502 {
 503     assert_service_name
 504     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 505 }
 506
 507 ctdb_check_tcp_init ()
 508 {
 509     _ctdb_check_tcp_common
 510     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 511     touch "$_ctdb_service_started_file"
 512 }
 513
 514 # Check whether something is listening on all of the given TCP ports
 515 # using the "ctdb checktcpport" command.
 516 ctdb_check_tcp_ports()
 517 {
 518     if [ -z "$1" ] ; then
 519         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 520         exit 1
 521     fi
 522
 523     for _p ; do  # process each function argument (port)
 524         _cmd="ctdb checktcpport $_p"
 525         _out=$($_cmd 2>&1)
 526         _ret=$?
 527         case "$_ret" in
 528             0)
 529                 _ctdb_check_tcp_common
 530                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 531                     echo "ERROR: $service_name tcp port $_p is not responding"
 532                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 533                 else
 534                     echo "INFO: $service_name tcp port $_p is not responding"
 535                 fi
 536
 537                 return 1
 538                 ;;
 539             98)
 540                 # Couldn't bind, something already listening, next port...
 541                 continue
 542                 ;;
 543             *)
 544                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 545                 debug <<EOF
 546 ctdb checktcpport (exited with $_ret) with output:
 547 $_out"
 548 EOF
 549                 return $_ret
 550         esac
 551     done
 552
 553     # All ports listening
 554     _ctdb_check_tcp_common
 555     rm -f "$_ctdb_service_started_file"
 556     return 0
 557 }
 558
 559 ######################################################
 560 # check a unix socket
 561 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 562 ######################################################
 563 ctdb_check_unix_socket() {
 564     socket_path="$1"
 565     [ -z "$socket_path" ] && return
 566
 567     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 568         echo "ERROR: $service_name socket $socket_path not found"
 569         return 1
 570     fi
 571 }
 572
 573 ######################################################
 574 # check a command returns zero status
 575 # usage: ctdb_check_command <command>
 576 ######################################################
 577 ctdb_check_command ()
 578 {
 579     _out=$("$@" 2>&1) || {
 580         echo "ERROR: $* returned error"
 581         echo "$_out" | debug
 582         exit 1
 583     }
 584 }
 585
 586 ################################################
 587 # kill off any TCP connections with the given IP
 588 ################################################
 589 kill_tcp_connections ()
 590 {
 591     _ip="$1"
 592
 593     _oneway=false
 594     if [ "$2" = "oneway" ] ; then
 595         _oneway=true
 596     fi
 597
 598     get_tcp_connections_for_ip "$_ip" | {
 599         _killcount=0
 600         _connections=""
 601         _nl="
 602 "
 603         while read _dst _src; do
 604             _destport="${_dst##*:}"
 605             __oneway=$_oneway
 606             case $_destport in
 607                 # we only do one-way killtcp for CIFS
 608                 139|445) __oneway=true ;;
 609             esac
 610
 611             echo "Killing TCP connection $_src $_dst"
 612             _connections="${_connections}${_nl}${_src} ${_dst}"
 613             if ! $__oneway ; then
 614                 _connections="${_connections}${_nl}${_dst} ${_src}"
 615             fi
 616
 617             _killcount=$(($_killcount + 1))
 618         done
 619
 620         if [ $_killcount -eq 0 ] ; then
 621             return
 622         fi
 623
 624         echo "$_connections" | ctdb killtcp || {
 625             echo "Failed to send killtcp control"
 626             return
 627         }
 628
 629         _count=0
 630         while : ; do
 631             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 632
 633             if [ $_remaining -eq 0 ] ; then
 634                 echo "Killed $_killcount TCP connections to released IP $_ip"
 635                 return
 636             fi
 637
 638             _count=$(($_count + 1))
 639             if [ $_count -gt 3 ] ; then
 640                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 641                 return
 642             fi
 643
 644             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 645             sleep 1
 646         done
 647     }
 648 }
 649
 650 ##################################################################
 651 # kill off the local end for any TCP connections with the given IP
 652 ##################################################################
 653 kill_tcp_connections_local_only ()
 654 {
 655     kill_tcp_connections "$1" "oneway"
 656 }
 657
 658 ##################################################################
 659 # tickle any TCP connections with the given IP
 660 ##################################################################
 661 tickle_tcp_connections ()
 662 {
 663     _ip="$1"
 664
 665     get_tcp_connections_for_ip "$_ip" |
 666     {
 667         _failed=false
 668
 669         while read dest src; do
 670             echo "Tickle TCP connection $src $dest"
 671             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 672             echo "Tickle TCP connection $dest $src"
 673             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 674         done
 675
 676         if $_failed ; then
 677             echo "Failed to send tickle control"
 678         fi
 679     }
 680 }
 681
 682 get_tcp_connections_for_ip ()
 683 {
 684     _ip="$1"
 685
 686     netstat -tn | awk -v ip=$_ip \
 687         'index($1, "tcp") == 1 && \
 688          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 689          && $6 == "ESTABLISHED" \
 690          {print $4" "$5}'
 691 }
 692
 693 ########################################################
 694 # start/stop the Ganesha nfs service
 695 ########################################################
 696 startstop_ganesha()
 697 {
 698     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 699     case "$1" in
 700         start)
 701             service "$_service_name" start
 702             ;;
 703         stop)
 704             service "$_service_name" stop
 705             ;;
 706         restart)
 707             service "$_service_name" stop
 708             nfs_dump_some_threads "rpc.statd"
 709             service "$_service_name" start
 710             ;;
 711     esac
 712 }
 713
 714 ########################################################
 715 # start/stop the nfs service on different platforms
 716 ########################################################
 717 startstop_nfs() {
 718         PLATFORM="unknown"
 719         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 720                 PLATFORM="sles"
 721         }
 722         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 723             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 724                 PLATFORM="rhel"
 725         }
 726
 727         case $PLATFORM in
 728         sles)
 729                 case $1 in
 730                 start)
 731                         service nfsserver start
 732                         ;;
 733                 stop)
 734                         service nfsserver stop > /dev/null 2>&1
 735                         ;;
 736                 restart)
 737                         set_proc "fs/nfsd/threads" 0
 738                         service nfsserver stop > /dev/null 2>&1
 739                         pkill -9 nfsd
 740                         nfs_dump_some_threads
 741                         service nfsserver start
 742                         ;;
 743                 esac
 744                 ;;
 745         rhel)
 746                 case $1 in
 747                 start)
 748                         service nfslock start
 749                         service nfs start
 750                         ;;
 751                 stop)
 752                         service nfs stop
 753                         service nfslock stop
 754                         ;;
 755                 restart)
 756                         set_proc "fs/nfsd/threads" 0
 757                         service nfs stop > /dev/null 2>&1
 758                         service nfslock stop > /dev/null 2>&1
 759                         pkill -9 nfsd
 760                         nfs_dump_some_threads
 761                         service nfslock start
 762                         service nfs start
 763                         ;;
 764                 esac
 765                 ;;
 766         *)
 767                 echo "Unknown platform. NFS is not supported with ctdb"
 768                 exit 1
 769                 ;;
 770         esac
 771 }
 772
 773 # Dump up to the configured number of nfsd thread backtraces.
 774 nfs_dump_some_threads ()
 775 {
 776     _prog="${1:-nfsd}"
 777
 778     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 779     [ $_num -gt 0 ] || return 0
 780
 781     program_stack_traces "$_prog" $_num
 782 }
 783
 784 ########################################################
 785 # start/stop the nfs lockmanager service on different platforms
 786 ########################################################
 787 startstop_nfslock() {
 788         PLATFORM="unknown"
 789         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 790                 PLATFORM="sles"
 791         }
 792         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 793             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 794                 PLATFORM="rhel"
 795         }
 796
 797         case $PLATFORM in
 798         sles)
 799                 # for sles there is no service for lockmanager
 800                 # so we instead just shutdown/restart nfs
 801                 case $1 in
 802                 start)
 803                         service nfsserver start
 804                         ;;
 805                 stop)
 806                         service nfsserver stop > /dev/null 2>&1
 807                         ;;
 808                 restart)
 809                         service nfsserver stop > /dev/null 2>&1
 810                         service nfsserver start
 811                         ;;
 812                 esac
 813                 ;;
 814         rhel)
 815                 case $1 in
 816                 start)
 817                         service nfslock start
 818                         ;;
 819                 stop)
 820                         service nfslock stop > /dev/null 2>&1
 821                         ;;
 822                 restart)
 823                         service nfslock stop > /dev/null 2>&1
 824                         service nfslock start
 825                         ;;
 826                 esac
 827                 ;;
 828         *)
 829                 echo "Unknown platform. NFS locking is not supported with ctdb"
 830                 exit 1
 831                 ;;
 832         esac
 833 }
 834
 835 # Periodically update the statd database
 836 nfs_statd_update ()
 837 {
 838     _update_period="$1"
 839
 840     _statd_update_trigger="$service_state_dir/update-trigger"
 841     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 842
 843     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 844     _current_time=$(date +"%s")
 845     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 846         touch "$_statd_update_trigger"
 847         $CTDB_BASE/statd-callout updatelocal &
 848         $CTDB_BASE/statd-callout updateremote &
 849     fi
 850 }
 851
 852 ########################################################
 853
 854 add_ip_to_iface ()
 855 {
 856     _iface=$1
 857     _ip=$2
 858     _maskbits=$3
 859
 860     # Ensure interface is up
 861     ip link set "$_iface" up || \
 862         die "Failed to bringup interface $_iface"
 863
 864     # Only need to define broadcast for IPv4
 865     case "$ip" in
 866         *:*) _bcast=""      ;;
 867         *)   _bcast="brd +" ;;
 868     esac
 869
 870     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 871         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 872         return 1
 873     }
 874
 875     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 876     if [ -z "$_bcast" ] ; then
 877         for _x in $(seq 1 10) ; do
 878             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 879             sleep 0.5
 880         done
 881
 882         # If the address was a duplicate then it won't be on the
 883         # interface so flag an error.
 884         _t=$(ip addr show to "${_ip}/128")
 885         case "$_t" in
 886             "")
 887                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 888                 return 1
 889                 ;;
 890             *tentative*|*dadfailed*)
 891                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 892                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 893                 return 1
 894                 ;;
 895         esac
 896     fi
 897 }
 898
 899 delete_ip_from_iface()
 900 {
 901     _iface=$1
 902     _ip=$2
 903     _maskbits=$3
 904
 905     # This could be set globally for all interfaces but it is probably
 906     # better to avoid surprises, so limit it the interfaces where CTDB
 907     # has public IP addresses.  There isn't anywhere else convenient
 908     # to do this so just set it each time.  This is much cheaper than
 909     # remembering and re-adding secondaries.
 910     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 911
 912     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 913         echo "Failed to del $_ip on dev $_iface"
 914         return 1
 915     }
 916 }
 917
 918 # If the given IP is hosted then print 2 items: maskbits and iface
 919 ip_maskbits_iface ()
 920 {
 921     _addr="$1"
 922
 923     case "$_addr" in
 924         *:*) _family="inet6" ; _bits=128 ;;
 925         *)   _family="inet"  ; _bits=32  ;;
 926     esac
 927
 928     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 929         awk -v family="${_family}" \
 930             'NR == 1 { iface = gensub(":$", "", 1, $2) } \
 931              $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }'
 932 }
 933
 934 drop_ip ()
 935 {
 936     _addr="${1%/*}"  # Remove optional maskbits
 937
 938     set -- $(ip_maskbits_iface $_addr)
 939     if [ -n "$1" ] ; then
 940         _maskbits="$1"
 941         _iface="$2"
 942         echo "Removing public address $_addr/$_maskbits from device $_iface"
 943         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 944     fi
 945 }
 946
 947 drop_all_public_ips ()
 948 {
 949     while read _ip _x ; do
 950         drop_ip "$_ip"
 951     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 952 }
 953
 954 flush_route_cache ()
 955 {
 956     set_proc sys/net/ipv4/route/flush 1
 957     set_proc sys/net/ipv6/route/flush 1
 958 }
 959
 960 ########################################################
 961 # Simple counters
 962 _ctdb_counter_common () {
 963     _service_name="${1:-${service_name:-${script_name}}}"
 964     _counter_file="$ctdb_fail_dir/$_service_name"
 965     mkdir -p "${_counter_file%/*}" # dirname
 966 }
 967 ctdb_counter_init () {
 968     _ctdb_counter_common "$1"
 969
 970     >"$_counter_file"
 971 }
 972 ctdb_counter_incr () {
 973     _ctdb_counter_common "$1"
 974
 975     # unary counting!
 976     echo -n 1 >> "$_counter_file"
 977 }
 978 ctdb_check_counter () {
 979     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 980     _op="${2:--ge}"  # an integer operator supported by test
 981     _limit="${3:-${service_fail_limit}}"
 982     shift 3
 983     _ctdb_counter_common "$1"
 984
 985     # unary counting!
 986     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 987     _hit=false
 988     if [ "$_op" != "%" ] ; then
 989         if [ $_size $_op $_limit ] ; then
 990             _hit=true
 991         fi
 992     else
 993         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
 994             _hit=true
 995         fi
 996     fi
 997     if $_hit ; then
 998         if [ "$_msg" = "error" ] ; then
 999             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1000             exit 1
1001         else
1002             return 1
1003         fi
1004     fi
1005 }
1006
1007 ########################################################
1008
1009 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1010 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1011
1012 ctdb_setup_service_state_dir ()
1013 {
1014     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1015     mkdir -p "$service_state_dir" || {
1016         echo "Error creating state dir \"$service_state_dir\""
1017         exit 1
1018     }
1019 }
1020
1021 ########################################################
1022 # Managed status history, for auto-start/stop
1023
1024 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1025
1026 _ctdb_managed_common ()
1027 {
1028     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1029 }
1030
1031 ctdb_service_managed ()
1032 {
1033     _ctdb_managed_common
1034     mkdir -p "$ctdb_managed_dir"
1035     touch "$_ctdb_managed_file"
1036 }
1037
1038 ctdb_service_unmanaged ()
1039 {
1040     _ctdb_managed_common
1041     rm -f "$_ctdb_managed_file"
1042 }
1043
1044 is_ctdb_previously_managed_service ()
1045 {
1046     _ctdb_managed_common
1047     [ -f "$_ctdb_managed_file" ]
1048 }
1049
1050 ########################################################
1051 # Check and set status
1052
1053 log_status_cat ()
1054 {
1055     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1056 }
1057
1058 ctdb_checkstatus ()
1059 {
1060     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1061         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1062         return 1
1063     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1064         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1065         return 2
1066     else
1067         return 0
1068     fi
1069 }
1070
1071 ctdb_setstatus ()
1072 {
1073     d="$ctdb_status_dir/$script_name"
1074     case "$1" in
1075         unhealthy|banned)
1076             mkdir -p "$d"
1077             cat "$2" >"$d/$1"
1078             ;;
1079         *)
1080             for i in "banned" "unhealthy" ; do
1081                 rm -f "$d/$i"
1082             done
1083             ;;
1084     esac
1085 }
1086
1087 ##################################################################
1088 # Reconfigure a service on demand
1089
1090 _ctdb_service_reconfigure_common ()
1091 {
1092     _d="$ctdb_status_dir/${service_name}"
1093     mkdir -p "$_d"
1094     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1095 }
1096
1097 ctdb_service_needs_reconfigure ()
1098 {
1099     _ctdb_service_reconfigure_common
1100     [ -e "$_ctdb_service_reconfigure_flag" ]
1101 }
1102
1103 ctdb_service_set_reconfigure ()
1104 {
1105     _ctdb_service_reconfigure_common
1106     >"$_ctdb_service_reconfigure_flag"
1107 }
1108
1109 ctdb_service_unset_reconfigure ()
1110 {
1111     _ctdb_service_reconfigure_common
1112     rm -f "$_ctdb_service_reconfigure_flag"
1113 }
1114
1115 ctdb_service_reconfigure ()
1116 {
1117     echo "Reconfiguring service \"${service_name}\"..."
1118     ctdb_service_unset_reconfigure
1119     service_reconfigure || return $?
1120     ctdb_counter_init
1121 }
1122
1123 # Default service_reconfigure() function does nothing.
1124 service_reconfigure ()
1125 {
1126     :
1127 }
1128
1129 ctdb_reconfigure_take_lock ()
1130 {
1131     _ctdb_service_reconfigure_common
1132     _lock="${_d}/reconfigure_lock"
1133     mkdir -p "${_lock%/*}" # dirname
1134     touch "$_lock"
1135
1136     (
1137         flock 0
1138         # This is overkill but will work if we need to extend this to
1139         # allow certain events to run multiple times in parallel
1140         # (e.g. takeip) and write multiple PIDs to the file.
1141         read _locker_event
1142         if [ -n "$_locker_event" ] ; then
1143             while read _pid ; do
1144                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1145                     kill -0 "$_pid" 2>/dev/null ; then
1146                     exit 1
1147                 fi
1148             done
1149         fi
1150
1151         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1152         exit 0
1153     ) <"$_lock"
1154 }
1155
1156 ctdb_reconfigure_release_lock ()
1157 {
1158     _ctdb_service_reconfigure_common
1159     _lock="${_d}/reconfigure_lock"
1160
1161     rm -f "$_lock"
1162 }
1163
1164 ctdb_replay_monitor_status ()
1165 {
1166     echo "Replaying previous status for this script due to reconfigure..."
1167     # Leading separator ('|') is missing in some versions...
1168     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1169     # Output looks like this:
1170     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1171     # This is the cheapest way of getting fields in the middle.
1172     set -- $(IFS="|" ; echo $_out)
1173     _code="$3"
1174     _status="$4"
1175     # The error output field can include colons so we'll try to
1176     # preserve them.  The weak checking at the beginning tries to make
1177     # this work for both broken (no leading '|') and fixed output.
1178     _out="${_out%|}"
1179     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1180     case "$_status" in
1181         OK) : ;;  # Do nothing special.
1182         TIMEDOUT)
1183             # Recast this as an error, since we can't exit with the
1184             # correct negative number.
1185             _code=1
1186             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1187             ;;
1188         DISABLED)
1189             # Recast this as an OK, since we can't exit with the
1190             # correct negative number.
1191             _code=0
1192             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1193             ;;
1194         *) : ;;  # Must be ERROR, do nothing special.
1195     esac
1196     if [ -n "$_err_out" ] ; then
1197         echo "$_err_out"
1198     fi
1199     exit $_code
1200 }
1201
1202 ctdb_service_check_reconfigure ()
1203 {
1204     assert_service_name
1205
1206     # We only care about some events in this function.  For others we
1207     # return now.
1208     case "$event_name" in
1209         monitor|ipreallocated|reconfigure) : ;;
1210         *) return 0 ;;
1211     esac
1212
1213     if ctdb_reconfigure_take_lock ; then
1214         # No events covered by this function are running, so proceed
1215         # with gay abandon.
1216         case "$event_name" in
1217             reconfigure)
1218                 (ctdb_service_reconfigure)
1219                 exit $?
1220                 ;;
1221             ipreallocated)
1222                 if ctdb_service_needs_reconfigure ; then
1223                     ctdb_service_reconfigure
1224                 fi
1225                 ;;
1226         esac
1227
1228         ctdb_reconfigure_release_lock
1229     else
1230         # Somebody else is running an event we don't want to collide
1231         # with.  We proceed with caution.
1232         case "$event_name" in
1233             reconfigure)
1234                 # Tell whoever called us to retry.
1235                 exit 2
1236                 ;;
1237             ipreallocated)
1238                 # Defer any scheduled reconfigure and just run the
1239                 # rest of the ipreallocated event, as per the
1240                 # eventscript.  There's an assumption here that the
1241                 # event doesn't depend on any scheduled reconfigure.
1242                 # This is true in the current code.
1243                 return 0
1244                 ;;
1245             monitor)
1246                 # There is most likely a reconfigure in progress so
1247                 # the service is possibly unstable.  As above, we
1248                 # defer any scheduled reconfigured.  We also replay
1249                 # the previous monitor status since that's the best
1250                 # information we have.
1251                 ctdb_replay_monitor_status
1252                 ;;
1253         esac
1254     fi
1255 }
1256
1257 ##################################################################
1258 # Does CTDB manage this service? - and associated auto-start/stop
1259
1260 ctdb_compat_managed_service ()
1261 {
1262     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1263         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1264     fi
1265 }
1266
1267 is_ctdb_managed_service ()
1268 {
1269     assert_service_name
1270
1271     # $t is used just for readability and to allow better accurate
1272     # matching via leading/trailing spaces
1273     t=" $CTDB_MANAGED_SERVICES "
1274
1275     # Return 0 if "<space>$service_name<space>" appears in $t
1276     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1277         return 0
1278     fi
1279
1280     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1281     # backward compatibility and try again.
1282     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1283     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1284     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1285     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1286     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1287     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1288     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1289     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1290     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1291
1292     t=" $CTDB_MANAGED_SERVICES "
1293
1294     # Return 0 if "<space>$service_name<space>" appears in $t
1295     [ "${t#* ${service_name} }" != "${t}" ]
1296 }
1297
1298 ctdb_start_stop_service ()
1299 {
1300     assert_service_name
1301
1302     # Allow service-start/service-stop pseudo-events to start/stop
1303     # services when we're not auto-starting/stopping and we're not
1304     # monitoring.
1305     case "$event_name" in
1306         service-start)
1307             if is_ctdb_managed_service ; then
1308                 die 'service-start event not permitted when service is managed'
1309             fi
1310             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1311                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1312             fi
1313             ctdb_service_start
1314             exit $?
1315             ;;
1316         service-stop)
1317             if is_ctdb_managed_service ; then
1318                 die 'service-stop event not permitted when service is managed'
1319             fi
1320             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1321                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1322             fi
1323             ctdb_service_stop
1324             exit $?
1325             ;;
1326     esac
1327
1328     # Do nothing unless configured to...
1329     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1330
1331     [ "$event_name" = "monitor" ] || return 0
1332
1333     if is_ctdb_managed_service ; then
1334         if ! is_ctdb_previously_managed_service ; then
1335             echo "Starting service \"$service_name\" - now managed"
1336             background_with_logging ctdb_service_start
1337             exit $?
1338         fi
1339     else
1340         if is_ctdb_previously_managed_service ; then
1341             echo "Stopping service \"$service_name\" - no longer managed"
1342             background_with_logging ctdb_service_stop
1343             exit $?
1344         fi
1345     fi
1346 }
1347
1348 ctdb_service_start ()
1349 {
1350     # The service is marked managed if we've ever tried to start it.
1351     ctdb_service_managed
1352
1353     service_start || return $?
1354
1355     ctdb_counter_init
1356     ctdb_check_tcp_init
1357 }
1358
1359 ctdb_service_stop ()
1360 {
1361     ctdb_service_unmanaged
1362     service_stop
1363 }
1364
1365 # Default service_start() and service_stop() functions.
1366
1367 # These may be overridden in an eventscript.
1368 service_start ()
1369 {
1370     service "$service_name" start
1371 }
1372
1373 service_stop ()
1374 {
1375     service "$service_name" stop
1376 }
1377
1378 ##################################################################
1379
1380 ctdb_standard_event_handler ()
1381 {
1382     case "$1" in
1383         status)
1384             ctdb_checkstatus
1385             exit
1386             ;;
1387         setstatus)
1388             shift
1389             ctdb_setstatus "$@"
1390             exit
1391             ;;
1392     esac
1393 }
1394
1395 # iptables doesn't like being re-entered, so flock-wrap it.
1396 iptables ()
1397 {
1398         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1399 }
1400 ip6tables ()
1401 {
1402         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/ip6tables "$@"
1403 }
1404 iptables_wrapper ()
1405 {
1406     _family="$1" ; shift
1407     if [ "$_family" = "inet6" ] ; then
1408         ip6tables "$@"
1409     else
1410         iptables "$@"
1411     fi
1412 }
1413
1414 # AIX (and perhaps others?) doesn't have mktemp
1415 if ! which mktemp >/dev/null 2>&1 ; then
1416     mktemp ()
1417     {
1418         _dir=false
1419         if [ "$1" = "-d" ] ; then
1420             _dir=true
1421             shift
1422         fi
1423         _d="${TMPDIR:-/tmp}"
1424         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1425             md5sum | \
1426             sed -e 's@\(..........\).*@\1@')
1427         _t="${_d}/tmp.${_hex10}"
1428         (
1429             umask 077
1430             if $_dir ; then
1431                 mkdir "$_t"
1432             else
1433                 >"$_t"
1434             fi
1435         )
1436         echo "$_t"
1437     }
1438 fi
1439
1440 ########################################################
1441 # tickle handling
1442 ########################################################
1443
1444 update_tickles ()
1445 {
1446         _port="$1"
1447
1448         tickledir="$CTDB_VARDIR/state/tickles"
1449         mkdir -p "$tickledir"
1450
1451         # Who am I?
1452         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1453
1454         # What public IPs do I hold?
1455         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1456
1457         # IPs as a regexp choice
1458         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1459
1460         # Record connections to our public IPs in a temporary file
1461         _my_connections="${tickledir}/${_port}.connections"
1462         rm -f "$_my_connections"
1463         netstat -tn |
1464         awk -v destpat="^${_ipschoice}:${_port}\$" \
1465           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1466         sort >"$_my_connections"
1467
1468         # Record our current tickles in a temporary file
1469         _my_tickles="${tickledir}/${_port}.tickles"
1470         rm -f "$_my_tickles"
1471         for _i in $_ips ; do
1472                 ctdb -X gettickles $_i $_port |
1473                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1474         done |
1475         sort >"$_my_tickles"
1476
1477         # Add tickles for connections that we haven't already got tickles for
1478         comm -23 "$_my_connections" "$_my_tickles" |
1479         while read _src _dst ; do
1480                 ctdb addtickle $_src $_dst
1481         done
1482
1483         # Remove tickles for connections that are no longer there
1484         comm -13 "$_my_connections" "$_my_tickles" |
1485         while read _src _dst ; do
1486                 ctdb deltickle $_src $_dst
1487         done
1488
1489         rm -f "$_my_connections" "$_my_tickles"
1490 }
1491
1492 ########################################################
1493 # load a site local config file
1494 ########################################################
1495
1496 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1497         . "$CTDB_RC_LOCAL"
1498 }
1499
1500 [ -x $CTDB_BASE/rc.local ] && {
1501         . $CTDB_BASE/rc.local
1502 }
1503
1504 [ -d $CTDB_BASE/rc.local.d ] && {
1505         for i in $CTDB_BASE/rc.local.d/* ; do
1506                 [ -x "$i" ] && . "$i"
1507         done
1508 }
1509
1510 script_name="${0##*/}"       # basename
1511 service_fail_limit=1
1512 event_name="$1"