ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x /usr/sbin/service ]; then
 175       $_nice /usr/sbin/service "$_service_name" "$_op"
 176   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 178   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 179       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 180   fi
 181 }
 182
 183 service()
 184 {
 185     _nice=""
 186     _service "$@"
 187 }
 188
 189 ######################################################
 190 # simulate /sbin/service (niced) on platforms that don't have it
 191 nice_service()
 192 {
 193     _nice="nice"
 194     _service "$@"
 195 }
 196
 197 ######################################################
 198 # wrapper around /proc/ settings to allow them to be hooked
 199 # for testing
 200 # 1st arg is relative path under /proc/, 2nd arg is value to set
 201 set_proc ()
 202 {
 203     echo "$2" >"/proc/$1"
 204 }
 205
 206 ######################################################
 207 # wrapper around getting file contents from /proc/ to allow
 208 # this to be hooked for testing
 209 # 1st arg is relative path under /proc/
 210 get_proc ()
 211 {
 212     cat "/proc/$1"
 213 }
 214
 215 ######################################################
 216 # Print up to $_max kernel stack traces for processes named $_program
 217 program_stack_traces ()
 218 {
 219     _prog="$1"
 220     _max="${2:-1}"
 221
 222     _count=1
 223     for _pid in $(pidof "$_prog") ; do
 224         [ $_count -le $_max ] || break
 225
 226         # Do this first to avoid racing with process exit
 227         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 228         if [ -n "$_stack" ] ; then
 229             echo "Stack trace for ${_prog}[${_pid}]:"
 230             echo "$_stack"
 231             _count=$(($_count + 1))
 232         fi
 233     done
 234 }
 235
 236 ######################################################
 237 # Check that an RPC service is healthy -
 238 # this includes allowing a certain number of failures
 239 # before marking the NFS service unhealthy.
 240 #
 241 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 242 #
 243 # each triple is a set of 3 arguments: an operator, a
 244 # fail count limit and an action string.
 245 #
 246 # For example:
 247 #
 248 #       nfs_check_rpc_service "lockd" \
 249 #           -ge 15 "verbose restart unhealthy" \
 250 #           -eq 10 "restart:bs"
 251 #
 252 # says that if lockd is down for 15 iterations then do
 253 # a verbose restart of lockd and mark the node unhealthy.
 254 # Before this, after 10 iterations of failure, the
 255 # service is restarted silently in the background.
 256 # Order is important: the number of failures need to be
 257 # specified in reverse order because processing stops
 258 # after the first condition that is true.
 259 ######################################################
 260 nfs_check_rpc_service ()
 261 {
 262     _prog_name="$1" ; shift
 263
 264     if _nfs_check_rpc_common "$_prog_name" ; then
 265         return
 266     fi
 267
 268     while [ -n "$3" ] ; do
 269         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 270             break
 271         fi
 272         shift 3
 273     done
 274 }
 275
 276 # The new way of doing things...
 277 nfs_check_rpc_services ()
 278 {
 279     # Files must end with .check - avoids editor backups, RPM fu, ...
 280     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 281         _t="${_f%.check}"
 282         _prog_name="${_t##*/[0-9][0-9].}"
 283
 284         if _nfs_check_rpc_common "$_prog_name" ; then
 285             # This RPC service is up, check next service...
 286             continue
 287         fi
 288
 289         # Check each line in the file in turn until one of the limit
 290         # checks is hit...
 291         while read _cmp _lim _rest ; do
 292             # Skip comments
 293             case "$_cmp" in
 294                 \#*) continue ;;
 295             esac
 296
 297             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 298                 # Limit was hit on this line, no further checking...
 299                 break
 300             fi
 301         done <"$_f"
 302     done
 303 }
 304
 305 _nfs_check_rpc_common ()
 306 {
 307     _prog_name="$1"
 308
 309     # Some platforms don't have separate programs for all services.
 310     case "$_prog_name" in
 311         statd)
 312             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 313     esac
 314
 315     case "$_prog_name" in
 316         nfsd)
 317             _rpc_prog=nfs
 318             _version=3
 319             ;;
 320         mountd)
 321             _rpc_prog=mountd
 322             _version=1
 323             ;;
 324         rquotad)
 325             _rpc_prog=rquotad
 326             _version=1
 327             ;;
 328         lockd)
 329             _rpc_prog=nlockmgr
 330             _version=4
 331             ;;
 332         statd)
 333             _rpc_prog=status
 334             _version=1
 335             ;;
 336         *)
 337             echo "Internal error: unknown RPC program \"$_prog_name\"."
 338             exit 1
 339     esac
 340
 341     _service_name="nfs_${_prog_name}"
 342
 343     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 344         ctdb_counter_init "$_service_name"
 345         return 0
 346     fi
 347
 348     ctdb_counter_incr "$_service_name"
 349
 350     return 1
 351 }
 352
 353 _nfs_check_rpc_action ()
 354 {
 355     _cmp="$1"
 356     _limit="$2"
 357     _actions="$3"
 358
 359     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 360         return 1
 361     fi
 362
 363     for _action in $_actions ; do
 364         case "$_action" in
 365             verbose)
 366                 echo "$ctdb_check_rpc_out"
 367                 ;;
 368             restart)
 369                 _nfs_restart_rpc_service "$_prog_name"
 370                 ;;
 371             restart:b)
 372                 _nfs_restart_rpc_service "$_prog_name" true
 373                 ;;
 374             unhealthy)
 375                 exit 1
 376                 ;;
 377             *)
 378                 echo "Internal error: unknown action \"$_action\"."
 379                 exit 1
 380         esac
 381     done
 382
 383     return 0
 384 }
 385
 386 _nfs_restart_rpc_service ()
 387 {
 388     _prog_name="$1"
 389     _background="${2:-false}"
 390
 391     if $_background ; then
 392         _maybe_background="background_with_logging"
 393     else
 394         _maybe_background=""
 395     fi
 396
 397     _p="rpc.${_prog_name}"
 398
 399     case "$_prog_name" in
 400         nfsd)
 401             echo "Trying to restart NFS service"
 402             $_maybe_background startstop_nfs restart
 403             ;;
 404         mountd)
 405             echo "Trying to restart $_prog_name [${_p}]"
 406             killall -q -9 "$_p"
 407             nfs_dump_some_threads "$_p"
 408             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 409             ;;
 410         rquotad)
 411             echo "Trying to restart $_prog_name [${_p}]"
 412             killall -q -9 "$_p"
 413             nfs_dump_some_threads "$_p"
 414             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 415             ;;
 416         lockd)
 417             echo "Trying to restart lock manager service"
 418             $_maybe_background startstop_nfslock restart
 419             ;;
 420         statd)
 421             echo "Trying to restart $_prog_name [${_p}]"
 422             killall -q -9 "$_p"
 423             nfs_dump_some_threads "$_p"
 424             $_maybe_background $_p \
 425                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 426                 ${STATD_PORT:+-p} $STATD_PORT \
 427                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 428             ;;
 429         *)
 430             echo "Internal error: unknown RPC program \"$_prog_name\"."
 431             exit 1
 432     esac
 433 }
 434
 435 ######################################################
 436 # check that a rpc server is registered with portmap
 437 # and responding to requests
 438 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 439 ######################################################
 440 ctdb_check_rpc ()
 441 {
 442     progname="$1"
 443     version="$2"
 444
 445     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 446
 447     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 448         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 449 $ctdb_check_rpc_out"
 450         echo "$ctdb_check_rpc_out"
 451         return 1
 452     fi
 453 }
 454
 455 ######################################################
 456 # Ensure $service_name is set
 457 assert_service_name ()
 458 {
 459     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 460 }
 461
 462 ######################################################
 463 # check a set of directories is available
 464 # return 1 on a missing directory
 465 # directories are read from stdin
 466 ######################################################
 467 ctdb_check_directories_probe()
 468 {
 469     while IFS="" read d ; do
 470         case "$d" in
 471             *%*)
 472                 continue
 473                 ;;
 474             *)
 475                 [ -d "${d}/." ] || return 1
 476         esac
 477     done
 478 }
 479
 480 ######################################################
 481 # check a set of directories is available
 482 # directories are read from stdin
 483 ######################################################
 484 ctdb_check_directories()
 485 {
 486     ctdb_check_directories_probe || {
 487         echo "ERROR: $service_name directory \"$d\" not available"
 488         exit 1
 489     }
 490 }
 491
 492 ######################################################
 493 # check a set of tcp ports
 494 # usage: ctdb_check_tcp_ports <ports...>
 495 ######################################################
 496
 497 # This flag file is created when a service is initially started.  It
 498 # is deleted the first time TCP port checks for that service succeed.
 499 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 500 # message if a port check fails.
 501 _ctdb_check_tcp_common ()
 502 {
 503     assert_service_name
 504     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 505 }
 506
 507 ctdb_check_tcp_init ()
 508 {
 509     _ctdb_check_tcp_common
 510     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 511     touch "$_ctdb_service_started_file"
 512 }
 513
 514 # Check whether something is listening on all of the given TCP ports
 515 # using the "ctdb checktcpport" command.
 516 ctdb_check_tcp_ports()
 517 {
 518     if [ -z "$1" ] ; then
 519         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 520         exit 1
 521     fi
 522
 523     for _p ; do  # process each function argument (port)
 524         _cmd="ctdb checktcpport $_p"
 525         _out=$($_cmd 2>&1)
 526         _ret=$?
 527         case "$_ret" in
 528             0)
 529                 _ctdb_check_tcp_common
 530                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 531                     echo "ERROR: $service_name tcp port $_p is not responding"
 532                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 533                 else
 534                     echo "INFO: $service_name tcp port $_p is not responding"
 535                 fi
 536
 537                 return 1
 538                 ;;
 539             98)
 540                 # Couldn't bind, something already listening, next port...
 541                 continue
 542                 ;;
 543             *)
 544                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 545                 debug <<EOF
 546 ctdb checktcpport (exited with $_ret) with output:
 547 $_out"
 548 EOF
 549                 return $_ret
 550         esac
 551     done
 552
 553     # All ports listening
 554     _ctdb_check_tcp_common
 555     rm -f "$_ctdb_service_started_file"
 556     return 0
 557 }
 558
 559 ######################################################
 560 # check a unix socket
 561 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 562 ######################################################
 563 ctdb_check_unix_socket() {
 564     socket_path="$1"
 565     [ -z "$socket_path" ] && return
 566
 567     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 568         echo "ERROR: $service_name socket $socket_path not found"
 569         return 1
 570     fi
 571 }
 572
 573 ######################################################
 574 # check a command returns zero status
 575 # usage: ctdb_check_command <command>
 576 ######################################################
 577 ctdb_check_command ()
 578 {
 579     _out=$("$@" 2>&1) || {
 580         echo "ERROR: $* returned error"
 581         echo "$_out" | debug
 582         exit 1
 583     }
 584 }
 585
 586 ################################################
 587 # kill off any TCP connections with the given IP
 588 ################################################
 589 kill_tcp_connections ()
 590 {
 591     _ip="$1"
 592
 593     _oneway=false
 594     if [ "$2" = "oneway" ] ; then
 595         _oneway=true
 596     fi
 597
 598     get_tcp_connections_for_ip "$_ip" | {
 599         _killcount=0
 600         _connections=""
 601         _nl="
 602 "
 603         while read _dst _src; do
 604             _destport="${_dst##*:}"
 605             __oneway=$_oneway
 606             case $_destport in
 607                 # we only do one-way killtcp for CIFS
 608                 139|445) __oneway=true ;;
 609             esac
 610
 611             echo "Killing TCP connection $_src $_dst"
 612             _connections="${_connections}${_nl}${_src} ${_dst}"
 613             if ! $__oneway ; then
 614                 _connections="${_connections}${_nl}${_dst} ${_src}"
 615             fi
 616
 617             _killcount=$(($_killcount + 1))
 618         done
 619
 620         if [ $_killcount -eq 0 ] ; then
 621             return
 622         fi
 623
 624         echo "$_connections" | ctdb killtcp || {
 625             echo "Failed to send killtcp control"
 626             return
 627         }
 628
 629         _count=0
 630         while : ; do
 631             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 632
 633             if [ $_remaining -eq 0 ] ; then
 634                 echo "Killed $_killcount TCP connections to released IP $_ip"
 635                 return
 636             fi
 637
 638             _count=$(($_count + 1))
 639             if [ $_count -gt 3 ] ; then
 640                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 641                 return
 642             fi
 643
 644             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 645             sleep 1
 646         done
 647     }
 648 }
 649
 650 ##################################################################
 651 # kill off the local end for any TCP connections with the given IP
 652 ##################################################################
 653 kill_tcp_connections_local_only ()
 654 {
 655     kill_tcp_connections "$1" "oneway"
 656 }
 657
 658 ##################################################################
 659 # tickle any TCP connections with the given IP
 660 ##################################################################
 661 tickle_tcp_connections ()
 662 {
 663     _ip="$1"
 664
 665     get_tcp_connections_for_ip "$_ip" |
 666     {
 667         _failed=false
 668
 669         while read dest src; do
 670             echo "Tickle TCP connection $src $dest"
 671             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 672             echo "Tickle TCP connection $dest $src"
 673             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 674         done
 675
 676         if $_failed ; then
 677             echo "Failed to send tickle control"
 678         fi
 679     }
 680 }
 681
 682 get_tcp_connections_for_ip ()
 683 {
 684     _ip="$1"
 685
 686     netstat -tn | awk -v ip=$_ip \
 687         'index($1, "tcp") == 1 && \
 688          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 689          && $6 == "ESTABLISHED" \
 690          {print $4" "$5}'
 691 }
 692
 693 ########################################################
 694 # start/stop the Ganesha nfs service
 695 ########################################################
 696 startstop_ganesha()
 697 {
 698     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 699     case "$1" in
 700         start)
 701             service "$_service_name" start
 702             ;;
 703         stop)
 704             service "$_service_name" stop
 705             ;;
 706         restart)
 707             service "$_service_name" stop
 708             nfs_dump_some_threads "rpc.statd"
 709             service "$_service_name" start
 710             ;;
 711     esac
 712 }
 713
 714 ########################################################
 715 # start/stop the nfs service on different platforms
 716 ########################################################
 717 startstop_nfs() {
 718         PLATFORM="unknown"
 719         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 720                 PLATFORM="sles"
 721         }
 722         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 723             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 724                 PLATFORM="rhel"
 725         }
 726
 727         case $PLATFORM in
 728         sles)
 729                 case $1 in
 730                 start)
 731                         service nfsserver start
 732                         ;;
 733                 stop)
 734                         service nfsserver stop > /dev/null 2>&1
 735                         ;;
 736                 restart)
 737                         set_proc "fs/nfsd/threads" 0
 738                         service nfsserver stop > /dev/null 2>&1
 739                         pkill -9 nfsd
 740                         nfs_dump_some_threads
 741                         service nfsserver start
 742                         ;;
 743                 esac
 744                 ;;
 745         rhel)
 746                 case $1 in
 747                 start)
 748                         service nfslock start
 749                         service nfs start
 750                         ;;
 751                 stop)
 752                         service nfs stop
 753                         service nfslock stop
 754                         ;;
 755                 restart)
 756                         set_proc "fs/nfsd/threads" 0
 757                         service nfs stop > /dev/null 2>&1
 758                         service nfslock stop > /dev/null 2>&1
 759                         pkill -9 nfsd
 760                         nfs_dump_some_threads
 761                         service nfslock start
 762                         service nfs start
 763                         ;;
 764                 esac
 765                 ;;
 766         *)
 767                 echo "Unknown platform. NFS is not supported with ctdb"
 768                 exit 1
 769                 ;;
 770         esac
 771 }
 772
 773 # Dump up to the configured number of nfsd thread backtraces.
 774 nfs_dump_some_threads ()
 775 {
 776     _prog="${1:-nfsd}"
 777
 778     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 779     [ $_num -gt 0 ] || return 0
 780
 781     program_stack_traces "$_prog" $_num
 782 }
 783
 784 ########################################################
 785 # start/stop the nfs lockmanager service on different platforms
 786 ########################################################
 787 startstop_nfslock() {
 788         PLATFORM="unknown"
 789         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 790                 PLATFORM="sles"
 791         }
 792         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 793             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 794                 PLATFORM="rhel"
 795         }
 796
 797         case $PLATFORM in
 798         sles)
 799                 # for sles there is no service for lockmanager
 800                 # so we instead just shutdown/restart nfs
 801                 case $1 in
 802                 start)
 803                         service nfsserver start
 804                         ;;
 805                 stop)
 806                         service nfsserver stop > /dev/null 2>&1
 807                         ;;
 808                 restart)
 809                         service nfsserver stop > /dev/null 2>&1
 810                         service nfsserver start
 811                         ;;
 812                 esac
 813                 ;;
 814         rhel)
 815                 case $1 in
 816                 start)
 817                         service nfslock start
 818                         ;;
 819                 stop)
 820                         service nfslock stop > /dev/null 2>&1
 821                         ;;
 822                 restart)
 823                         service nfslock stop > /dev/null 2>&1
 824                         service nfslock start
 825                         ;;
 826                 esac
 827                 ;;
 828         *)
 829                 echo "Unknown platform. NFS locking is not supported with ctdb"
 830                 exit 1
 831                 ;;
 832         esac
 833 }
 834
 835 # Periodically update the statd database
 836 nfs_statd_update ()
 837 {
 838     _update_period="$1"
 839
 840     _statd_update_trigger="$service_state_dir/update-trigger"
 841     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 842
 843     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 844     _current_time=$(date +"%s")
 845     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 846         touch "$_statd_update_trigger"
 847         $CTDB_BASE/statd-callout updatelocal &
 848         $CTDB_BASE/statd-callout updateremote &
 849     fi
 850 }
 851
 852 ########################################################
 853
 854 add_ip_to_iface ()
 855 {
 856     _iface=$1
 857     _ip=$2
 858     _maskbits=$3
 859
 860     # Ensure interface is up
 861     ip link set "$_iface" up || \
 862         die "Failed to bringup interface $_iface"
 863
 864     # Only need to define broadcast for IPv4
 865     case "$ip" in
 866         *:*) _bcast=""      ;;
 867         *)   _bcast="brd +" ;;
 868     esac
 869
 870     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 871         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 872         return 1
 873     }
 874
 875     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 876     if [ -z "$_bcast" ] ; then
 877         for _x in $(seq 1 10) ; do
 878             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 879             sleep 0.5
 880         done
 881
 882         # If the address was a duplicate then it won't be on the
 883         # interface so flag an error.
 884         _t=$(ip addr show to "${_ip}/128")
 885         case "$_t" in
 886             "")
 887                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 888                 return 1
 889                 ;;
 890             *tentative*|*dadfailed*)
 891                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 892                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 893                 return 1
 894                 ;;
 895         esac
 896     fi
 897 }
 898
 899 delete_ip_from_iface()
 900 {
 901     _iface=$1
 902     _ip=$2
 903     _maskbits=$3
 904
 905     # This could be set globally for all interfaces but it is probably
 906     # better to avoid surprises, so limit it the interfaces where CTDB
 907     # has public IP addresses.  There isn't anywhere else convenient
 908     # to do this so just set it each time.  This is much cheaper than
 909     # remembering and re-adding secondaries.
 910     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 911
 912     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 913         echo "Failed to del $_ip on dev $_iface"
 914         return 1
 915     }
 916 }
 917
 918 # If the given IP is hosted then print 2 items: maskbits and iface
 919 ip_maskbits_iface ()
 920 {
 921     _addr="$1"
 922
 923     case "$_addr" in
 924         *:*) _family="inet6" ; _bits=128 ;;
 925         *)   _family="inet"  ; _bits=32  ;;
 926     esac
 927
 928     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 929         awk -v family="${_family}" \
 930             'NR == 1 { iface = $2; sub(":$", "", iface) } \
 931              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
 932                            print mask, iface, family }'
 933 }
 934
 935 drop_ip ()
 936 {
 937     _addr="${1%/*}"  # Remove optional maskbits
 938
 939     set -- $(ip_maskbits_iface $_addr)
 940     if [ -n "$1" ] ; then
 941         _maskbits="$1"
 942         _iface="$2"
 943         echo "Removing public address $_addr/$_maskbits from device $_iface"
 944         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 945     fi
 946 }
 947
 948 drop_all_public_ips ()
 949 {
 950     while read _ip _x ; do
 951         drop_ip "$_ip"
 952     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 953 }
 954
 955 flush_route_cache ()
 956 {
 957     set_proc sys/net/ipv4/route/flush 1
 958     set_proc sys/net/ipv6/route/flush 1
 959 }
 960
 961 ########################################################
 962 # Simple counters
 963 _ctdb_counter_common () {
 964     _service_name="${1:-${service_name:-${script_name}}}"
 965     _counter_file="$ctdb_fail_dir/$_service_name"
 966     mkdir -p "${_counter_file%/*}" # dirname
 967 }
 968 ctdb_counter_init () {
 969     _ctdb_counter_common "$1"
 970
 971     >"$_counter_file"
 972 }
 973 ctdb_counter_incr () {
 974     _ctdb_counter_common "$1"
 975
 976     # unary counting!
 977     echo -n 1 >> "$_counter_file"
 978 }
 979 ctdb_check_counter () {
 980     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 981     _op="${2:--ge}"  # an integer operator supported by test
 982     _limit="${3:-${service_fail_limit}}"
 983     shift 3
 984     _ctdb_counter_common "$1"
 985
 986     # unary counting!
 987     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 988     _hit=false
 989     if [ "$_op" != "%" ] ; then
 990         if [ $_size $_op $_limit ] ; then
 991             _hit=true
 992         fi
 993     else
 994         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
 995             _hit=true
 996         fi
 997     fi
 998     if $_hit ; then
 999         if [ "$_msg" = "error" ] ; then
1000             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1001             exit 1
1002         else
1003             return 1
1004         fi
1005     fi
1006 }
1007
1008 ########################################################
1009
1010 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1011 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1012
1013 ctdb_setup_service_state_dir ()
1014 {
1015     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1016     mkdir -p "$service_state_dir" || {
1017         echo "Error creating state dir \"$service_state_dir\""
1018         exit 1
1019     }
1020 }
1021
1022 ########################################################
1023 # Managed status history, for auto-start/stop
1024
1025 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1026
1027 _ctdb_managed_common ()
1028 {
1029     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1030 }
1031
1032 ctdb_service_managed ()
1033 {
1034     _ctdb_managed_common
1035     mkdir -p "$ctdb_managed_dir"
1036     touch "$_ctdb_managed_file"
1037 }
1038
1039 ctdb_service_unmanaged ()
1040 {
1041     _ctdb_managed_common
1042     rm -f "$_ctdb_managed_file"
1043 }
1044
1045 is_ctdb_previously_managed_service ()
1046 {
1047     _ctdb_managed_common
1048     [ -f "$_ctdb_managed_file" ]
1049 }
1050
1051 ########################################################
1052 # Check and set status
1053
1054 log_status_cat ()
1055 {
1056     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1057 }
1058
1059 ctdb_checkstatus ()
1060 {
1061     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1062         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1063         return 1
1064     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1065         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1066         return 2
1067     else
1068         return 0
1069     fi
1070 }
1071
1072 ctdb_setstatus ()
1073 {
1074     d="$ctdb_status_dir/$script_name"
1075     case "$1" in
1076         unhealthy|banned)
1077             mkdir -p "$d"
1078             cat "$2" >"$d/$1"
1079             ;;
1080         *)
1081             for i in "banned" "unhealthy" ; do
1082                 rm -f "$d/$i"
1083             done
1084             ;;
1085     esac
1086 }
1087
1088 ##################################################################
1089 # Reconfigure a service on demand
1090
1091 _ctdb_service_reconfigure_common ()
1092 {
1093     _d="$ctdb_status_dir/${service_name}"
1094     mkdir -p "$_d"
1095     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1096 }
1097
1098 ctdb_service_needs_reconfigure ()
1099 {
1100     _ctdb_service_reconfigure_common
1101     [ -e "$_ctdb_service_reconfigure_flag" ]
1102 }
1103
1104 ctdb_service_set_reconfigure ()
1105 {
1106     _ctdb_service_reconfigure_common
1107     >"$_ctdb_service_reconfigure_flag"
1108 }
1109
1110 ctdb_service_unset_reconfigure ()
1111 {
1112     _ctdb_service_reconfigure_common
1113     rm -f "$_ctdb_service_reconfigure_flag"
1114 }
1115
1116 ctdb_service_reconfigure ()
1117 {
1118     echo "Reconfiguring service \"${service_name}\"..."
1119     ctdb_service_unset_reconfigure
1120     service_reconfigure || return $?
1121     ctdb_counter_init
1122 }
1123
1124 # Default service_reconfigure() function does nothing.
1125 service_reconfigure ()
1126 {
1127     :
1128 }
1129
1130 ctdb_reconfigure_take_lock ()
1131 {
1132     _ctdb_service_reconfigure_common
1133     _lock="${_d}/reconfigure_lock"
1134     mkdir -p "${_lock%/*}" # dirname
1135     touch "$_lock"
1136
1137     (
1138         flock 0
1139         # This is overkill but will work if we need to extend this to
1140         # allow certain events to run multiple times in parallel
1141         # (e.g. takeip) and write multiple PIDs to the file.
1142         read _locker_event
1143         if [ -n "$_locker_event" ] ; then
1144             while read _pid ; do
1145                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1146                     kill -0 "$_pid" 2>/dev/null ; then
1147                     exit 1
1148                 fi
1149             done
1150         fi
1151
1152         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1153         exit 0
1154     ) <"$_lock"
1155 }
1156
1157 ctdb_reconfigure_release_lock ()
1158 {
1159     _ctdb_service_reconfigure_common
1160     _lock="${_d}/reconfigure_lock"
1161
1162     rm -f "$_lock"
1163 }
1164
1165 ctdb_replay_monitor_status ()
1166 {
1167     echo "Replaying previous status for this script due to reconfigure..."
1168     # Leading separator ('|') is missing in some versions...
1169     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1170     # Output looks like this:
1171     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1172     # This is the cheapest way of getting fields in the middle.
1173     set -- $(IFS="|" ; echo $_out)
1174     _code="$3"
1175     _status="$4"
1176     # The error output field can include colons so we'll try to
1177     # preserve them.  The weak checking at the beginning tries to make
1178     # this work for both broken (no leading '|') and fixed output.
1179     _out="${_out%|}"
1180     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1181     case "$_status" in
1182         OK) : ;;  # Do nothing special.
1183         TIMEDOUT)
1184             # Recast this as an error, since we can't exit with the
1185             # correct negative number.
1186             _code=1
1187             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1188             ;;
1189         DISABLED)
1190             # Recast this as an OK, since we can't exit with the
1191             # correct negative number.
1192             _code=0
1193             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1194             ;;
1195         *) : ;;  # Must be ERROR, do nothing special.
1196     esac
1197     if [ -n "$_err_out" ] ; then
1198         echo "$_err_out"
1199     fi
1200     exit $_code
1201 }
1202
1203 ctdb_service_check_reconfigure ()
1204 {
1205     assert_service_name
1206
1207     # We only care about some events in this function.  For others we
1208     # return now.
1209     case "$event_name" in
1210         monitor|ipreallocated|reconfigure) : ;;
1211         *) return 0 ;;
1212     esac
1213
1214     if ctdb_reconfigure_take_lock ; then
1215         # No events covered by this function are running, so proceed
1216         # with gay abandon.
1217         case "$event_name" in
1218             reconfigure)
1219                 (ctdb_service_reconfigure)
1220                 exit $?
1221                 ;;
1222             ipreallocated)
1223                 if ctdb_service_needs_reconfigure ; then
1224                     ctdb_service_reconfigure
1225                 fi
1226                 ;;
1227         esac
1228
1229         ctdb_reconfigure_release_lock
1230     else
1231         # Somebody else is running an event we don't want to collide
1232         # with.  We proceed with caution.
1233         case "$event_name" in
1234             reconfigure)
1235                 # Tell whoever called us to retry.
1236                 exit 2
1237                 ;;
1238             ipreallocated)
1239                 # Defer any scheduled reconfigure and just run the
1240                 # rest of the ipreallocated event, as per the
1241                 # eventscript.  There's an assumption here that the
1242                 # event doesn't depend on any scheduled reconfigure.
1243                 # This is true in the current code.
1244                 return 0
1245                 ;;
1246             monitor)
1247                 # There is most likely a reconfigure in progress so
1248                 # the service is possibly unstable.  As above, we
1249                 # defer any scheduled reconfigured.  We also replay
1250                 # the previous monitor status since that's the best
1251                 # information we have.
1252                 ctdb_replay_monitor_status
1253                 ;;
1254         esac
1255     fi
1256 }
1257
1258 ##################################################################
1259 # Does CTDB manage this service? - and associated auto-start/stop
1260
1261 ctdb_compat_managed_service ()
1262 {
1263     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1264         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1265     fi
1266 }
1267
1268 is_ctdb_managed_service ()
1269 {
1270     assert_service_name
1271
1272     # $t is used just for readability and to allow better accurate
1273     # matching via leading/trailing spaces
1274     t=" $CTDB_MANAGED_SERVICES "
1275
1276     # Return 0 if "<space>$service_name<space>" appears in $t
1277     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1278         return 0
1279     fi
1280
1281     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1282     # backward compatibility and try again.
1283     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1284     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1285     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1286     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1287     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1288     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1289     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1290     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1291     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1292
1293     t=" $CTDB_MANAGED_SERVICES "
1294
1295     # Return 0 if "<space>$service_name<space>" appears in $t
1296     [ "${t#* ${service_name} }" != "${t}" ]
1297 }
1298
1299 ctdb_start_stop_service ()
1300 {
1301     assert_service_name
1302
1303     # Allow service-start/service-stop pseudo-events to start/stop
1304     # services when we're not auto-starting/stopping and we're not
1305     # monitoring.
1306     case "$event_name" in
1307         service-start)
1308             if is_ctdb_managed_service ; then
1309                 die 'service-start event not permitted when service is managed'
1310             fi
1311             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1312                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1313             fi
1314             ctdb_service_start
1315             exit $?
1316             ;;
1317         service-stop)
1318             if is_ctdb_managed_service ; then
1319                 die 'service-stop event not permitted when service is managed'
1320             fi
1321             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1322                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1323             fi
1324             ctdb_service_stop
1325             exit $?
1326             ;;
1327     esac
1328
1329     # Do nothing unless configured to...
1330     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1331
1332     [ "$event_name" = "monitor" ] || return 0
1333
1334     if is_ctdb_managed_service ; then
1335         if ! is_ctdb_previously_managed_service ; then
1336             echo "Starting service \"$service_name\" - now managed"
1337             background_with_logging ctdb_service_start
1338             exit $?
1339         fi
1340     else
1341         if is_ctdb_previously_managed_service ; then
1342             echo "Stopping service \"$service_name\" - no longer managed"
1343             background_with_logging ctdb_service_stop
1344             exit $?
1345         fi
1346     fi
1347 }
1348
1349 ctdb_service_start ()
1350 {
1351     # The service is marked managed if we've ever tried to start it.
1352     ctdb_service_managed
1353
1354     service_start || return $?
1355
1356     ctdb_counter_init
1357     ctdb_check_tcp_init
1358 }
1359
1360 ctdb_service_stop ()
1361 {
1362     ctdb_service_unmanaged
1363     service_stop
1364 }
1365
1366 # Default service_start() and service_stop() functions.
1367
1368 # These may be overridden in an eventscript.
1369 service_start ()
1370 {
1371     service "$service_name" start
1372 }
1373
1374 service_stop ()
1375 {
1376     service "$service_name" stop
1377 }
1378
1379 ##################################################################
1380
1381 ctdb_standard_event_handler ()
1382 {
1383     case "$1" in
1384         status)
1385             ctdb_checkstatus
1386             exit
1387             ;;
1388         setstatus)
1389             shift
1390             ctdb_setstatus "$@"
1391             exit
1392             ;;
1393     esac
1394 }
1395
1396 iptables_wrapper ()
1397 {
1398     _family="$1" ; shift
1399     if [ "$_family" = "inet6" ] ; then
1400         _iptables_cmd="ip6tables"
1401     else
1402         _iptables_cmd="iptables"
1403     fi
1404
1405     # iptables doesn't like being re-entered, so flock-wrap it.
1406     flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1407 }
1408
1409 # AIX (and perhaps others?) doesn't have mktemp
1410 if ! which mktemp >/dev/null 2>&1 ; then
1411     mktemp ()
1412     {
1413         _dir=false
1414         if [ "$1" = "-d" ] ; then
1415             _dir=true
1416             shift
1417         fi
1418         _d="${TMPDIR:-/tmp}"
1419         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1420             md5sum | \
1421             sed -e 's@\(..........\).*@\1@')
1422         _t="${_d}/tmp.${_hex10}"
1423         (
1424             umask 077
1425             if $_dir ; then
1426                 mkdir "$_t"
1427             else
1428                 >"$_t"
1429             fi
1430         )
1431         echo "$_t"
1432     }
1433 fi
1434
1435 ########################################################
1436 # tickle handling
1437 ########################################################
1438
1439 update_tickles ()
1440 {
1441         _port="$1"
1442
1443         tickledir="$CTDB_VARDIR/state/tickles"
1444         mkdir -p "$tickledir"
1445
1446         # Who am I?
1447         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1448
1449         # What public IPs do I hold?
1450         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1451
1452         # IPs as a regexp choice
1453         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1454
1455         # Record connections to our public IPs in a temporary file
1456         _my_connections="${tickledir}/${_port}.connections"
1457         rm -f "$_my_connections"
1458         netstat -tn |
1459         awk -v destpat="^${_ipschoice}:${_port}\$" \
1460           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1461         sort >"$_my_connections"
1462
1463         # Record our current tickles in a temporary file
1464         _my_tickles="${tickledir}/${_port}.tickles"
1465         rm -f "$_my_tickles"
1466         for _i in $_ips ; do
1467                 ctdb -X gettickles $_i $_port |
1468                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1469         done |
1470         sort >"$_my_tickles"
1471
1472         # Add tickles for connections that we haven't already got tickles for
1473         comm -23 "$_my_connections" "$_my_tickles" |
1474         while read _src _dst ; do
1475                 ctdb addtickle $_src $_dst
1476         done
1477
1478         # Remove tickles for connections that are no longer there
1479         comm -13 "$_my_connections" "$_my_tickles" |
1480         while read _src _dst ; do
1481                 ctdb deltickle $_src $_dst
1482         done
1483
1484         rm -f "$_my_connections" "$_my_tickles"
1485 }
1486
1487 ########################################################
1488 # load a site local config file
1489 ########################################################
1490
1491 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1492         . "$CTDB_RC_LOCAL"
1493 }
1494
1495 [ -x $CTDB_BASE/rc.local ] && {
1496         . $CTDB_BASE/rc.local
1497 }
1498
1499 [ -d $CTDB_BASE/rc.local.d ] && {
1500         for i in $CTDB_BASE/rc.local.d/* ; do
1501                 [ -x "$i" ] && . "$i"
1502         done
1503 }
1504
1505 script_name="${0##*/}"       # basename
1506 service_fail_limit=1
1507 event_name="$1"