ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         elif ! tty -s ; then
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     fi
  72 }
  73
  74 die ()
  75 {
  76     _msg="$1"
  77     _rc="${2:-1}"
  78
  79     echo "$_msg"
  80     exit $_rc
  81 }
  82
  83 # Log given message or stdin to either syslog or a CTDB log file
  84 # $1 is the tag passed to logger if syslog is in use.
  85 script_log ()
  86 {
  87     _tag="$1" ; shift
  88
  89     case "$CTDB_LOGGING" in
  90         file:*|"")
  91             if [ -n "$CTDB_LOGGING" ] ; then
  92                 _file="${CTDB_LOGGING#file:}"
  93             else
  94                 _file="/var/log/log.ctdb"
  95             fi
  96             {
  97                 if [ -n "$*" ] ; then
  98                     echo "$*"
  99                 else
 100                     cat
 101                 fi
 102             } >>"$_file"
 103             ;;
 104         *)
 105             # Handle all syslog:* variants here too.  There's no tool to do
 106             # the lossy things, so just use logger.
 107             logger -t "ctdbd: ${_tag}" $*
 108             ;;
 109     esac
 110 }
 111
 112 # When things are run in the background in an eventscript then logging
 113 # output might get lost.  This is the "solution".  :-)
 114 background_with_logging ()
 115 {
 116     (
 117         "$@" 2>&1 </dev/null |
 118         script_log "${script_name}&"
 119     )&
 120
 121     return 0
 122 }
 123
 124 ##############################################################
 125 # check number of args for different events
 126 ctdb_check_args ()
 127 {
 128     case "$1" in
 129         takeip|releaseip)
 130             if [ $# != 4 ]; then
 131                 echo "ERROR: must supply interface, IP and maskbits"
 132                 exit 1
 133             fi
 134             ;;
 135         updateip)
 136             if [ $# != 5 ]; then
 137                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 138                 exit 1
 139             fi
 140             ;;
 141     esac
 142 }
 143
 144 ##############################################################
 145 # determine on what type of system (init style) we are running
 146 detect_init_style()
 147 {
 148     # only do detection if not already set:
 149     [ -z "$CTDB_INIT_STYLE" ] || return
 150
 151     if [ -x /sbin/startproc ]; then
 152         CTDB_INIT_STYLE="suse"
 153     elif [ -x /sbin/start-stop-daemon ]; then
 154         CTDB_INIT_STYLE="debian"
 155     else
 156         CTDB_INIT_STYLE="redhat"
 157     fi
 158 }
 159
 160 ######################################################
 161 # simulate /sbin/service on platforms that don't have it
 162 # _service() makes it easier to hook the service() function for
 163 # testing.
 164 _service ()
 165 {
 166   _service_name="$1"
 167   _op="$2"
 168
 169   # do nothing, when no service was specified
 170   [ -z "$_service_name" ] && return
 171
 172   if [ -x /sbin/service ]; then
 173       $_nice /sbin/service "$_service_name" "$_op"
 174   elif [ -x /usr/sbin/service ]; then
 175       $_nice /usr/sbin/service "$_service_name" "$_op"
 176   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 177       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 178   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 179       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 180   fi
 181 }
 182
 183 service()
 184 {
 185     _nice=""
 186     _service "$@"
 187 }
 188
 189 ######################################################
 190 # simulate /sbin/service (niced) on platforms that don't have it
 191 nice_service()
 192 {
 193     _nice="nice"
 194     _service "$@"
 195 }
 196
 197 ######################################################
 198 # wrapper around /proc/ settings to allow them to be hooked
 199 # for testing
 200 # 1st arg is relative path under /proc/, 2nd arg is value to set
 201 set_proc ()
 202 {
 203     echo "$2" >"/proc/$1"
 204 }
 205
 206 ######################################################
 207 # wrapper around getting file contents from /proc/ to allow
 208 # this to be hooked for testing
 209 # 1st arg is relative path under /proc/
 210 get_proc ()
 211 {
 212     cat "/proc/$1"
 213 }
 214
 215 ######################################################
 216 # Print up to $_max kernel stack traces for processes named $_program
 217 program_stack_traces ()
 218 {
 219     _prog="$1"
 220     _max="${2:-1}"
 221
 222     _count=1
 223     for _pid in $(pidof "$_prog") ; do
 224         [ $_count -le $_max ] || break
 225
 226         # Do this first to avoid racing with process exit
 227         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 228         if [ -n "$_stack" ] ; then
 229             echo "Stack trace for ${_prog}[${_pid}]:"
 230             echo "$_stack"
 231             _count=$(($_count + 1))
 232         fi
 233     done
 234 }
 235
 236 ######################################################
 237 # Check that an RPC service is healthy -
 238 # this includes allowing a certain number of failures
 239 # before marking the NFS service unhealthy.
 240 #
 241 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 242 #
 243 # each triple is a set of 3 arguments: an operator, a
 244 # fail count limit and an action string.
 245 #
 246 # For example:
 247 #
 248 #       nfs_check_rpc_service "lockd" \
 249 #           -ge 15 "verbose restart unhealthy" \
 250 #           -eq 10 "restart:bs"
 251 #
 252 # says that if lockd is down for 15 iterations then do
 253 # a verbose restart of lockd and mark the node unhealthy.
 254 # Before this, after 10 iterations of failure, the
 255 # service is restarted silently in the background.
 256 # Order is important: the number of failures need to be
 257 # specified in reverse order because processing stops
 258 # after the first condition that is true.
 259 ######################################################
 260 nfs_check_rpc_service ()
 261 {
 262     _prog_name="$1" ; shift
 263
 264     if _nfs_check_rpc_common "$_prog_name" ; then
 265         return
 266     fi
 267
 268     while [ -n "$3" ] ; do
 269         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 270             break
 271         fi
 272         shift 3
 273     done
 274 }
 275
 276 # The new way of doing things...
 277 nfs_check_rpc_services ()
 278 {
 279     # Files must end with .check - avoids editor backups, RPM fu, ...
 280     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 281         _t="${_f%.check}"
 282         _prog_name="${_t##*/[0-9][0-9].}"
 283
 284         if _nfs_check_rpc_common "$_prog_name" ; then
 285             # This RPC service is up, check next service...
 286             continue
 287         fi
 288
 289         # Check each line in the file in turn until one of the limit
 290         # checks is hit...
 291         while read _cmp _lim _rest ; do
 292             # Skip comments
 293             case "$_cmp" in
 294                 \#*) continue ;;
 295             esac
 296
 297             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 298                 # Limit was hit on this line, no further checking...
 299                 break
 300             fi
 301         done <"$_f"
 302     done
 303 }
 304
 305 _nfs_check_rpc_common ()
 306 {
 307     _prog_name="$1"
 308
 309     # Some platforms don't have separate programs for all services.
 310     case "$_prog_name" in
 311         statd)
 312             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 313     esac
 314
 315     case "$_prog_name" in
 316         nfsd)
 317             _rpc_prog=nfs
 318             _version=3
 319             ;;
 320         mountd)
 321             _rpc_prog=mountd
 322             _version=1
 323             ;;
 324         rquotad)
 325             _rpc_prog=rquotad
 326             _version=1
 327             ;;
 328         lockd)
 329             _rpc_prog=nlockmgr
 330             _version=4
 331             ;;
 332         statd)
 333             _rpc_prog=status
 334             _version=1
 335             ;;
 336         *)
 337             echo "Internal error: unknown RPC program \"$_prog_name\"."
 338             exit 1
 339     esac
 340
 341     _service_name="nfs_${_prog_name}"
 342
 343     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 344         ctdb_counter_init "$_service_name"
 345         return 0
 346     fi
 347
 348     ctdb_counter_incr "$_service_name"
 349
 350     return 1
 351 }
 352
 353 _nfs_check_rpc_action ()
 354 {
 355     _cmp="$1"
 356     _limit="$2"
 357     _actions="$3"
 358
 359     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 360         return 1
 361     fi
 362
 363     for _action in $_actions ; do
 364         case "$_action" in
 365             verbose)
 366                 echo "$ctdb_check_rpc_out"
 367                 ;;
 368             restart)
 369                 _nfs_restart_rpc_service "$_prog_name"
 370                 ;;
 371             restart:b)
 372                 _nfs_restart_rpc_service "$_prog_name" true
 373                 ;;
 374             unhealthy)
 375                 exit 1
 376                 ;;
 377             *)
 378                 echo "Internal error: unknown action \"$_action\"."
 379                 exit 1
 380         esac
 381     done
 382
 383     return 0
 384 }
 385
 386 _nfs_restart_rpc_service ()
 387 {
 388     _prog_name="$1"
 389     _background="${2:-false}"
 390
 391     if $_background ; then
 392         _maybe_background="background_with_logging"
 393     else
 394         _maybe_background=""
 395     fi
 396
 397     _p="rpc.${_prog_name}"
 398
 399     case "$_prog_name" in
 400         nfsd)
 401             echo "Trying to restart NFS service"
 402             $_maybe_background startstop_nfs restart
 403             ;;
 404         mountd)
 405             echo "Trying to restart $_prog_name [${_p}]"
 406             killall -q -9 "$_p"
 407             nfs_dump_some_threads "$_p"
 408             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 409             ;;
 410         rquotad)
 411             echo "Trying to restart $_prog_name [${_p}]"
 412             killall -q -9 "$_p"
 413             nfs_dump_some_threads "$_p"
 414             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 415             ;;
 416         lockd)
 417             echo "Trying to restart lock manager service"
 418             $_maybe_background startstop_nfslock restart
 419             ;;
 420         statd)
 421             echo "Trying to restart $_prog_name [${_p}]"
 422             killall -q -9 "$_p"
 423             nfs_dump_some_threads "$_p"
 424             $_maybe_background $_p \
 425                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 426                 ${STATD_PORT:+-p} $STATD_PORT \
 427                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 428             ;;
 429         *)
 430             echo "Internal error: unknown RPC program \"$_prog_name\"."
 431             exit 1
 432     esac
 433 }
 434
 435 ######################################################
 436 # check that a rpc server is registered with portmap
 437 # and responding to requests
 438 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 439 ######################################################
 440 ctdb_check_rpc ()
 441 {
 442     progname="$1"
 443     version="$2"
 444
 445     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 446
 447     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
 448         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 449 $ctdb_check_rpc_out"
 450         echo "$ctdb_check_rpc_out"
 451         return 1
 452     fi
 453 }
 454
 455 ######################################################
 456 # Ensure $service_name is set
 457 assert_service_name ()
 458 {
 459     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 460 }
 461
 462 ######################################################
 463 # check a set of directories is available
 464 # return 1 on a missing directory
 465 # directories are read from stdin
 466 ######################################################
 467 ctdb_check_directories_probe()
 468 {
 469     while IFS="" read d ; do
 470         case "$d" in
 471             *%*)
 472                 continue
 473                 ;;
 474             *)
 475                 [ -d "${d}/." ] || return 1
 476         esac
 477     done
 478 }
 479
 480 ######################################################
 481 # check a set of directories is available
 482 # directories are read from stdin
 483 ######################################################
 484 ctdb_check_directories()
 485 {
 486     ctdb_check_directories_probe || {
 487         echo "ERROR: $service_name directory \"$d\" not available"
 488         exit 1
 489     }
 490 }
 491
 492 ######################################################
 493 # check a set of tcp ports
 494 # usage: ctdb_check_tcp_ports <ports...>
 495 ######################################################
 496
 497 # This flag file is created when a service is initially started.  It
 498 # is deleted the first time TCP port checks for that service succeed.
 499 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 500 # message if a port check fails.
 501 _ctdb_check_tcp_common ()
 502 {
 503     assert_service_name
 504     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 505 }
 506
 507 ctdb_check_tcp_init ()
 508 {
 509     _ctdb_check_tcp_common
 510     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 511     touch "$_ctdb_service_started_file"
 512 }
 513
 514 # Check whether something is listening on all of the given TCP ports
 515 # using the "ctdb checktcpport" command.
 516 ctdb_check_tcp_ports()
 517 {
 518     if [ -z "$1" ] ; then
 519         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 520         exit 1
 521     fi
 522
 523     for _p ; do  # process each function argument (port)
 524         _cmd="ctdb checktcpport $_p"
 525         _out=$($_cmd 2>&1)
 526         _ret=$?
 527         case "$_ret" in
 528             0)
 529                 _ctdb_check_tcp_common
 530                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 531                     echo "ERROR: $service_name tcp port $_p is not responding"
 532                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 533                 else
 534                     echo "INFO: $service_name tcp port $_p is not responding"
 535                 fi
 536
 537                 return 1
 538                 ;;
 539             98)
 540                 # Couldn't bind, something already listening, next port...
 541                 continue
 542                 ;;
 543             *)
 544                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 545                 debug <<EOF
 546 ctdb checktcpport (exited with $_ret) with output:
 547 $_out"
 548 EOF
 549                 return $_ret
 550         esac
 551     done
 552
 553     # All ports listening
 554     _ctdb_check_tcp_common
 555     rm -f "$_ctdb_service_started_file"
 556     return 0
 557 }
 558
 559 ######################################################
 560 # check a unix socket
 561 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 562 ######################################################
 563 ctdb_check_unix_socket() {
 564     socket_path="$1"
 565     [ -z "$socket_path" ] && return
 566
 567     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 568         echo "ERROR: $service_name socket $socket_path not found"
 569         return 1
 570     fi
 571 }
 572
 573 ######################################################
 574 # check a command returns zero status
 575 # usage: ctdb_check_command <command>
 576 ######################################################
 577 ctdb_check_command ()
 578 {
 579     _out=$("$@" 2>&1) || {
 580         echo "ERROR: $* returned error"
 581         echo "$_out" | debug
 582         exit 1
 583     }
 584 }
 585
 586 ################################################
 587 # kill off any TCP connections with the given IP
 588 ################################################
 589 kill_tcp_connections ()
 590 {
 591     _ip="$1"
 592
 593     _oneway=false
 594     if [ "$2" = "oneway" ] ; then
 595         _oneway=true
 596     fi
 597
 598     get_tcp_connections_for_ip "$_ip" | {
 599         _killcount=0
 600         _connections=""
 601         _nl="
 602 "
 603         while read _dst _src; do
 604             _destport="${_dst##*:}"
 605             __oneway=$_oneway
 606             case $_destport in
 607                 # we only do one-way killtcp for CIFS
 608                 139|445) __oneway=true ;;
 609             esac
 610
 611             echo "Killing TCP connection $_src $_dst"
 612             _connections="${_connections}${_nl}${_src} ${_dst}"
 613             if ! $__oneway ; then
 614                 _connections="${_connections}${_nl}${_dst} ${_src}"
 615             fi
 616
 617             _killcount=$(($_killcount + 1))
 618         done
 619
 620         if [ $_killcount -eq 0 ] ; then
 621             return
 622         fi
 623
 624         echo "$_connections" | ctdb killtcp || {
 625             echo "Failed to send killtcp control"
 626             return
 627         }
 628
 629         _count=0
 630         while : ; do
 631             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 632
 633             if [ $_remaining -eq 0 ] ; then
 634                 echo "Killed $_killcount TCP connections to released IP $_ip"
 635                 return
 636             fi
 637
 638             _count=$(($_count + 1))
 639             if [ $_count -gt 3 ] ; then
 640                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 641                 return
 642             fi
 643
 644             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 645             sleep 1
 646         done
 647     }
 648 }
 649
 650 ##################################################################
 651 # kill off the local end for any TCP connections with the given IP
 652 ##################################################################
 653 kill_tcp_connections_local_only ()
 654 {
 655     kill_tcp_connections "$1" "oneway"
 656 }
 657
 658 ##################################################################
 659 # tickle any TCP connections with the given IP
 660 ##################################################################
 661 tickle_tcp_connections ()
 662 {
 663     _ip="$1"
 664
 665     get_tcp_connections_for_ip "$_ip" |
 666     {
 667         _failed=false
 668
 669         while read dest src; do
 670             echo "Tickle TCP connection $src $dest"
 671             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 672             echo "Tickle TCP connection $dest $src"
 673             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 674         done
 675
 676         if $_failed ; then
 677             echo "Failed to send tickle control"
 678         fi
 679     }
 680 }
 681
 682 get_tcp_connections_for_ip ()
 683 {
 684     _ip="$1"
 685
 686     netstat -tn | awk -v ip=$_ip \
 687         'index($1, "tcp") == 1 && \
 688          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 689          && $6 == "ESTABLISHED" \
 690          {print $4" "$5}'
 691 }
 692
 693 ##################################################################
 694 # use statd-callout to update NFS lock info
 695 ##################################################################
 696 nfs_update_lock_info ()
 697 {
 698     if [ -x "$CTDB_BASE/statd-callout" ] ; then
 699         "$CTDB_BASE/statd-callout" update
 700     fi
 701 }
 702
 703 ########################################################
 704 # start/stop the Ganesha nfs service
 705 ########################################################
 706 startstop_ganesha()
 707 {
 708     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 709     case "$1" in
 710         start)
 711             service "$_service_name" start
 712             ;;
 713         stop)
 714             service "$_service_name" stop
 715             ;;
 716         restart)
 717             service "$_service_name" stop
 718             nfs_dump_some_threads "rpc.statd"
 719             service "$_service_name" start
 720             ;;
 721     esac
 722 }
 723
 724 ########################################################
 725 # start/stop the nfs service on different platforms
 726 ########################################################
 727 startstop_nfs() {
 728         PLATFORM="unknown"
 729         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 730                 PLATFORM="sles"
 731         }
 732         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 733             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 734                 PLATFORM="rhel"
 735         }
 736
 737         case $PLATFORM in
 738         sles)
 739                 case $1 in
 740                 start)
 741                         service nfsserver start
 742                         ;;
 743                 stop)
 744                         service nfsserver stop > /dev/null 2>&1
 745                         ;;
 746                 restart)
 747                         set_proc "fs/nfsd/threads" 0
 748                         service nfsserver stop > /dev/null 2>&1
 749                         pkill -9 nfsd
 750                         nfs_dump_some_threads
 751                         service nfsserver start
 752                         ;;
 753                 esac
 754                 ;;
 755         rhel)
 756                 case $1 in
 757                 start)
 758                         service nfslock start
 759                         service nfs start
 760                         ;;
 761                 stop)
 762                         service nfs stop
 763                         service nfslock stop
 764                         ;;
 765                 restart)
 766                         set_proc "fs/nfsd/threads" 0
 767                         service nfs stop > /dev/null 2>&1
 768                         service nfslock stop > /dev/null 2>&1
 769                         pkill -9 nfsd
 770                         nfs_dump_some_threads
 771                         service nfslock start
 772                         service nfs start
 773                         ;;
 774                 esac
 775                 ;;
 776         *)
 777                 echo "Unknown platform. NFS is not supported with ctdb"
 778                 exit 1
 779                 ;;
 780         esac
 781 }
 782
 783 # Dump up to the configured number of nfsd thread backtraces.
 784 nfs_dump_some_threads ()
 785 {
 786     _prog="${1:-nfsd}"
 787
 788     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 789     [ $_num -gt 0 ] || return 0
 790
 791     program_stack_traces "$_prog" $_num
 792 }
 793
 794 ########################################################
 795 # start/stop the nfs lockmanager service on different platforms
 796 ########################################################
 797 startstop_nfslock() {
 798         PLATFORM="unknown"
 799         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 800                 PLATFORM="sles"
 801         }
 802         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 803             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 804                 PLATFORM="rhel"
 805         }
 806
 807         case $PLATFORM in
 808         sles)
 809                 # for sles there is no service for lockmanager
 810                 # so we instead just shutdown/restart nfs
 811                 case $1 in
 812                 start)
 813                         service nfsserver start
 814                         ;;
 815                 stop)
 816                         service nfsserver stop > /dev/null 2>&1
 817                         ;;
 818                 restart)
 819                         service nfsserver stop > /dev/null 2>&1
 820                         service nfsserver start
 821                         ;;
 822                 esac
 823                 ;;
 824         rhel)
 825                 case $1 in
 826                 start)
 827                         service nfslock start
 828                         ;;
 829                 stop)
 830                         service nfslock stop > /dev/null 2>&1
 831                         ;;
 832                 restart)
 833                         service nfslock stop > /dev/null 2>&1
 834                         service nfslock start
 835                         ;;
 836                 esac
 837                 ;;
 838         *)
 839                 echo "Unknown platform. NFS locking is not supported with ctdb"
 840                 exit 1
 841                 ;;
 842         esac
 843 }
 844
 845 # Periodically update the statd database
 846 nfs_statd_update ()
 847 {
 848     _update_period="$1"
 849
 850     _statd_update_trigger="$service_state_dir/update-trigger"
 851     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
 852
 853     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
 854     _current_time=$(date +"%s")
 855     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
 856         touch "$_statd_update_trigger"
 857         $CTDB_BASE/statd-callout updatelocal &
 858         $CTDB_BASE/statd-callout updateremote &
 859     fi
 860 }
 861
 862 ########################################################
 863
 864 add_ip_to_iface ()
 865 {
 866     _iface=$1
 867     _ip=$2
 868     _maskbits=$3
 869
 870     # Ensure interface is up
 871     ip link set "$_iface" up || \
 872         die "Failed to bringup interface $_iface"
 873
 874     # Only need to define broadcast for IPv4
 875     case "$ip" in
 876         *:*) _bcast=""      ;;
 877         *)   _bcast="brd +" ;;
 878     esac
 879
 880     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 881         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 882         return 1
 883     }
 884
 885     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 886     if [ -z "$_bcast" ] ; then
 887         for _x in $(seq 1 10) ; do
 888             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 889             sleep 0.5
 890         done
 891
 892         # If the address was a duplicate then it won't be on the
 893         # interface so flag an error.
 894         _t=$(ip addr show to "${_ip}/128")
 895         case "$_t" in
 896             "")
 897                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 898                 return 1
 899                 ;;
 900             *tentative*|*dadfailed*)
 901                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 902                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 903                 return 1
 904                 ;;
 905         esac
 906     fi
 907 }
 908
 909 delete_ip_from_iface()
 910 {
 911     _iface=$1
 912     _ip=$2
 913     _maskbits=$3
 914
 915     # This could be set globally for all interfaces but it is probably
 916     # better to avoid surprises, so limit it the interfaces where CTDB
 917     # has public IP addresses.  There isn't anywhere else convenient
 918     # to do this so just set it each time.  This is much cheaper than
 919     # remembering and re-adding secondaries.
 920     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 921
 922     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 923         echo "Failed to del $_ip on dev $_iface"
 924         return 1
 925     }
 926 }
 927
 928 # If the given IP is hosted then print 2 items: maskbits and iface
 929 ip_maskbits_iface ()
 930 {
 931     _addr="$1"
 932
 933     case "$_addr" in
 934         *:*) _family="inet6" ; _bits=128 ;;
 935         *)   _family="inet"  ; _bits=32  ;;
 936     esac
 937
 938     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 939         awk -v family="${_family}" \
 940             'NR == 1 { iface = $2; sub(":$", "", iface) } \
 941              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
 942                            print mask, iface, family }'
 943 }
 944
 945 drop_ip ()
 946 {
 947     _addr="${1%/*}"  # Remove optional maskbits
 948
 949     set -- $(ip_maskbits_iface $_addr)
 950     if [ -n "$1" ] ; then
 951         _maskbits="$1"
 952         _iface="$2"
 953         echo "Removing public address $_addr/$_maskbits from device $_iface"
 954         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 955     fi
 956 }
 957
 958 drop_all_public_ips ()
 959 {
 960     while read _ip _x ; do
 961         drop_ip "$_ip"
 962     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 963 }
 964
 965 flush_route_cache ()
 966 {
 967     set_proc sys/net/ipv4/route/flush 1
 968     set_proc sys/net/ipv6/route/flush 1
 969 }
 970
 971 ########################################################
 972 # Simple counters
 973 _ctdb_counter_common () {
 974     _service_name="${1:-${service_name:-${script_name}}}"
 975     _counter_file="$ctdb_fail_dir/$_service_name"
 976     mkdir -p "${_counter_file%/*}" # dirname
 977 }
 978 ctdb_counter_init () {
 979     _ctdb_counter_common "$1"
 980
 981     >"$_counter_file"
 982 }
 983 ctdb_counter_incr () {
 984     _ctdb_counter_common "$1"
 985
 986     # unary counting!
 987     echo -n 1 >> "$_counter_file"
 988 }
 989 ctdb_check_counter () {
 990     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 991     _op="${2:--ge}"  # an integer operator supported by test
 992     _limit="${3:-${service_fail_limit}}"
 993     shift 3
 994     _ctdb_counter_common "$1"
 995
 996     # unary counting!
 997     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 998     _hit=false
 999     if [ "$_op" != "%" ] ; then
1000         if [ $_size $_op $_limit ] ; then
1001             _hit=true
1002         fi
1003     else
1004         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1005             _hit=true
1006         fi
1007     fi
1008     if $_hit ; then
1009         if [ "$_msg" = "error" ] ; then
1010             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1011             exit 1
1012         else
1013             return 1
1014         fi
1015     fi
1016 }
1017
1018 ########################################################
1019
1020 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1021 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1022
1023 ctdb_setup_service_state_dir ()
1024 {
1025     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1026     mkdir -p "$service_state_dir" || {
1027         echo "Error creating state dir \"$service_state_dir\""
1028         exit 1
1029     }
1030 }
1031
1032 ########################################################
1033 # Managed status history, for auto-start/stop
1034
1035 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1036
1037 _ctdb_managed_common ()
1038 {
1039     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1040 }
1041
1042 ctdb_service_managed ()
1043 {
1044     _ctdb_managed_common
1045     mkdir -p "$ctdb_managed_dir"
1046     touch "$_ctdb_managed_file"
1047 }
1048
1049 ctdb_service_unmanaged ()
1050 {
1051     _ctdb_managed_common
1052     rm -f "$_ctdb_managed_file"
1053 }
1054
1055 is_ctdb_previously_managed_service ()
1056 {
1057     _ctdb_managed_common
1058     [ -f "$_ctdb_managed_file" ]
1059 }
1060
1061 ########################################################
1062 # Check and set status
1063
1064 log_status_cat ()
1065 {
1066     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1067 }
1068
1069 ctdb_checkstatus ()
1070 {
1071     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1072         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1073         return 1
1074     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1075         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1076         return 2
1077     else
1078         return 0
1079     fi
1080 }
1081
1082 ctdb_setstatus ()
1083 {
1084     d="$ctdb_status_dir/$script_name"
1085     case "$1" in
1086         unhealthy|banned)
1087             mkdir -p "$d"
1088             cat "$2" >"$d/$1"
1089             ;;
1090         *)
1091             for i in "banned" "unhealthy" ; do
1092                 rm -f "$d/$i"
1093             done
1094             ;;
1095     esac
1096 }
1097
1098 ##################################################################
1099 # Reconfigure a service on demand
1100
1101 _ctdb_service_reconfigure_common ()
1102 {
1103     _d="$ctdb_status_dir/${service_name}"
1104     mkdir -p "$_d"
1105     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1106 }
1107
1108 ctdb_service_needs_reconfigure ()
1109 {
1110     _ctdb_service_reconfigure_common
1111     [ -e "$_ctdb_service_reconfigure_flag" ]
1112 }
1113
1114 ctdb_service_set_reconfigure ()
1115 {
1116     _ctdb_service_reconfigure_common
1117     >"$_ctdb_service_reconfigure_flag"
1118 }
1119
1120 ctdb_service_unset_reconfigure ()
1121 {
1122     _ctdb_service_reconfigure_common
1123     rm -f "$_ctdb_service_reconfigure_flag"
1124 }
1125
1126 ctdb_service_reconfigure ()
1127 {
1128     echo "Reconfiguring service \"${service_name}\"..."
1129     ctdb_service_unset_reconfigure
1130     service_reconfigure || return $?
1131     ctdb_counter_init
1132 }
1133
1134 # Default service_reconfigure() function does nothing.
1135 service_reconfigure ()
1136 {
1137     :
1138 }
1139
1140 ctdb_reconfigure_take_lock ()
1141 {
1142     _ctdb_service_reconfigure_common
1143     _lock="${_d}/reconfigure_lock"
1144     mkdir -p "${_lock%/*}" # dirname
1145     touch "$_lock"
1146
1147     (
1148         flock 0
1149         # This is overkill but will work if we need to extend this to
1150         # allow certain events to run multiple times in parallel
1151         # (e.g. takeip) and write multiple PIDs to the file.
1152         read _locker_event
1153         if [ -n "$_locker_event" ] ; then
1154             while read _pid ; do
1155                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1156                     kill -0 "$_pid" 2>/dev/null ; then
1157                     exit 1
1158                 fi
1159             done
1160         fi
1161
1162         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1163         exit 0
1164     ) <"$_lock"
1165 }
1166
1167 ctdb_reconfigure_release_lock ()
1168 {
1169     _ctdb_service_reconfigure_common
1170     _lock="${_d}/reconfigure_lock"
1171
1172     rm -f "$_lock"
1173 }
1174
1175 ctdb_replay_monitor_status ()
1176 {
1177     echo "Replaying previous status for this script due to reconfigure..."
1178     # Leading separator ('|') is missing in some versions...
1179     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1180     # Output looks like this:
1181     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1182     # This is the cheapest way of getting fields in the middle.
1183     set -- $(IFS="|" ; echo $_out)
1184     _code="$3"
1185     _status="$4"
1186     # The error output field can include colons so we'll try to
1187     # preserve them.  The weak checking at the beginning tries to make
1188     # this work for both broken (no leading '|') and fixed output.
1189     _out="${_out%|}"
1190     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1191     case "$_status" in
1192         OK) : ;;  # Do nothing special.
1193         TIMEDOUT)
1194             # Recast this as an error, since we can't exit with the
1195             # correct negative number.
1196             _code=1
1197             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1198             ;;
1199         DISABLED)
1200             # Recast this as an OK, since we can't exit with the
1201             # correct negative number.
1202             _code=0
1203             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1204             ;;
1205         *) : ;;  # Must be ERROR, do nothing special.
1206     esac
1207     if [ -n "$_err_out" ] ; then
1208         echo "$_err_out"
1209     fi
1210     exit $_code
1211 }
1212
1213 ctdb_service_check_reconfigure ()
1214 {
1215     assert_service_name
1216
1217     # We only care about some events in this function.  For others we
1218     # return now.
1219     case "$event_name" in
1220         monitor|ipreallocated|reconfigure) : ;;
1221         *) return 0 ;;
1222     esac
1223
1224     if ctdb_reconfigure_take_lock ; then
1225         # No events covered by this function are running, so proceed
1226         # with gay abandon.
1227         case "$event_name" in
1228             reconfigure)
1229                 (ctdb_service_reconfigure)
1230                 exit $?
1231                 ;;
1232             ipreallocated)
1233                 if ctdb_service_needs_reconfigure ; then
1234                     ctdb_service_reconfigure
1235                 fi
1236                 ;;
1237         esac
1238
1239         ctdb_reconfigure_release_lock
1240     else
1241         # Somebody else is running an event we don't want to collide
1242         # with.  We proceed with caution.
1243         case "$event_name" in
1244             reconfigure)
1245                 # Tell whoever called us to retry.
1246                 exit 2
1247                 ;;
1248             ipreallocated)
1249                 # Defer any scheduled reconfigure and just run the
1250                 # rest of the ipreallocated event, as per the
1251                 # eventscript.  There's an assumption here that the
1252                 # event doesn't depend on any scheduled reconfigure.
1253                 # This is true in the current code.
1254                 return 0
1255                 ;;
1256             monitor)
1257                 # There is most likely a reconfigure in progress so
1258                 # the service is possibly unstable.  As above, we
1259                 # defer any scheduled reconfigured.  We also replay
1260                 # the previous monitor status since that's the best
1261                 # information we have.
1262                 ctdb_replay_monitor_status
1263                 ;;
1264         esac
1265     fi
1266 }
1267
1268 ##################################################################
1269 # Does CTDB manage this service? - and associated auto-start/stop
1270
1271 ctdb_compat_managed_service ()
1272 {
1273     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1274         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1275     fi
1276 }
1277
1278 is_ctdb_managed_service ()
1279 {
1280     assert_service_name
1281
1282     # $t is used just for readability and to allow better accurate
1283     # matching via leading/trailing spaces
1284     t=" $CTDB_MANAGED_SERVICES "
1285
1286     # Return 0 if "<space>$service_name<space>" appears in $t
1287     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1288         return 0
1289     fi
1290
1291     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1292     # backward compatibility and try again.
1293     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1294     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1295     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1296     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1297     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1298     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1299     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1300     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1301     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1302
1303     t=" $CTDB_MANAGED_SERVICES "
1304
1305     # Return 0 if "<space>$service_name<space>" appears in $t
1306     [ "${t#* ${service_name} }" != "${t}" ]
1307 }
1308
1309 ctdb_start_stop_service ()
1310 {
1311     assert_service_name
1312
1313     # Allow service-start/service-stop pseudo-events to start/stop
1314     # services when we're not auto-starting/stopping and we're not
1315     # monitoring.
1316     case "$event_name" in
1317         service-start)
1318             if is_ctdb_managed_service ; then
1319                 die 'service-start event not permitted when service is managed'
1320             fi
1321             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1322                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1323             fi
1324             ctdb_service_start
1325             exit $?
1326             ;;
1327         service-stop)
1328             if is_ctdb_managed_service ; then
1329                 die 'service-stop event not permitted when service is managed'
1330             fi
1331             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1332                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1333             fi
1334             ctdb_service_stop
1335             exit $?
1336             ;;
1337     esac
1338
1339     # Do nothing unless configured to...
1340     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1341
1342     [ "$event_name" = "monitor" ] || return 0
1343
1344     if is_ctdb_managed_service ; then
1345         if ! is_ctdb_previously_managed_service ; then
1346             echo "Starting service \"$service_name\" - now managed"
1347             background_with_logging ctdb_service_start
1348             exit $?
1349         fi
1350     else
1351         if is_ctdb_previously_managed_service ; then
1352             echo "Stopping service \"$service_name\" - no longer managed"
1353             background_with_logging ctdb_service_stop
1354             exit $?
1355         fi
1356     fi
1357 }
1358
1359 ctdb_service_start ()
1360 {
1361     # The service is marked managed if we've ever tried to start it.
1362     ctdb_service_managed
1363
1364     service_start || return $?
1365
1366     ctdb_counter_init
1367     ctdb_check_tcp_init
1368 }
1369
1370 ctdb_service_stop ()
1371 {
1372     ctdb_service_unmanaged
1373     service_stop
1374 }
1375
1376 # Default service_start() and service_stop() functions.
1377
1378 # These may be overridden in an eventscript.
1379 service_start ()
1380 {
1381     service "$service_name" start
1382 }
1383
1384 service_stop ()
1385 {
1386     service "$service_name" stop
1387 }
1388
1389 ##################################################################
1390
1391 ctdb_standard_event_handler ()
1392 {
1393     case "$1" in
1394         status)
1395             ctdb_checkstatus
1396             exit
1397             ;;
1398         setstatus)
1399             shift
1400             ctdb_setstatus "$@"
1401             exit
1402             ;;
1403     esac
1404 }
1405
1406 iptables_wrapper ()
1407 {
1408     _family="$1" ; shift
1409     if [ "$_family" = "inet6" ] ; then
1410         _iptables_cmd="ip6tables"
1411     else
1412         _iptables_cmd="iptables"
1413     fi
1414
1415     # iptables doesn't like being re-entered, so flock-wrap it.
1416     flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1417 }
1418
1419 # AIX (and perhaps others?) doesn't have mktemp
1420 if ! which mktemp >/dev/null 2>&1 ; then
1421     mktemp ()
1422     {
1423         _dir=false
1424         if [ "$1" = "-d" ] ; then
1425             _dir=true
1426             shift
1427         fi
1428         _d="${TMPDIR:-/tmp}"
1429         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1430             md5sum | \
1431             sed -e 's@\(..........\).*@\1@')
1432         _t="${_d}/tmp.${_hex10}"
1433         (
1434             umask 077
1435             if $_dir ; then
1436                 mkdir "$_t"
1437             else
1438                 >"$_t"
1439             fi
1440         )
1441         echo "$_t"
1442     }
1443 fi
1444
1445 ########################################################
1446 # tickle handling
1447 ########################################################
1448
1449 update_tickles ()
1450 {
1451         _port="$1"
1452
1453         tickledir="$CTDB_VARDIR/state/tickles"
1454         mkdir -p "$tickledir"
1455
1456         # Who am I?
1457         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1458
1459         # What public IPs do I hold?
1460         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1461
1462         # IPs as a regexp choice
1463         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1464
1465         # Record connections to our public IPs in a temporary file
1466         _my_connections="${tickledir}/${_port}.connections"
1467         rm -f "$_my_connections"
1468         netstat -tn |
1469         awk -v destpat="^${_ipschoice}:${_port}\$" \
1470           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1471         sort >"$_my_connections"
1472
1473         # Record our current tickles in a temporary file
1474         _my_tickles="${tickledir}/${_port}.tickles"
1475         rm -f "$_my_tickles"
1476         for _i in $_ips ; do
1477                 ctdb -X gettickles $_i $_port |
1478                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1479         done |
1480         sort >"$_my_tickles"
1481
1482         # Add tickles for connections that we haven't already got tickles for
1483         comm -23 "$_my_connections" "$_my_tickles" |
1484         while read _src _dst ; do
1485                 ctdb addtickle $_src $_dst
1486         done
1487
1488         # Remove tickles for connections that are no longer there
1489         comm -13 "$_my_connections" "$_my_tickles" |
1490         while read _src _dst ; do
1491                 ctdb deltickle $_src $_dst
1492         done
1493
1494         rm -f "$_my_connections" "$_my_tickles"
1495 }
1496
1497 ########################################################
1498 # load a site local config file
1499 ########################################################
1500
1501 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1502         . "$CTDB_RC_LOCAL"
1503 }
1504
1505 [ -x $CTDB_BASE/rc.local ] && {
1506         . $CTDB_BASE/rc.local
1507 }
1508
1509 [ -d $CTDB_BASE/rc.local.d ] && {
1510         for i in $CTDB_BASE/rc.local.d/* ; do
1511                 [ -x "$i" ] && . "$i"
1512         done
1513 }
1514
1515 script_name="${0##*/}"       # basename
1516 service_fail_limit=1
1517 event_name="$1"