ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         else
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     else
  72         if [ -z "$1" ] ; then
  73             cat >/dev/null
  74         fi
  75     fi
  76 }
  77
  78 die ()
  79 {
  80     _msg="$1"
  81     _rc="${2:-1}"
  82
  83     echo "$_msg"
  84     exit $_rc
  85 }
  86
  87 # Log given message or stdin to either syslog or a CTDB log file
  88 # $1 is the tag passed to logger if syslog is in use.
  89 script_log ()
  90 {
  91     _tag="$1" ; shift
  92
  93     case "$CTDB_LOGGING" in
  94         file:*|"")
  95             if [ -n "$CTDB_LOGGING" ] ; then
  96                 _file="${CTDB_LOGGING#file:}"
  97             else
  98                 _file="/var/log/log.ctdb"
  99             fi
 100             {
 101                 if [ -n "$*" ] ; then
 102                     echo "$*"
 103                 else
 104                     cat
 105                 fi
 106             } >>"$_file"
 107             ;;
 108         *)
 109             # Handle all syslog:* variants here too.  There's no tool to do
 110             # the lossy things, so just use logger.
 111             logger -t "ctdbd: ${_tag}" $*
 112             ;;
 113     esac
 114 }
 115
 116 # When things are run in the background in an eventscript then logging
 117 # output might get lost.  This is the "solution".  :-)
 118 background_with_logging ()
 119 {
 120     (
 121         "$@" 2>&1 </dev/null |
 122         script_log "${script_name}&"
 123     )&
 124
 125     return 0
 126 }
 127
 128 ##############################################################
 129 # check number of args for different events
 130 ctdb_check_args ()
 131 {
 132     case "$1" in
 133         takeip|releaseip)
 134             if [ $# != 4 ]; then
 135                 echo "ERROR: must supply interface, IP and maskbits"
 136                 exit 1
 137             fi
 138             ;;
 139         updateip)
 140             if [ $# != 5 ]; then
 141                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 142                 exit 1
 143             fi
 144             ;;
 145     esac
 146 }
 147
 148 ##############################################################
 149 # determine on what type of system (init style) we are running
 150 detect_init_style()
 151 {
 152     # only do detection if not already set:
 153     [ -z "$CTDB_INIT_STYLE" ] || return
 154
 155     if [ -x /sbin/startproc ]; then
 156         CTDB_INIT_STYLE="suse"
 157     elif [ -x /sbin/start-stop-daemon ]; then
 158         CTDB_INIT_STYLE="debian"
 159     else
 160         CTDB_INIT_STYLE="redhat"
 161     fi
 162 }
 163
 164 ######################################################
 165 # simulate /sbin/service on platforms that don't have it
 166 # _service() makes it easier to hook the service() function for
 167 # testing.
 168 _service ()
 169 {
 170   _service_name="$1"
 171   _op="$2"
 172
 173   # do nothing, when no service was specified
 174   [ -z "$_service_name" ] && return
 175
 176   if [ -x /sbin/service ]; then
 177       $_nice /sbin/service "$_service_name" "$_op"
 178   elif [ -x /usr/sbin/service ]; then
 179       $_nice /usr/sbin/service "$_service_name" "$_op"
 180   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 181       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 182   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 183       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 184   fi
 185 }
 186
 187 service()
 188 {
 189     _nice=""
 190     _service "$@"
 191 }
 192
 193 ######################################################
 194 # simulate /sbin/service (niced) on platforms that don't have it
 195 nice_service()
 196 {
 197     _nice="nice"
 198     _service "$@"
 199 }
 200
 201 ######################################################
 202 # Cached retrieval of PNN from local node.  This never changes so why
 203 # open a client connection to the server each time this is needed?
 204 # This sets $pnn - this avoid an unnecessary subprocess.
 205 ctdb_get_pnn ()
 206 {
 207     _pnn_file="$CTDB_VARDIR/state/my-pnn"
 208     if [ ! -f "$_pnn_file" ] ; then
 209         ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
 210     fi
 211
 212     read pnn <"$_pnn_file"
 213 }
 214
 215 ######################################################
 216 # wrapper around /proc/ settings to allow them to be hooked
 217 # for testing
 218 # 1st arg is relative path under /proc/, 2nd arg is value to set
 219 set_proc ()
 220 {
 221     echo "$2" >"/proc/$1"
 222 }
 223
 224 ######################################################
 225 # wrapper around getting file contents from /proc/ to allow
 226 # this to be hooked for testing
 227 # 1st arg is relative path under /proc/
 228 get_proc ()
 229 {
 230     cat "/proc/$1"
 231 }
 232
 233 ######################################################
 234 # Print up to $_max kernel stack traces for processes named $_program
 235 program_stack_traces ()
 236 {
 237     _prog="$1"
 238     _max="${2:-1}"
 239
 240     _count=1
 241     for _pid in $(pidof "$_prog") ; do
 242         [ $_count -le $_max ] || break
 243
 244         # Do this first to avoid racing with process exit
 245         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 246         if [ -n "$_stack" ] ; then
 247             echo "Stack trace for ${_prog}[${_pid}]:"
 248             echo "$_stack"
 249             _count=$(($_count + 1))
 250         fi
 251     done
 252 }
 253
 254 ######################################################
 255 # Check that an RPC service is healthy -
 256 # this includes allowing a certain number of failures
 257 # before marking the NFS service unhealthy.
 258 #
 259 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 260 #
 261 # each triple is a set of 3 arguments: an operator, a
 262 # fail count limit and an action string.
 263 #
 264 # For example:
 265 #
 266 #       nfs_check_rpc_service "lockd" \
 267 #           -ge 15 "verbose restart unhealthy" \
 268 #           -eq 10 "restart:bs"
 269 #
 270 # says that if lockd is down for 15 iterations then do
 271 # a verbose restart of lockd and mark the node unhealthy.
 272 # Before this, after 10 iterations of failure, the
 273 # service is restarted silently in the background.
 274 # Order is important: the number of failures need to be
 275 # specified in reverse order because processing stops
 276 # after the first condition that is true.
 277 ######################################################
 278 nfs_check_rpc_service ()
 279 {
 280     _prog_name="$1" ; shift
 281
 282     if _nfs_check_rpc_common "$_prog_name" ; then
 283         return
 284     fi
 285
 286     while [ -n "$3" ] ; do
 287         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 288             break
 289         fi
 290         shift 3
 291     done
 292 }
 293
 294 # The new way of doing things...
 295 nfs_check_rpc_services ()
 296 {
 297     # Files must end with .check - avoids editor backups, RPM fu, ...
 298     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 299         _t="${_f%.check}"
 300         _prog_name="${_t##*/[0-9][0-9].}"
 301
 302         if _nfs_check_rpc_common "$_prog_name" ; then
 303             # This RPC service is up, check next service...
 304             continue
 305         fi
 306
 307         # Check each line in the file in turn until one of the limit
 308         # checks is hit...
 309         while read _cmp _lim _rest ; do
 310             # Skip comments
 311             case "$_cmp" in
 312                 \#*) continue ;;
 313             esac
 314
 315             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 316                 # Limit was hit on this line, no further checking...
 317                 break
 318             fi
 319         done <"$_f"
 320     done
 321 }
 322
 323 _nfs_check_rpc_common ()
 324 {
 325     _prog_name="$1"
 326
 327     # Some platforms don't have separate programs for all services.
 328     case "$_prog_name" in
 329         statd)
 330             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 331     esac
 332
 333     case "$_prog_name" in
 334         nfsd)
 335             _rpc_prog=nfs
 336             _version=3
 337             ;;
 338         mountd)
 339             _rpc_prog=mountd
 340             _version=1
 341             ;;
 342         rquotad)
 343             _rpc_prog=rquotad
 344             _version=1
 345             ;;
 346         lockd)
 347             _rpc_prog=nlockmgr
 348             _version=4
 349             ;;
 350         statd)
 351             _rpc_prog=status
 352             _version=1
 353             ;;
 354         *)
 355             echo "Internal error: unknown RPC program \"$_prog_name\"."
 356             exit 1
 357     esac
 358
 359     _service_name="nfs_${_prog_name}"
 360
 361     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
 362         ctdb_counter_init "$_service_name"
 363         return 0
 364     fi
 365
 366     ctdb_counter_incr "$_service_name"
 367
 368     return 1
 369 }
 370
 371 _nfs_check_rpc_action ()
 372 {
 373     _cmp="$1"
 374     _limit="$2"
 375     _actions="$3"
 376
 377     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 378         return 1
 379     fi
 380
 381     for _action in $_actions ; do
 382         case "$_action" in
 383             verbose)
 384                 echo "$ctdb_check_rpc_out"
 385                 ;;
 386             restart)
 387                 _nfs_restart_rpc_service "$_prog_name"
 388                 ;;
 389             restart:b)
 390                 _nfs_restart_rpc_service "$_prog_name" true
 391                 ;;
 392             unhealthy)
 393                 exit 1
 394                 ;;
 395             *)
 396                 echo "Internal error: unknown action \"$_action\"."
 397                 exit 1
 398         esac
 399     done
 400
 401     return 0
 402 }
 403
 404 _nfs_restart_rpc_service ()
 405 {
 406     _prog_name="$1"
 407     _background="${2:-false}"
 408
 409     if $_background ; then
 410         _maybe_background="background_with_logging"
 411     else
 412         _maybe_background=""
 413     fi
 414
 415     _p="rpc.${_prog_name}"
 416
 417     case "$_prog_name" in
 418         nfsd)
 419             echo "Trying to restart NFS service"
 420             $_maybe_background startstop_nfs restart
 421             ;;
 422         mountd)
 423             echo "Trying to restart $_prog_name [${_p}]"
 424             killall -q -9 "$_p"
 425             nfs_dump_some_threads "$_p"
 426             $_maybe_background $_p $RPCMOUNTDOPTS \
 427                                ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 428             ;;
 429         rquotad)
 430             echo "Trying to restart $_prog_name [${_p}]"
 431             killall -q -9 "$_p"
 432             nfs_dump_some_threads "$_p"
 433             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 434             ;;
 435         lockd)
 436             echo "Trying to restart lock manager service"
 437             $_maybe_background startstop_nfslock restart
 438             ;;
 439         statd)
 440             echo "Trying to restart $_prog_name [${_p}]"
 441             killall -q -9 "$_p"
 442             nfs_dump_some_threads "$_p"
 443             $_maybe_background $_p \
 444                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 445                 ${STATD_PORT:+-p} $STATD_PORT \
 446                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 447             ;;
 448         *)
 449             echo "Internal error: unknown RPC program \"$_prog_name\"."
 450             exit 1
 451     esac
 452 }
 453
 454 ######################################################
 455 # check that a rpc server is registered with portmap
 456 # and responding to requests
 457 # usage: ctdb_check_rpc SERVICE_NAME VERSION
 458 ######################################################
 459 ctdb_check_rpc ()
 460 {
 461     progname="$1"
 462     version="$2"
 463
 464     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 465
 466     if ! ctdb_check_rpc_out=$(rpcinfo -T tcp $_localhost $progname $version 2>&1) ; then
 467         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 468 $ctdb_check_rpc_out"
 469         echo "$ctdb_check_rpc_out"
 470         return 1
 471     fi
 472 }
 473
 474 ######################################################
 475 # Ensure $service_name is set
 476 assert_service_name ()
 477 {
 478     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 479 }
 480
 481 ######################################################
 482 # check a set of directories is available
 483 # return 1 on a missing directory
 484 # directories are read from stdin
 485 ######################################################
 486 ctdb_check_directories_probe()
 487 {
 488     while IFS="" read d ; do
 489         case "$d" in
 490             *%*)
 491                 continue
 492                 ;;
 493             *)
 494                 [ -d "${d}/." ] || return 1
 495         esac
 496     done
 497 }
 498
 499 ######################################################
 500 # check a set of directories is available
 501 # directories are read from stdin
 502 ######################################################
 503 ctdb_check_directories()
 504 {
 505     ctdb_check_directories_probe || {
 506         echo "ERROR: $service_name directory \"$d\" not available"
 507         exit 1
 508     }
 509 }
 510
 511 ######################################################
 512 # check a set of tcp ports
 513 # usage: ctdb_check_tcp_ports <ports...>
 514 ######################################################
 515
 516 # This flag file is created when a service is initially started.  It
 517 # is deleted the first time TCP port checks for that service succeed.
 518 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 519 # message if a port check fails.
 520 _ctdb_check_tcp_common ()
 521 {
 522     assert_service_name
 523     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 524 }
 525
 526 ctdb_check_tcp_init ()
 527 {
 528     _ctdb_check_tcp_common
 529     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 530     touch "$_ctdb_service_started_file"
 531 }
 532
 533 # Check whether something is listening on all of the given TCP ports
 534 # using the "ctdb checktcpport" command.
 535 ctdb_check_tcp_ports()
 536 {
 537     if [ -z "$1" ] ; then
 538         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 539         exit 1
 540     fi
 541
 542     for _p ; do  # process each function argument (port)
 543         _cmd="ctdb checktcpport $_p"
 544         _out=$($_cmd 2>&1)
 545         _ret=$?
 546         case "$_ret" in
 547             0)
 548                 _ctdb_check_tcp_common
 549                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 550                     echo "ERROR: $service_name tcp port $_p is not responding"
 551                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 552                 else
 553                     echo "INFO: $service_name tcp port $_p is not responding"
 554                 fi
 555
 556                 return 1
 557                 ;;
 558             98)
 559                 # Couldn't bind, something already listening, next port...
 560                 continue
 561                 ;;
 562             *)
 563                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 564                 debug <<EOF
 565 ctdb checktcpport (exited with $_ret) with output:
 566 $_out"
 567 EOF
 568                 return $_ret
 569         esac
 570     done
 571
 572     # All ports listening
 573     _ctdb_check_tcp_common
 574     rm -f "$_ctdb_service_started_file"
 575     return 0
 576 }
 577
 578 ######################################################
 579 # check a unix socket
 580 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 581 ######################################################
 582 ctdb_check_unix_socket() {
 583     socket_path="$1"
 584     [ -z "$socket_path" ] && return
 585
 586     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 587         echo "ERROR: $service_name socket $socket_path not found"
 588         return 1
 589     fi
 590 }
 591
 592 ######################################################
 593 # check a command returns zero status
 594 # usage: ctdb_check_command <command>
 595 ######################################################
 596 ctdb_check_command ()
 597 {
 598     _out=$("$@" 2>&1) || {
 599         echo "ERROR: $* returned error"
 600         echo "$_out" | debug
 601         exit 1
 602     }
 603 }
 604
 605 ################################################
 606 # kill off any TCP connections with the given IP
 607 ################################################
 608 kill_tcp_connections ()
 609 {
 610     _ip="$1"
 611
 612     _oneway=false
 613     if [ "$2" = "oneway" ] ; then
 614         _oneway=true
 615     fi
 616
 617     get_tcp_connections_for_ip "$_ip" | {
 618         _killcount=0
 619         _connections=""
 620         _nl="
 621 "
 622         while read _dst _src; do
 623             _destport="${_dst##*:}"
 624             __oneway=$_oneway
 625             case $_destport in
 626                 # we only do one-way killtcp for CIFS
 627                 139|445) __oneway=true ;;
 628             esac
 629
 630             echo "Killing TCP connection $_src $_dst"
 631             _connections="${_connections}${_nl}${_src} ${_dst}"
 632             if ! $__oneway ; then
 633                 _connections="${_connections}${_nl}${_dst} ${_src}"
 634             fi
 635
 636             _killcount=$(($_killcount + 1))
 637         done
 638
 639         if [ $_killcount -eq 0 ] ; then
 640             return
 641         fi
 642
 643         echo "$_connections" | ctdb killtcp || {
 644             echo "Failed to send killtcp control"
 645             return
 646         }
 647
 648         _count=0
 649         while : ; do
 650             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 651
 652             if [ $_remaining -eq 0 ] ; then
 653                 echo "Killed $_killcount TCP connections to released IP $_ip"
 654                 return
 655             fi
 656
 657             _count=$(($_count + 1))
 658             if [ $_count -gt 3 ] ; then
 659                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 660                 return
 661             fi
 662
 663             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 664             sleep 1
 665         done
 666     }
 667 }
 668
 669 ##################################################################
 670 # kill off the local end for any TCP connections with the given IP
 671 ##################################################################
 672 kill_tcp_connections_local_only ()
 673 {
 674     kill_tcp_connections "$1" "oneway"
 675 }
 676
 677 ##################################################################
 678 # tickle any TCP connections with the given IP
 679 ##################################################################
 680 tickle_tcp_connections ()
 681 {
 682     _ip="$1"
 683
 684     get_tcp_connections_for_ip "$_ip" |
 685     {
 686         _failed=false
 687
 688         while read dest src; do
 689             echo "Tickle TCP connection $src $dest"
 690             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 691             echo "Tickle TCP connection $dest $src"
 692             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 693         done
 694
 695         if $_failed ; then
 696             echo "Failed to send tickle control"
 697         fi
 698     }
 699 }
 700
 701 get_tcp_connections_for_ip ()
 702 {
 703     _ip="$1"
 704
 705     netstat -tn | awk -v ip=$_ip \
 706         'index($1, "tcp") == 1 && \
 707          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 708          && $6 == "ESTABLISHED" \
 709          {print $4" "$5}'
 710 }
 711
 712 ##################################################################
 713 # use statd-callout to update NFS lock info
 714 ##################################################################
 715 nfs_update_lock_info ()
 716 {
 717     if [ -x "$CTDB_BASE/statd-callout" ] ; then
 718         "$CTDB_BASE/statd-callout" update
 719     fi
 720 }
 721
 722 ########################################################
 723 # start/stop the Ganesha nfs service
 724 ########################################################
 725 startstop_ganesha()
 726 {
 727     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 728     case "$1" in
 729         start)
 730             service "$_service_name" start
 731             ;;
 732         stop)
 733             service "$_service_name" stop
 734             ;;
 735         restart)
 736             service "$_service_name" stop
 737             nfs_dump_some_threads "rpc.statd"
 738             service "$_service_name" start
 739             ;;
 740     esac
 741 }
 742
 743 ########################################################
 744 # start/stop the nfs service on different platforms
 745 ########################################################
 746 startstop_nfs() {
 747         PLATFORM="unknown"
 748         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 749                 PLATFORM="sles"
 750         }
 751         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 752             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 753                 PLATFORM="rhel"
 754         }
 755
 756         case $PLATFORM in
 757         sles)
 758                 case $1 in
 759                 start)
 760                         service nfsserver start
 761                         ;;
 762                 stop)
 763                         service nfsserver stop > /dev/null 2>&1
 764                         ;;
 765                 restart)
 766                         set_proc "fs/nfsd/threads" 0
 767                         service nfsserver stop > /dev/null 2>&1
 768                         pkill -9 nfsd
 769                         nfs_dump_some_threads
 770                         service nfsserver start
 771                         ;;
 772                 esac
 773                 ;;
 774         rhel)
 775                 case $1 in
 776                 start)
 777                         service nfslock start
 778                         service nfs start
 779                         ;;
 780                 stop)
 781                         service nfs stop
 782                         service nfslock stop
 783                         ;;
 784                 restart)
 785                         set_proc "fs/nfsd/threads" 0
 786                         service nfs stop > /dev/null 2>&1
 787                         service nfslock stop > /dev/null 2>&1
 788                         pkill -9 nfsd
 789                         nfs_dump_some_threads
 790                         service nfslock start
 791                         service nfs start
 792                         ;;
 793                 esac
 794                 ;;
 795         *)
 796                 echo "Unknown platform. NFS is not supported with ctdb"
 797                 exit 1
 798                 ;;
 799         esac
 800 }
 801
 802 # Dump up to the configured number of nfsd thread backtraces.
 803 nfs_dump_some_threads ()
 804 {
 805     _prog="${1:-nfsd}"
 806
 807     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 808     [ $_num -gt 0 ] || return 0
 809
 810     program_stack_traces "$_prog" $_num
 811 }
 812
 813 ########################################################
 814 # start/stop the nfs lockmanager service on different platforms
 815 ########################################################
 816 startstop_nfslock() {
 817         PLATFORM="unknown"
 818         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 819                 PLATFORM="sles"
 820         }
 821         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 822             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 823                 PLATFORM="rhel"
 824         }
 825
 826         case $PLATFORM in
 827         sles)
 828                 # for sles there is no service for lockmanager
 829                 # so we instead just shutdown/restart nfs
 830                 case $1 in
 831                 start)
 832                         service nfsserver start
 833                         ;;
 834                 stop)
 835                         service nfsserver stop > /dev/null 2>&1
 836                         ;;
 837                 restart)
 838                         service nfsserver stop > /dev/null 2>&1
 839                         service nfsserver start
 840                         ;;
 841                 esac
 842                 ;;
 843         rhel)
 844                 case $1 in
 845                 start)
 846                         service nfslock start
 847                         ;;
 848                 stop)
 849                         service nfslock stop > /dev/null 2>&1
 850                         ;;
 851                 restart)
 852                         service nfslock stop > /dev/null 2>&1
 853                         service nfslock start
 854                         ;;
 855                 esac
 856                 ;;
 857         *)
 858                 echo "Unknown platform. NFS locking is not supported with ctdb"
 859                 exit 1
 860                 ;;
 861         esac
 862 }
 863
 864 ########################################################
 865
 866 add_ip_to_iface ()
 867 {
 868     _iface=$1
 869     _ip=$2
 870     _maskbits=$3
 871
 872     # Ensure interface is up
 873     ip link set "$_iface" up || \
 874         die "Failed to bringup interface $_iface"
 875
 876     # Only need to define broadcast for IPv4
 877     case "$ip" in
 878         *:*) _bcast=""      ;;
 879         *)   _bcast="brd +" ;;
 880     esac
 881
 882     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 883         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 884         return 1
 885     }
 886
 887     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 888     if [ -z "$_bcast" ] ; then
 889         for _x in $(seq 1 10) ; do
 890             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 891             sleep 0.5
 892         done
 893
 894         # If the address was a duplicate then it won't be on the
 895         # interface so flag an error.
 896         _t=$(ip addr show to "${_ip}/128")
 897         case "$_t" in
 898             "")
 899                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 900                 return 1
 901                 ;;
 902             *tentative*|*dadfailed*)
 903                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 904                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 905                 return 1
 906                 ;;
 907         esac
 908     fi
 909 }
 910
 911 delete_ip_from_iface()
 912 {
 913     _iface=$1
 914     _ip=$2
 915     _maskbits=$3
 916
 917     # This could be set globally for all interfaces but it is probably
 918     # better to avoid surprises, so limit it the interfaces where CTDB
 919     # has public IP addresses.  There isn't anywhere else convenient
 920     # to do this so just set it each time.  This is much cheaper than
 921     # remembering and re-adding secondaries.
 922     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 923
 924     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 925         echo "Failed to del $_ip on dev $_iface"
 926         return 1
 927     }
 928 }
 929
 930 # If the given IP is hosted then print 2 items: maskbits and iface
 931 ip_maskbits_iface ()
 932 {
 933     _addr="$1"
 934
 935     case "$_addr" in
 936         *:*) _family="inet6" ; _bits=128 ;;
 937         *)   _family="inet"  ; _bits=32  ;;
 938     esac
 939
 940     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 941         awk -v family="${_family}" \
 942             'NR == 1 { iface = $2; sub(":$", "", iface) } \
 943              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
 944                            print mask, iface, family }'
 945 }
 946
 947 drop_ip ()
 948 {
 949     _addr="${1%/*}"  # Remove optional maskbits
 950
 951     set -- $(ip_maskbits_iface $_addr)
 952     if [ -n "$1" ] ; then
 953         _maskbits="$1"
 954         _iface="$2"
 955         echo "Removing public address $_addr/$_maskbits from device $_iface"
 956         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 957     fi
 958 }
 959
 960 drop_all_public_ips ()
 961 {
 962     while read _ip _x ; do
 963         drop_ip "$_ip"
 964     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 965 }
 966
 967 flush_route_cache ()
 968 {
 969     set_proc sys/net/ipv4/route/flush 1
 970     set_proc sys/net/ipv6/route/flush 1
 971 }
 972
 973 ########################################################
 974 # Simple counters
 975 _ctdb_counter_common () {
 976     _service_name="${1:-${service_name:-${script_name}}}"
 977     _counter_file="$ctdb_fail_dir/$_service_name"
 978     mkdir -p "${_counter_file%/*}" # dirname
 979 }
 980 ctdb_counter_init () {
 981     _ctdb_counter_common "$1"
 982
 983     >"$_counter_file"
 984 }
 985 ctdb_counter_incr () {
 986     _ctdb_counter_common "$1"
 987
 988     # unary counting!
 989     echo -n 1 >> "$_counter_file"
 990 }
 991 ctdb_check_counter () {
 992     _msg="${1:-error}"  # "error"  - anything else is silent on fail
 993     _op="${2:--ge}"  # an integer operator supported by test
 994     _limit="${3:-${service_fail_limit}}"
 995     shift 3
 996     _ctdb_counter_common "$1"
 997
 998     # unary counting!
 999     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
1000     _hit=false
1001     if [ "$_op" != "%" ] ; then
1002         if [ $_size $_op $_limit ] ; then
1003             _hit=true
1004         fi
1005     else
1006         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1007             _hit=true
1008         fi
1009     fi
1010     if $_hit ; then
1011         if [ "$_msg" = "error" ] ; then
1012             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1013             exit 1
1014         else
1015             return 1
1016         fi
1017     fi
1018 }
1019
1020 ########################################################
1021
1022 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1023 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1024
1025 ctdb_setup_service_state_dir ()
1026 {
1027     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1028     mkdir -p "$service_state_dir" || {
1029         echo "Error creating state dir \"$service_state_dir\""
1030         exit 1
1031     }
1032 }
1033
1034 ########################################################
1035 # Managed status history, for auto-start/stop
1036
1037 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1038
1039 _ctdb_managed_common ()
1040 {
1041     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1042 }
1043
1044 ctdb_service_managed ()
1045 {
1046     _ctdb_managed_common
1047     mkdir -p "$ctdb_managed_dir"
1048     touch "$_ctdb_managed_file"
1049 }
1050
1051 ctdb_service_unmanaged ()
1052 {
1053     _ctdb_managed_common
1054     rm -f "$_ctdb_managed_file"
1055 }
1056
1057 is_ctdb_previously_managed_service ()
1058 {
1059     _ctdb_managed_common
1060     [ -f "$_ctdb_managed_file" ]
1061 }
1062
1063 ########################################################
1064 # Check and set status
1065
1066 log_status_cat ()
1067 {
1068     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1069 }
1070
1071 ctdb_checkstatus ()
1072 {
1073     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1074         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1075         return 1
1076     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1077         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1078         return 2
1079     else
1080         return 0
1081     fi
1082 }
1083
1084 ctdb_setstatus ()
1085 {
1086     d="$ctdb_status_dir/$script_name"
1087     case "$1" in
1088         unhealthy|banned)
1089             mkdir -p "$d"
1090             cat "$2" >"$d/$1"
1091             ;;
1092         *)
1093             for i in "banned" "unhealthy" ; do
1094                 rm -f "$d/$i"
1095             done
1096             ;;
1097     esac
1098 }
1099
1100 ##################################################################
1101 # Reconfigure a service on demand
1102
1103 _ctdb_service_reconfigure_common ()
1104 {
1105     _d="$ctdb_status_dir/${service_name}"
1106     mkdir -p "$_d"
1107     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1108 }
1109
1110 ctdb_service_needs_reconfigure ()
1111 {
1112     _ctdb_service_reconfigure_common
1113     [ -e "$_ctdb_service_reconfigure_flag" ]
1114 }
1115
1116 ctdb_service_set_reconfigure ()
1117 {
1118     _ctdb_service_reconfigure_common
1119     >"$_ctdb_service_reconfigure_flag"
1120 }
1121
1122 ctdb_service_unset_reconfigure ()
1123 {
1124     _ctdb_service_reconfigure_common
1125     rm -f "$_ctdb_service_reconfigure_flag"
1126 }
1127
1128 ctdb_service_reconfigure ()
1129 {
1130     echo "Reconfiguring service \"${service_name}\"..."
1131     ctdb_service_unset_reconfigure
1132     service_reconfigure || return $?
1133     ctdb_counter_init
1134 }
1135
1136 # Default service_reconfigure() function does nothing.
1137 service_reconfigure ()
1138 {
1139     :
1140 }
1141
1142 ctdb_reconfigure_take_lock ()
1143 {
1144     _ctdb_service_reconfigure_common
1145     _lock="${_d}/reconfigure_lock"
1146     mkdir -p "${_lock%/*}" # dirname
1147     touch "$_lock"
1148
1149     (
1150         flock 0
1151         # This is overkill but will work if we need to extend this to
1152         # allow certain events to run multiple times in parallel
1153         # (e.g. takeip) and write multiple PIDs to the file.
1154         read _locker_event
1155         if [ -n "$_locker_event" ] ; then
1156             while read _pid ; do
1157                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1158                     kill -0 "$_pid" 2>/dev/null ; then
1159                     exit 1
1160                 fi
1161             done
1162         fi
1163
1164         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1165         exit 0
1166     ) <"$_lock"
1167 }
1168
1169 ctdb_reconfigure_release_lock ()
1170 {
1171     _ctdb_service_reconfigure_common
1172     _lock="${_d}/reconfigure_lock"
1173
1174     rm -f "$_lock"
1175 }
1176
1177 ctdb_replay_monitor_status ()
1178 {
1179     echo "Replaying previous status for this script due to reconfigure..."
1180     # Leading separator ('|') is missing in some versions...
1181     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1182     # Output looks like this:
1183     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1184     # This is the cheapest way of getting fields in the middle.
1185     set -- $(IFS="|" ; echo $_out)
1186     _code="$3"
1187     _status="$4"
1188     # The error output field can include colons so we'll try to
1189     # preserve them.  The weak checking at the beginning tries to make
1190     # this work for both broken (no leading '|') and fixed output.
1191     _out="${_out%|}"
1192     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1193     case "$_status" in
1194         OK) : ;;  # Do nothing special.
1195         TIMEDOUT)
1196             # Recast this as an error, since we can't exit with the
1197             # correct negative number.
1198             _code=1
1199             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1200             ;;
1201         DISABLED)
1202             # Recast this as an OK, since we can't exit with the
1203             # correct negative number.
1204             _code=0
1205             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1206             ;;
1207         *) : ;;  # Must be ERROR, do nothing special.
1208     esac
1209     if [ -n "$_err_out" ] ; then
1210         echo "$_err_out"
1211     fi
1212     exit $_code
1213 }
1214
1215 ctdb_service_check_reconfigure ()
1216 {
1217     assert_service_name
1218
1219     # We only care about some events in this function.  For others we
1220     # return now.
1221     case "$event_name" in
1222         monitor|ipreallocated|reconfigure) : ;;
1223         *) return 0 ;;
1224     esac
1225
1226     if ctdb_reconfigure_take_lock ; then
1227         # No events covered by this function are running, so proceed
1228         # with gay abandon.
1229         case "$event_name" in
1230             reconfigure)
1231                 (ctdb_service_reconfigure)
1232                 exit $?
1233                 ;;
1234             ipreallocated)
1235                 if ctdb_service_needs_reconfigure ; then
1236                     ctdb_service_reconfigure
1237                 fi
1238                 ;;
1239         esac
1240
1241         ctdb_reconfigure_release_lock
1242     else
1243         # Somebody else is running an event we don't want to collide
1244         # with.  We proceed with caution.
1245         case "$event_name" in
1246             reconfigure)
1247                 # Tell whoever called us to retry.
1248                 exit 2
1249                 ;;
1250             ipreallocated)
1251                 # Defer any scheduled reconfigure and just run the
1252                 # rest of the ipreallocated event, as per the
1253                 # eventscript.  There's an assumption here that the
1254                 # event doesn't depend on any scheduled reconfigure.
1255                 # This is true in the current code.
1256                 return 0
1257                 ;;
1258             monitor)
1259                 # There is most likely a reconfigure in progress so
1260                 # the service is possibly unstable.  As above, we
1261                 # defer any scheduled reconfigured.  We also replay
1262                 # the previous monitor status since that's the best
1263                 # information we have.
1264                 ctdb_replay_monitor_status
1265                 ;;
1266         esac
1267     fi
1268 }
1269
1270 ##################################################################
1271 # Does CTDB manage this service? - and associated auto-start/stop
1272
1273 ctdb_compat_managed_service ()
1274 {
1275     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1276         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1277     fi
1278 }
1279
1280 is_ctdb_managed_service ()
1281 {
1282     assert_service_name
1283
1284     # $t is used just for readability and to allow better accurate
1285     # matching via leading/trailing spaces
1286     t=" $CTDB_MANAGED_SERVICES "
1287
1288     # Return 0 if "<space>$service_name<space>" appears in $t
1289     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1290         return 0
1291     fi
1292
1293     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1294     # backward compatibility and try again.
1295     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1296     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1297     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1298     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1299     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1300     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1301     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1302     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1303     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1304
1305     t=" $CTDB_MANAGED_SERVICES "
1306
1307     # Return 0 if "<space>$service_name<space>" appears in $t
1308     [ "${t#* ${service_name} }" != "${t}" ]
1309 }
1310
1311 ctdb_start_stop_service ()
1312 {
1313     assert_service_name
1314
1315     # Allow service-start/service-stop pseudo-events to start/stop
1316     # services when we're not auto-starting/stopping and we're not
1317     # monitoring.
1318     case "$event_name" in
1319         service-start)
1320             if is_ctdb_managed_service ; then
1321                 die 'service-start event not permitted when service is managed'
1322             fi
1323             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1324                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1325             fi
1326             ctdb_service_start
1327             exit $?
1328             ;;
1329         service-stop)
1330             if is_ctdb_managed_service ; then
1331                 die 'service-stop event not permitted when service is managed'
1332             fi
1333             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1334                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1335             fi
1336             ctdb_service_stop
1337             exit $?
1338             ;;
1339     esac
1340
1341     # Do nothing unless configured to...
1342     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1343
1344     [ "$event_name" = "monitor" ] || return 0
1345
1346     if is_ctdb_managed_service ; then
1347         if ! is_ctdb_previously_managed_service ; then
1348             echo "Starting service \"$service_name\" - now managed"
1349             background_with_logging ctdb_service_start
1350             exit $?
1351         fi
1352     else
1353         if is_ctdb_previously_managed_service ; then
1354             echo "Stopping service \"$service_name\" - no longer managed"
1355             background_with_logging ctdb_service_stop
1356             exit $?
1357         fi
1358     fi
1359 }
1360
1361 ctdb_service_start ()
1362 {
1363     # The service is marked managed if we've ever tried to start it.
1364     ctdb_service_managed
1365
1366     service_start || return $?
1367
1368     ctdb_counter_init
1369     ctdb_check_tcp_init
1370 }
1371
1372 ctdb_service_stop ()
1373 {
1374     ctdb_service_unmanaged
1375     service_stop
1376 }
1377
1378 # Default service_start() and service_stop() functions.
1379
1380 # These may be overridden in an eventscript.
1381 service_start ()
1382 {
1383     service "$service_name" start
1384 }
1385
1386 service_stop ()
1387 {
1388     service "$service_name" stop
1389 }
1390
1391 ##################################################################
1392
1393 ctdb_standard_event_handler ()
1394 {
1395     case "$1" in
1396         status)
1397             ctdb_checkstatus
1398             exit
1399             ;;
1400         setstatus)
1401             shift
1402             ctdb_setstatus "$@"
1403             exit
1404             ;;
1405     esac
1406 }
1407
1408 iptables_wrapper ()
1409 {
1410     _family="$1" ; shift
1411     if [ "$_family" = "inet6" ] ; then
1412         _iptables_cmd="ip6tables"
1413     else
1414         _iptables_cmd="iptables"
1415     fi
1416
1417     # iptables doesn't like being re-entered, so flock-wrap it.
1418     flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1419 }
1420
1421 # AIX (and perhaps others?) doesn't have mktemp
1422 if ! which mktemp >/dev/null 2>&1 ; then
1423     mktemp ()
1424     {
1425         _dir=false
1426         if [ "$1" = "-d" ] ; then
1427             _dir=true
1428             shift
1429         fi
1430         _d="${TMPDIR:-/tmp}"
1431         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1432             md5sum | \
1433             sed -e 's@\(..........\).*@\1@')
1434         _t="${_d}/tmp.${_hex10}"
1435         (
1436             umask 077
1437             if $_dir ; then
1438                 mkdir "$_t"
1439             else
1440                 >"$_t"
1441             fi
1442         )
1443         echo "$_t"
1444     }
1445 fi
1446
1447 ########################################################
1448 # tickle handling
1449 ########################################################
1450
1451 update_tickles ()
1452 {
1453         _port="$1"
1454
1455         tickledir="$CTDB_VARDIR/state/tickles"
1456         mkdir -p "$tickledir"
1457
1458         # Who am I?
1459         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1460
1461         # What public IPs do I hold?
1462         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1463
1464         # IPs as a regexp choice
1465         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1466
1467         # Record connections to our public IPs in a temporary file
1468         _my_connections="${tickledir}/${_port}.connections"
1469         rm -f "$_my_connections"
1470         netstat -tn |
1471         awk -v destpat="^${_ipschoice}:${_port}\$" \
1472           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1473         sort >"$_my_connections"
1474
1475         # Record our current tickles in a temporary file
1476         _my_tickles="${tickledir}/${_port}.tickles"
1477         rm -f "$_my_tickles"
1478         for _i in $_ips ; do
1479                 ctdb -X gettickles $_i $_port |
1480                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1481         done |
1482         sort >"$_my_tickles"
1483
1484         # Add tickles for connections that we haven't already got tickles for
1485         comm -23 "$_my_connections" "$_my_tickles" |
1486         while read _src _dst ; do
1487                 ctdb addtickle $_src $_dst
1488         done
1489
1490         # Remove tickles for connections that are no longer there
1491         comm -13 "$_my_connections" "$_my_tickles" |
1492         while read _src _dst ; do
1493                 ctdb deltickle $_src $_dst
1494         done
1495
1496         rm -f "$_my_connections" "$_my_tickles"
1497 }
1498
1499 ########################################################
1500 # load a site local config file
1501 ########################################################
1502
1503 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1504         . "$CTDB_RC_LOCAL"
1505 }
1506
1507 [ -x $CTDB_BASE/rc.local ] && {
1508         . $CTDB_BASE/rc.local
1509 }
1510
1511 [ -d $CTDB_BASE/rc.local.d ] && {
1512         for i in $CTDB_BASE/rc.local.d/* ; do
1513                 [ -x "$i" ] && . "$i"
1514         done
1515 }
1516
1517 script_name="${0##*/}"       # basename
1518 service_fail_limit=1
1519 event_name="$1"