config/functions

   1 # utility functions for ctdb event scripts
   2
   3 PATH=/bin:/usr/bin:/usr/sbin:/sbin:$PATH
   4
   5 #######################################
   6 # pull in a system config file, if any
   7 _loadconfig() {
   8
   9     if [ -z "$1" ] ; then
  10         foo="${service_config:-${service_name}}"
  11         if [ -n "$foo" ] ; then
  12             loadconfig "$foo"
  13         fi
  14     elif [ "$1" != "ctdb" ] ; then
  15         loadconfig "ctdb"
  16     fi
  17
  18
  19     if [ -f /etc/sysconfig/$1 ]; then
  20         . /etc/sysconfig/$1
  21     elif [ -f /etc/default/$1 ]; then
  22         . /etc/default/$1
  23     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  24         . $CTDB_BASE/sysconfig/$1
  25     fi
  26 }
  27
  28 loadconfig () {
  29     _loadconfig "$@"
  30 }
  31
  32 ##############################################################
  33 # determine on what type of system (init style) we are running
  34 detect_init_style() {
  35     # only do detection if not already set:
  36     test "x$CTDB_INIT_STYLE" != "x" && return
  37
  38     if [ -x /sbin/startproc ]; then
  39         CTDB_INIT_STYLE="suse"
  40     elif [ -x /sbin/start-stop-daemon ]; then
  41         CTDB_INIT_STYLE="debian"
  42     else
  43         CTDB_INIT_STYLE="redhat"
  44     fi
  45 }
  46
  47 ######################################################
  48 # simulate /sbin/service on platforms that don't have it
  49 service() {
  50   _service_name="$1"
  51   _op="$2"
  52
  53   # do nothing, when no service was specified
  54   [ -z "$_service_name" ] && return
  55
  56   if [ -x /sbin/service ]; then
  57       /sbin/service "$_service_name" "$_op"
  58   elif [ -x /etc/init.d/$_service_name ]; then
  59       /etc/init.d/$_service_name "$_op"
  60   elif [ -x /etc/rc.d/init.d/$_service_name ]; then
  61       /etc/rc.d/init.d/$_service_name "$_op"
  62   fi
  63 }
  64
  65 ######################################################
  66 # simulate /sbin/service (niced) on platforms that don't have it
  67 nice_service() {
  68   _service_name="$1"
  69   _op="$2"
  70
  71   # do nothing, when no service was specified
  72   [ -z "$_service_name" ] && return
  73
  74   if [ -x /sbin/service ]; then
  75       nice /sbin/service "$_service_name" "$_op"
  76   elif [ -x /etc/init.d/$_service_name ]; then
  77       nice /etc/init.d/$_service_name "$_op"
  78   elif [ -x /etc/rc.d/init.d/$_service_name ]; then
  79       nice /etc/rc.d/init.d/$_service_name "$_op"
  80   fi
  81 }
  82
  83 ######################################################
  84 # wait for a command to return a zero exit status
  85 # usage: ctdb_wait_command SERVICE_NAME <command>
  86 ######################################################
  87 ctdb_wait_command() {
  88   service_name="$1"
  89   wait_cmd="$2"
  90   [ -z "$wait_cmd" ] && return;
  91   all_ok=0
  92   echo "Waiting for service $service_name to start"
  93   while [ $all_ok -eq 0 ]; do
  94           $wait_cmd > /dev/null 2>&1 && all_ok=1
  95           ctdb status > /dev/null 2>&1 || {
  96                 echo "ctdb daemon has died. Exiting wait for $service_name"
  97                 exit 1
  98           }
  99           [ $all_ok -eq 1 ] || sleep 1
 100   done
 101   echo "Local service $service_name is up"
 102 }
 103
 104
 105 ######################################################
 106 # wait for a set of tcp ports
 107 # usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
 108 ######################################################
 109 ctdb_wait_tcp_ports() {
 110   service_name="$1"
 111   shift
 112   wait_ports="$*"
 113   [ -z "$wait_ports" ] && return;
 114   all_ok=0
 115   echo "Waiting for tcp service $service_name to start"
 116   while [ $all_ok -eq 0 ]; do
 117           all_ok=1
 118           for p in $wait_ports; do
 119               if [ -x /usr/bin/netcat ]; then
 120                   /usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
 121               elif [ -x /usr/bin/nc ]; then
 122                   /usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
 123               elif [ -x /usr/bin/netstat ]; then
 124                   (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
 125               elif [ -x /bin/netstat ]; then
 126                   (netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
 127               else
 128                   echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
 129                   return 127
 130               fi
 131           done
 132           [ $all_ok -eq 1 ] || sleep 1
 133           ctdb status > /dev/null 2>&1 || {
 134                 echo "ctdb daemon has died. Exiting tcp wait $service_name"
 135                 return 1
 136           }
 137   done
 138   echo "Local tcp services for $service_name are up"
 139 }
 140
 141
 142 ######################################################
 143 # check that a rpc server is registered with portmap
 144 # and responding to requests
 145 # usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
 146 ######################################################
 147 ctdb_check_rpc() {
 148     progname="$1"
 149     prognum="$2"
 150     version="$3"
 151
 152     ctdb_check_rpc_out=$(rpcinfo -u localhost $prognum $version 2>&1)
 153     if [ $? -ne 0 ] ; then
 154         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
 155 $ctdb_check_rpc_out"
 156         echo "$ctdb_check_rpc_out"
 157         return 1
 158     fi
 159 }
 160
 161 ######################################################
 162 # check a set of directories is available
 163 # return 1 on a missing directory
 164 # usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
 165 ######################################################
 166 ctdb_check_directories_probe() {
 167     while IFS="" read d ; do
 168         case "$d" in
 169             *%*)
 170                 continue
 171                 ;;
 172             *)
 173                 [ -d "${d}/." ] || return 1
 174         esac
 175     done
 176 }
 177
 178 ######################################################
 179 # check a set of directories is available
 180 # usage: ctdb_check_directories SERVICE_NAME <directories...>
 181 ######################################################
 182 ctdb_check_directories() {
 183     n="${1:-${service_name}}"
 184     ctdb_check_directories_probe || {
 185         echo "ERROR: $n directory \"$d\" not available"
 186         exit 1
 187     }
 188 }
 189
 190 ######################################################
 191 # check a set of tcp ports
 192 # usage: ctdb_check_tcp_ports <ports...>
 193 ######################################################
 194
 195 # This flag file is created when a service is initially started.  It
 196 # is deleted the first time TCP port checks for that service succeed.
 197 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 198 # message if a port check fails.
 199 _ctdb_check_tcp_common ()
 200 {
 201     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 202 }
 203
 204 ctdb_check_tcp_init ()
 205 {
 206     _ctdb_check_tcp_common
 207     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 208     touch "$_ctdb_service_started_file"
 209 }
 210
 211 ctdb_check_tcp_ports()
 212 {
 213     if [ -z "$1" ] ; then
 214         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 215         exit 1
 216     fi
 217
 218     # Set default value for CTDB_TCP_PORT_CHECKS if unset.
 219     # If any of these defaults are unsupported then this variable can
 220     # be overridden in /etc/sysconfig/ctdb or via a file in
 221     # /etc/ctdb/rc.local.d/.
 222     : ${CTDB_TCP_PORT_CHECKERS:=ctdb nmap netstat}
 223
 224     for _c in $CTDB_TCP_PORT_CHECKERS ; do
 225         ctdb_check_tcp_ports_$_c "$@"
 226         case "$?" in
 227             0)
 228                 _ctdb_check_tcp_common
 229                 rm -f "$_ctdb_service_started_file"
 230                 return 0
 231                 ;;
 232             1)
 233                 _ctdb_check_tcp_common
 234                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 235                     echo "ERROR: $service_name tcp port $_p is not responding"
 236                     cat <<EOF
 237 $ctdb_check_tcp_ports_debug
 238 EOF
 239                 else
 240                     echo "INFO: $service_name tcp port $_p is not responding"
 241                 fi
 242
 243                 return 1
 244                 ;;
 245             127)
 246                 : # Not implemented
 247                 ;;
 248             *)
 249
 250         esac
 251     done
 252
 253     echo "INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS=\"$CTDB_TCP_PORT_CHECKERS\""
 254
 255     return 127
 256 }
 257
 258 ctdb_check_tcp_ports_netstat ()
 259 {
 260     _cmd='netstat -l -t -n'
 261     _ns=$($_cmd 2>&1)
 262     if [ $? -eq 127 ] ; then
 263         # netstat probably not installed - unlikely?
 264         ctdb_check_tcp_ports_debug="$_ns"
 265         return 127
 266     fi
 267
 268     for _p ; do  # process each function argument (port)
 269         for _a in '0\.0\.0\.0' '::' ; do
 270             _pat="[[:space:]]${_a}:${_p}[[:space:]]+[^[:space:]]+[[:space:]]+LISTEN"
 271             if echo "$_ns" | grep -E -q "$_pat" ; then
 272                 # We matched the port, so process next port
 273                 continue 2
 274             fi
 275         done
 276
 277         # We didn't match the port, so flag an error.
 278         ctdb_check_tcp_ports_debug="$_cmd shows this output:
 279 $_ns"
 280         return 1
 281     done
 282
 283     return 0
 284 }
 285
 286 ctdb_check_tcp_ports_nmap ()
 287 {
 288     # nmap wants a comma-separated list of ports
 289     _ports=""
 290     for _p ; do
 291         _ports="${_ports}${_ports:+,}${_p}"
 292     done
 293
 294     _cmd="nmap -n -oG - -PS 127.0.0.1 -p $_ports"
 295
 296     _nmap_out=$($_cmd 2>&1)
 297     if [ $? -eq 127 ] ; then
 298         # nmap probably not installed
 299         ctdb_check_tcp_ports_debug="$_nmap_out"
 300         return 127
 301     fi
 302
 303     # get the port-related output
 304     _port_info=$(echo "$_nmap_out" | sed -n -r -e 's@^.*Ports:[[:space:]]@@p')
 305
 306     for _p ; do
 307         # looking for something like this:
 308         #  445/open/tcp//microsoft-ds///
 309         # possibly followed by a comma
 310         _t="$_p/open/tcp//"
 311         case "$_port_info" in
 312             # The info we're after must be either at the beginning of
 313             # the string or it must follow a space.
 314             $_t*|*\ $_t*) : ;;
 315             *)
 316                 # Nope, flag an error...
 317                 ctdb_check_tcp_ports_debug="$_cmd shows this output:
 318 $_nmap_out"
 319                 return 1
 320         esac
 321     done
 322
 323     return 0
 324 }
 325
 326 # Use the new "ctdb checktcpport" command to check the port.
 327 # This is very cheap.
 328 ctdb_check_tcp_ports_ctdb ()
 329 {
 330     for _p ; do  # process each function argument (port)
 331         _cmd="ctdb checktcpport $_p"
 332         _out=$($_cmd 2>&1)
 333         _ret=$?
 334         case "$_ret" in
 335             0)
 336                 ctdb_check_tcp_ports_debug="\"$_cmd\" was able to bind to port"
 337                 return 1
 338                 ;;
 339             98)
 340                 # Couldn't bind, something already listening, next port...
 341                 continue
 342                 ;;
 343             *)
 344                 ctdb_check_tcp_ports_debug="$_cmd (exited with $_ret) with output:
 345 $_out"
 346                 # assume not implemented
 347                 return 127
 348         esac
 349     done
 350
 351     return 0
 352 }
 353
 354 ######################################################
 355 # check a unix socket
 356 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 357 ######################################################
 358 ctdb_check_unix_socket() {
 359     socket_path="$1"
 360     [ -z "$socket_path" ] && return
 361
 362     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 363         echo "ERROR: $service_name socket $socket_path not found"
 364         return 1
 365     fi
 366 }
 367
 368 ######################################################
 369 # check a command returns zero status
 370 # usage: ctdb_check_command SERVICE_NAME <command>
 371 ######################################################
 372 ctdb_check_command() {
 373   service_name="$1"
 374   wait_cmd="$2"
 375   [ -z "$wait_cmd" ] && return;
 376   $wait_cmd > /dev/null 2>&1 || {
 377       echo "ERROR: $service_name - $wait_cmd returned error"
 378       exit 1
 379   }
 380 }
 381
 382 ################################################
 383 # kill off any TCP connections with the given IP
 384 ################################################
 385 kill_tcp_connections() {
 386     _IP="$1"
 387     _failed=0
 388
 389     _killcount=0
 390     connfile="$CTDB_VARDIR/state/connections.$_IP"
 391     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 392     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 393
 394     while read dest src; do
 395         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 396         srcport=`echo $src | sed -e "s/^.*://"`
 397         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 398         destport=`echo $dest | sed -e "s/^.*://"`
 399         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 400         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 401         case $destport in
 402           # we only do one-way killtcp for CIFS
 403           139|445) : ;;
 404           # for all others we do 2-way
 405           *)
 406                 ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 407                 ;;
 408         esac
 409         _killcount=`expr $_killcount + 1`
 410      done < $connfile
 411     /bin/rm -f $connfile
 412
 413     [ $_failed = 0 ] || {
 414         echo "Failed to send killtcp control"
 415         return;
 416     }
 417     [ $_killcount -gt 0 ] || {
 418         return;
 419     }
 420     _count=0
 421     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 422         sleep 1
 423         _count=`expr $_count + 1`
 424         [ $_count -gt 3 ] && {
 425             echo "Timed out killing tcp connections for IP $_IP"
 426             return;
 427         }
 428     done
 429     echo "killed $_killcount TCP connections to released IP $_IP"
 430 }
 431
 432 ##################################################################
 433 # kill off the local end for any TCP connections with the given IP
 434 ##################################################################
 435 kill_tcp_connections_local_only() {
 436     _IP="$1"
 437     _failed=0
 438
 439     _killcount=0
 440     connfile="$CTDB_VARDIR/state/connections.$_IP"
 441     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 442     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 443
 444     while read dest src; do
 445         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 446         srcport=`echo $src | sed -e "s/^.*://"`
 447         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 448         destport=`echo $dest | sed -e "s/^.*://"`
 449         echo "Killing TCP connection $srcip:$srcport $destip:$destport"
 450         ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 451         _killcount=`expr $_killcount + 1`
 452      done < $connfile
 453     /bin/rm -f $connfile
 454
 455     [ $_failed = 0 ] || {
 456         echo "Failed to send killtcp control"
 457         return;
 458     }
 459     [ $_killcount -gt 0 ] || {
 460         return;
 461     }
 462     _count=0
 463     while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
 464         sleep 1
 465         _count=`expr $_count + 1`
 466         [ $_count -gt 3 ] && {
 467             echo "Timed out killing tcp connections for IP $_IP"
 468             return;
 469         }
 470     done
 471     echo "killed $_killcount TCP connections to released IP $_IP"
 472 }
 473
 474 ##################################################################
 475 # tickle any TCP connections with the given IP
 476 ##################################################################
 477 tickle_tcp_connections() {
 478     _IP="$1"
 479     _failed=0
 480
 481     _killcount=0
 482     connfile="$CTDB_VARDIR/state/connections.$_IP"
 483     netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
 484     netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
 485
 486     while read dest src; do
 487         srcip=`echo $src | sed -e "s/:[^:]*$//"`
 488         srcport=`echo $src | sed -e "s/^.*://"`
 489         destip=`echo $dest | sed -e "s/:[^:]*$//"`
 490         destport=`echo $dest | sed -e "s/^.*://"`
 491         echo "Tickle TCP connection $srcip:$srcport $destip:$destport"
 492         ctdb tickle $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
 493         echo "Tickle TCP connection $destip:$destport $srcip:$srcport"
 494         ctdb tickle $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
 495      done < $connfile
 496     /bin/rm -f $connfile
 497
 498     [ $_failed = 0 ] || {
 499         echo "Failed to send tickle control"
 500         return;
 501     }
 502 }
 503
 504 ########################################################
 505 # start/stop the nfs service on different platforms
 506 ########################################################
 507 startstop_nfs() {
 508         PLATFORM="unknown"
 509         [ -x /etc/init.d/nfsserver ] && {
 510                 PLATFORM="sles"
 511         }
 512         [ -x /etc/init.d/nfslock ] && {
 513                 PLATFORM="rhel"
 514         }
 515
 516         case $PLATFORM in
 517         sles)
 518                 case $1 in
 519                 start)
 520                         service nfsserver start
 521                         ;;
 522                 stop)
 523                         service nfsserver stop > /dev/null 2>&1
 524                         ;;
 525                 restart)
 526                         echo 0 >/proc/fs/nfsd/threads
 527                         service nfsserver stop > /dev/null 2>&1
 528                         pkill -9 nfsd
 529                         service nfsserver start
 530                         ;;
 531                 esac
 532                 ;;
 533         rhel)
 534                 case $1 in
 535                 start)
 536                         service nfslock start
 537                         service nfs start
 538                         ;;
 539                 stop)
 540                         service nfs stop > /dev/null 2>&1
 541                         service nfslock stop > /dev/null 2>&1
 542                         ;;
 543                 restart)
 544                         echo 0 >/proc/fs/nfsd/threads
 545                         service nfs stop > /dev/null 2>&1
 546                         service nfslock stop > /dev/null 2>&1
 547                         pkill -9 nfsd
 548                         service nfslock start
 549                         service nfs start
 550                         ;;
 551                 esac
 552                 ;;
 553         *)
 554                 echo "Unknown platform. NFS is not supported with ctdb"
 555                 exit 1
 556                 ;;
 557         esac
 558 }
 559
 560 ########################################################
 561 # start/stop the nfs lockmanager service on different platforms
 562 ########################################################
 563 startstop_nfslock() {
 564         PLATFORM="unknown"
 565         [ -x /etc/init.d/nfsserver ] && {
 566                 PLATFORM="sles"
 567         }
 568         [ -x /etc/init.d/nfslock ] && {
 569                 PLATFORM="rhel"
 570         }
 571
 572         case $PLATFORM in
 573         sles)
 574                 # for sles there is no service for lockmanager
 575                 # so we instead just shutdown/restart nfs
 576                 case $1 in
 577                 start)
 578                         service nfsserver start
 579                         ;;
 580                 stop)
 581                         service nfsserver stop > /dev/null 2>&1
 582                         ;;
 583                 restart)
 584                         service nfsserver stop
 585                         service nfsserver start
 586                         ;;
 587                 esac
 588                 ;;
 589         rhel)
 590                 case $1 in
 591                 start)
 592                         service nfslock start
 593                         ;;
 594                 stop)
 595                         service nfslock stop > /dev/null 2>&1
 596                         ;;
 597                 restart)
 598                         service nfslock stop
 599                         service nfslock start
 600                         ;;
 601                 esac
 602                 ;;
 603         *)
 604                 echo "Unknown platform. NFS locking is not supported with ctdb"
 605                 exit 1
 606                 ;;
 607         esac
 608 }
 609
 610 # better use delete_ip_from_iface() together with add_ip_to_iface
 611 # remove_ip should be removed in future
 612 remove_ip() {
 613         local _ip_maskbits=$1
 614         local _iface=$2
 615         local _ip=`echo "$_ip_maskbits" | cut -d '/' -f1`
 616         local _maskbits=`echo "$_ip_maskbits" | cut -d '/' -f2`
 617
 618         delete_ip_from_iface "$_iface" "$_ip" "$_maskbits"
 619         return $?
 620 }
 621
 622 add_ip_to_iface()
 623 {
 624         local _iface=$1
 625         local _ip=$2
 626         local _maskbits=$3
 627         local _state_dir="$CTDB_VARDIR/state/interface_modify"
 628         local _lockfile="$_state_dir/$_iface.flock"
 629         local _readd_base="$_state_dir/$_iface.readd.d"
 630
 631         mkdir -p $_state_dir || {
 632                 ret=$?
 633                 echo "Failed to mkdir -p $_state_dir - $ret"
 634                 return $ret
 635         }
 636
 637         test -f $_lockfile || {
 638                 touch $_lockfile
 639         }
 640
 641         flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh add "$_iface" "$_ip" "$_maskbits" "$_readd_base"
 642         return $?
 643 }
 644
 645 delete_ip_from_iface()
 646 {
 647         local _iface=$1
 648         local _ip=$2
 649         local _maskbits=$3
 650         local _state_dir="$CTDB_VARDIR/state/interface_modify"
 651         local _lockfile="$_state_dir/$_iface.flock"
 652         local _readd_base="$_state_dir/$_iface.readd.d"
 653
 654         mkdir -p $_state_dir || {
 655                 ret=$?
 656                 echo "Failed to mkdir -p $_state_dir - $ret"
 657                 return $ret
 658         }
 659
 660         test -f $_lockfile || {
 661                 touch $_lockfile
 662         }
 663
 664         flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh delete "$_iface" "$_ip" "$_maskbits" "$_readd_base"
 665         return $?
 666 }
 667
 668 setup_iface_ip_readd_script()
 669 {
 670         local _iface=$1
 671         local _ip=$2
 672         local _maskbits=$3
 673         local _readd_script=$4
 674         local _state_dir="$CTDB_VARDIR/state/interface_modify"
 675         local _lockfile="$_state_dir/$_iface.flock"
 676         local _readd_base="$_state_dir/$_iface.readd.d"
 677
 678         mkdir -p $_state_dir || {
 679                 ret=$?
 680                 echo "Failed to mkdir -p $_state_dir - $ret"
 681                 return $ret
 682         }
 683
 684         test -f $_lockfile || {
 685                 touch $_lockfile
 686         }
 687
 688         flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh readd_script "$_iface" "$_ip" "$_maskbits" "$_readd_base" "$_readd_script"
 689         return $?
 690 }
 691
 692 ########################################################
 693 # some simple logic for counting events - per eventscript
 694 # usage: ctdb_counter_init
 695 #        ctdb_counter_incr
 696 #        ctdb_check_counter_limit <limit>
 697 # ctdb_check_counter_limit succeeds when count >= <limit>
 698 ########################################################
 699 _ctdb_counter_common () {
 700     _counter_file="$ctdb_fail_dir/$service_name"
 701     mkdir -p "${_counter_file%/*}" # dirname
 702 }
 703 ctdb_counter_init () {
 704     _ctdb_counter_common
 705
 706     >"$_counter_file"
 707 }
 708 ctdb_counter_incr () {
 709     _ctdb_counter_common
 710
 711     # unary counting!
 712     echo -n 1 >> "$_counter_file"
 713 }
 714 ctdb_check_counter_limit () {
 715     _ctdb_counter_common
 716
 717     _limit="${1:-${service_fail_limit}}"
 718     _quiet="$2"
 719
 720     # unary counting!
 721     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 722     if [ $_size -ge $_limit ] ; then
 723         echo "ERROR: more than $_limit consecutive failures for $service_name, marking cluster unhealthy"
 724         exit 1
 725     elif [ $_size -gt 0 -a -z "$_quiet" ] ; then
 726         echo "WARNING: less than $_limit consecutive failures ($_size) for $service_name, not unhealthy yet"
 727     fi
 728 }
 729 ctdb_check_counter_equal () {
 730     _ctdb_counter_common
 731
 732     _limit=$1
 733
 734     # unary counting!
 735     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
 736     if [ $_size -eq $_limit ] ; then
 737         return 1
 738     fi
 739     return 0
 740 }
 741
 742 ########################################################
 743
 744 ctdb_spool_dir="/var/spool/ctdb"
 745 ctdb_status_dir="$ctdb_spool_dir/status"
 746 ctdb_fail_dir="$ctdb_spool_dir/failcount"
 747 ctdb_active_dir="$ctdb_spool_dir/active"
 748
 749 log_status_cat ()
 750 {
 751     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
 752 }
 753
 754 ctdb_checkstatus ()
 755 {
 756     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
 757         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
 758         return 1
 759     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
 760         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
 761         return 2
 762     else
 763         return 0
 764     fi
 765 }
 766
 767 ctdb_setstatus ()
 768 {
 769     d="$ctdb_status_dir/$script_name"
 770     case "$1" in
 771         unhealthy|banned)
 772             mkdir -p "$d"
 773             cat "$2" >"$d/$1"
 774             ;;
 775         *)
 776             for i in "banned" "unhealthy" ; do
 777                 rm -f "$d/$i"
 778             done
 779             ;;
 780     esac
 781 }
 782
 783 ctdb_service_needs_reconfigure ()
 784 {
 785     [ -e "$ctdb_status_dir/$service_name/reconfigure" ]
 786 }
 787
 788 ctdb_service_set_reconfigure ()
 789 {
 790     d="$ctdb_status_dir/$service_name"
 791     mkdir -p "$d"
 792     >"$d/reconfigure"
 793 }
 794
 795 ctdb_service_unset_reconfigure ()
 796 {
 797     rm -f "$ctdb_status_dir/$service_name/reconfigure"
 798 }
 799
 800 ctdb_service_reconfigure ()
 801 {
 802     echo "Reconfiguring service \"$service_name\"..."
 803     if [ -n "$service_reconfigure" ] ; then
 804         eval $service_reconfigure
 805     else
 806         service "$service_name" restart
 807     fi
 808     ctdb_service_unset_reconfigure
 809     ctdb_counter_init
 810 }
 811
 812 ctdb_compat_managed_service ()
 813 {
 814     if [ "$1" = "yes" ] ; then
 815         t="$t $2 "
 816     fi
 817 }
 818
 819 is_ctdb_managed_service ()
 820 {
 821     _service_name="${1:-${service_name}}"
 822
 823     t=" $CTDB_MANAGED_SERVICES "
 824
 825     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
 826     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
 827     ctdb_compat_managed_service "$CTDB_MANAGES_SCP"      "scp"
 828     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
 829     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
 830     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
 831     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
 832     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
 833     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
 834
 835     # Returns 0 if "<space>$_service_name<space>" appears in $t
 836     [ "${t#* ${_service_name} }" != "${t}" ]
 837 }
 838
 839 ctdb_start_stop_service ()
 840 {
 841     # Do nothing unless configured to...
 842     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
 843
 844     _service_name="${1:-${service_name}}"
 845
 846     [ "$event_name" = "monitor" ] || return 0
 847
 848     _active="$ctdb_active_dir/$_service_name"
 849     if is_ctdb_managed_service "$_service_name"; then
 850         if ! [ -e "$_active" ] ; then
 851             echo "Starting service $_service_name"
 852             ctdb_service_start || exit $?
 853             mkdir -p "$ctdb_active_dir"
 854             touch "$_active"
 855             exit 0
 856         fi
 857     else
 858         if [ -e "$_active" ] ; then
 859             echo "Stopping service $_service_name"
 860             ctdb_service_stop || exit $?
 861             rm -f "$_active"
 862             exit 0
 863         fi
 864     fi
 865 }
 866
 867 ctdb_service_start ()
 868 {
 869     if [ -n "$service_start" ] ; then
 870         eval $service_start || return $?
 871     else
 872         service "$service_name" start || return $?
 873     fi
 874     ctdb_counter_init
 875     ctdb_check_tcp_init
 876 }
 877
 878 ctdb_service_stop ()
 879 {
 880     if [ -n "$service_stop" ] ; then
 881         eval $service_stop
 882     else
 883         service "$service_name" stop
 884     fi
 885 }
 886
 887 ctdb_standard_event_handler ()
 888 {
 889     case "$1" in
 890         status)
 891             ctdb_checkstatus
 892             exit
 893             ;;
 894         setstatus)
 895             shift
 896             ctdb_setstatus "$@"
 897             exit
 898             ;;
 899     esac
 900 }
 901
 902 ipv4_host_addr_to_net_addr()
 903 {
 904         local HOST=$1
 905         local MASKBITS=$2
 906
 907         local HOST0=$(echo $HOST | awk -F . '{print $4}')
 908         local HOST1=$(echo $HOST | awk -F . '{print $3}')
 909         local HOST2=$(echo $HOST | awk -F . '{print $2}')
 910         local HOST3=$(echo $HOST | awk -F . '{print $1}')
 911
 912         local HOST_NUM=$(( $HOST0 + $HOST1 * 256 + $HOST2 * (256 ** 2) + $HOST3 * (256 ** 3) ))
 913
 914         local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
 915
 916         local NET_NUM=$(( $HOST_NUM & $MASK_NUM))
 917
 918         local NET0=$(( $NET_NUM & 255 ))
 919         local NET1=$(( ($NET_NUM & (255 * 256)) / 256 ))
 920         local NET2=$(( ($NET_NUM & (255 * 256**2)) / 256**2 ))
 921         local NET3=$(( ($NET_NUM & (255 * 256**3)) / 256**3 ))
 922
 923         echo "$NET3.$NET2.$NET1.$NET0"
 924 }
 925
 926 ipv4_maskbits_to_net_mask()
 927 {
 928         local MASKBITS=$1
 929
 930         local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
 931
 932         local MASK0=$(( $MASK_NUM & 255 ))
 933         local MASK1=$(( ($MASK_NUM & (255 * 256)) / 256 ))
 934         local MASK2=$(( ($MASK_NUM & (255 * 256**2)) / 256**2 ))
 935         local MASK3=$(( ($MASK_NUM & (255 * 256**3)) / 256**3 ))
 936
 937         echo "$MASK3.$MASK2.$MASK1.$MASK0"
 938 }
 939
 940 ipv4_is_valid_addr()
 941 {
 942         local ADDR=$1
 943         local fail=0
 944
 945         local N=`echo $ADDR | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//'`
 946         test -n "$N" && fail=1
 947
 948         local ADDR0=$(echo $ADDR | awk -F . '{print $4}')
 949         local ADDR1=$(echo $ADDR | awk -F . '{print $3}')
 950         local ADDR2=$(echo $ADDR | awk -F . '{print $2}')
 951         local ADDR3=$(echo $ADDR | awk -F . '{print $1}')
 952
 953         test "$ADDR0" -gt 255 && fail=1
 954         test "$ADDR1" -gt 255 && fail=1
 955         test "$ADDR2" -gt 255 && fail=1
 956         test "$ADDR3" -gt 255 && fail=1
 957
 958         test x"$fail" != x"0" && {
 959                 #echo "IPv4: '$ADDR' is not a valid address"
 960                 return 1;
 961         }
 962
 963         return 0;
 964 }
 965
 966 # iptables doesn't like being re-entered, so flock-wrap it.
 967 iptables()
 968 {
 969         flock -w 30 /var/ctdb/iptables-ctdb.flock /sbin/iptables "$@"
 970 }
 971
 972 ########################################################
 973 # tickle handling
 974 ########################################################
 975
 976 # Temporary directory for tickles.
 977 tickledir="$CTDB_VARDIR/state/tickles"
 978 mkdir -p "$tickledir"
 979
 980 update_tickles ()
 981 {
 982         _port="$1"
 983
 984         mkdir -p "$tickledir" # Just in case
 985
 986         # Who am I?
 987         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
 988
 989         # What public IPs do I hold?
 990         _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
 991
 992         # IPs as a regexp choice
 993         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
 994
 995         # Record connections to our public IPs in a temporary file
 996         _my_connections="${tickledir}/${_port}.connections"
 997         rm -f "$_my_connections"
 998         netstat -tn |
 999         awk -v destpat="^${_ipschoice}:${_port}\$" \
1000           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1001         sort >"$_my_connections"
1002
1003         # Record our current tickles in a temporary file
1004         _my_tickles="${tickledir}/${_port}.tickles"
1005         rm -f "$_my_tickles"
1006         for _i in $_ips ; do
1007                 ctdb -Y gettickles $_i $_port |
1008                 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1009         done |
1010         sort >"$_my_tickles"
1011
1012         # Add tickles for connections that we haven't already got tickles for
1013         comm -23 "$_my_connections" "$_my_tickles" |
1014         while read _src _dst ; do
1015                 ctdb addtickle $_src $_dst
1016         done
1017
1018         # Remove tickles for connections that are no longer there
1019         comm -13 "$_my_connections" "$_my_tickles" |
1020         while read _src _dst ; do
1021                 ctdb deltickle $_src $_dst
1022         done
1023
1024         rm -f "$_my_connections" "$_my_tickles"
1025 }
1026
1027 ########################################################
1028 # load a site local config file
1029 ########################################################
1030
1031 [ -x $CTDB_BASE/rc.local ] && {
1032         . $CTDB_BASE/rc.local
1033 }
1034
1035 [ -d $CTDB_BASE/rc.local.d ] && {
1036         for i in $CTDB_BASE/rc.local.d/* ; do
1037                 [ -x "$i" ] && . "$i"
1038         done
1039 }
1040
1041 script_name="${0##*/}"       # basename
1042 service_name="$script_name"  # default is just the script name
1043 service_fail_limit=1
1044 event_name="$1"