ctdb/tests/local_daemons.sh

   1 #!/bin/sh
   2
   3 set -u
   4
   5 export CTDB_TEST_MODE="yes"
   6
   7 # Following 2 lines may be modified by installation script
   8 CTDB_TESTS_ARE_INSTALLED=false
   9 CTDB_TEST_DIR=$(dirname "$0")
  10 export CTDB_TESTS_ARE_INSTALLED CTDB_TEST_DIR
  11
  12 export TEST_SCRIPTS_DIR="${CTDB_TEST_DIR}/scripts"
  13
  14 . "${TEST_SCRIPTS_DIR}/common.sh"
  15
  16 # common.sh will set TEST_SUBDIR to a stupid value when installed
  17 # because common.sh is usually sourced by a test.  TEST_SUBDIR needs
  18 # to be correctly set so setup_ctdb_base() finds the etc-ctdb/
  19 # subdirectory and the test event script is correctly installed, so
  20 # fix it.
  21 TEST_SUBDIR="$CTDB_TEST_DIR"
  22
  23 if ! $CTDB_TESTS_ARE_INSTALLED ; then
  24         hdir="$CTDB_SCRIPTS_HELPER_BINDIR"
  25         export CTDB_EVENTD="${hdir}/ctdb-eventd"
  26         export CTDB_EVENT_HELPER="${hdir}/ctdb-event"
  27         export CTDB_LOCK_HELPER="${hdir}/ctdb_lock_helper"
  28         export CTDB_RECOVERY_HELPER="${hdir}/ctdb_recovery_helper"
  29         export CTDB_TAKEOVER_HELPER="${hdir}/ctdb_takeover_helper"
  30         export CTDB_CLUSTER_MUTEX_HELPER="${hdir}/ctdb_mutex_fcntl_helper"
  31 fi
  32
  33 ########################################
  34
  35 # If the given IP is hosted then print 2 items: maskbits and iface
  36 have_ip ()
  37 {
  38         _addr="$1"
  39
  40         case "$_addr" in
  41         *:*) _bits=128 ;;
  42         *)   _bits=32  ;;
  43         esac
  44
  45         _t=$(ip addr show to "${_addr}/${_bits}")
  46         [ -n "$_t" ]
  47 }
  48
  49 setup_nodes ()
  50 {
  51         _num_nodes="$1"
  52         _use_ipv6="$2"
  53
  54         _have_all_ips=true
  55         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  56                 if $_use_ipv6 ; then
  57                         _j=$(printf "%04x" $((0x5f00 + 1 + _i)) )
  58                         _node_ip="fd00::5357:${_j}"
  59                         if have_ip "$_node_ip" ; then
  60                                 echo "$_node_ip"
  61                         else
  62                                 cat >&2 <<EOF
  63 ERROR: ${_node_ip} not on an interface, please add it
  64 EOF
  65                                 _have_all_ips=false
  66                         fi
  67                 else
  68                         _c=$(( _i / 100 ))
  69                         _d=$(( 1 + (_i % 100) ))
  70                         echo "127.0.${_c}.${_d}"
  71                 fi
  72         done
  73
  74         # Fail if we don't have all of the IPv6 addresses assigned
  75         $_have_all_ips
  76 }
  77
  78 setup_public_addresses ()
  79 {
  80         _num_nodes="$1"
  81         _node_no_ips="$2"
  82         _use_ipv6="$3"
  83
  84         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  85                 if  [ "$_i" -eq "$_node_no_ips" ] ; then
  86                         continue
  87                 fi
  88
  89                 # 2 public addresses on most nodes, just to make
  90                 # things interesting
  91                 if $_use_ipv6 ; then
  92                         printf 'fc00:10::1:%x/64 lo\n' $((1 + _i))
  93                         printf 'fc00:10::2:%x/64 lo\n' $((1 + _i))
  94                 else
  95                         _c1=$(( 100 + (_i / 100) ))
  96                         _c2=$(( 200 + (_i / 100) ))
  97                         _d=$(( 1 + (_i % 100) ))
  98                         printf '192.168.%d.%d/24 lo\n' "$_c1" "$_d"
  99                         printf '192.168.%d.%d/24 lo\n' "$_c2" "$_d"
 100                 fi
 101         done
 102 }
 103
 104 setup_socket_wrapper ()
 105 {
 106         _socket_wrapper_so="$1"
 107
 108         _so="${directory}/libsocket-wrapper.so"
 109         if [ ! -f "$_socket_wrapper_so" ] ; then
 110                 die "$0 setup: Unable to find ${_socket_wrapper_so}"
 111         fi
 112
 113         # Find absoluate path if only relative is given
 114         case "$_socket_wrapper_so" in
 115         /*) : ;;
 116         *) _socket_wrapper_so="${PWD}/${_socket_wrapper_so}" ;;
 117         esac
 118
 119         rm -f "$_so"
 120         ln -s "$_socket_wrapper_so" "$_so"
 121
 122         _d="${directory}/sw"
 123         rm -rf "$_d"
 124         mkdir -p "$_d"
 125 }
 126
 127 local_daemons_setup_usage ()
 128 {
 129         cat >&2 <<EOF
 130 $0 <directory> setup [ <options>... ]
 131
 132 Options:
 133   -F            Disable failover (default: failover enabled)
 134   -N <file>     Nodes file (default: automatically generated)
 135   -n <num>      Number of nodes (default: 3)
 136   -P <file>     Public addresses file (default: automatically generated)
 137   -R            Use a command for the recovery lock (default: use a file)
 138   -S <library>  Socket wrapper shared library to preload (default: none)
 139   -6            Generate IPv6 IPs for nodes, public addresses (default: IPv4)
 140 EOF
 141
 142         exit 1
 143 }
 144
 145 local_daemons_setup ()
 146 {
 147         _disable_failover=false
 148         _nodes_file=""
 149         _num_nodes=3
 150         _public_addresses_file=""
 151         _recovery_lock_use_command=false
 152         _socket_wrapper=""
 153         _use_ipv6=false
 154
 155         set -e
 156
 157         while getopts "FN:n:P:RS:6h?" _opt ; do
 158                 case "$_opt" in
 159                 F) _disable_failover=true ;;
 160                 N) _nodes_file="$OPTARG" ;;
 161                 n) _num_nodes="$OPTARG" ;;
 162                 P) _public_addresses_file="$OPTARG" ;;
 163                 R) _recovery_lock_use_command=true ;;
 164                 S) _socket_wrapper="$OPTARG" ;;
 165                 6) _use_ipv6=true ;;
 166                 \?|h) local_daemons_setup_usage ;;
 167                 esac
 168         done
 169         shift $((OPTIND - 1))
 170
 171         mkdir -p "$directory"
 172
 173         _nodes_all="${directory}/nodes"
 174         if [ -n "$_nodes_file" ] ; then
 175                 cp "$_nodes_file" "$_nodes_all"
 176         else
 177                 setup_nodes "$_num_nodes" $_use_ipv6 >"$_nodes_all"
 178         fi
 179
 180         # If there are (strictly) greater than 2 nodes then we'll
 181         # "randomly" choose a node to have no public addresses
 182         _node_no_ips=-1
 183         if [ "$_num_nodes" -gt 2 ] ; then
 184                 _node_no_ips=$(($$ % _num_nodes))
 185         fi
 186
 187         _public_addresses_all="${directory}/public_addresses"
 188         if [ -n "$_public_addresses_file" ] ; then
 189                 cp "$_public_addresses_file" "$_public_addresses_all"
 190         else
 191                 setup_public_addresses "$_num_nodes" \
 192                                        $_node_no_ips \
 193                                        $_use_ipv6 >"$_public_addresses_all"
 194         fi
 195
 196         _recovery_lock="${directory}/rec.lock"
 197         if $_recovery_lock_use_command ; then
 198                 _helper="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb_mutex_fcntl_helper"
 199                 _recovery_lock="! ${_helper} ${_recovery_lock}"
 200         fi
 201
 202         if [ -n "$_socket_wrapper" ] ; then
 203                 setup_socket_wrapper "$_socket_wrapper"
 204         fi
 205
 206         for _n in $(seq 0 $((_num_nodes - 1))) ; do
 207                 setup_ctdb_base "$directory" "node.${_n}" \
 208                                 functions notify.sh debug-hung-script.sh
 209
 210                 cp "$_nodes_all" "${CTDB_BASE}/nodes"
 211
 212                 _public_addresses="${CTDB_BASE}/public_addresses"
 213
 214                 if  [ -z "$_public_addresses_file" ] && \
 215                             [ $_node_no_ips -eq "$_n" ] ; then
 216                         echo "Node ${_n} will have no public IPs."
 217                         : >"$_public_addresses"
 218                 else
 219                         cp "$_public_addresses_all" "$_public_addresses"
 220                 fi
 221
 222                 _node_ip=$(sed -n -e "$((_n + 1))p" "$_nodes_all")
 223
 224                 _db_dir="${CTDB_BASE}/db"
 225                 for _d in "volatile" "persistent" "state" ; do
 226                         mkdir -p "${_db_dir}/${_d}"
 227                 done
 228
 229                 cat >"${CTDB_BASE}/ctdb.conf" <<EOF
 230 [logging]
 231         location = file:${CTDB_BASE}/log.ctdb
 232         log level = INFO
 233
 234 [cluster]
 235         recovery lock = ${_recovery_lock}
 236         node address = ${_node_ip}
 237
 238 [database]
 239         volatile database directory = ${_db_dir}/volatile
 240         persistent database directory = ${_db_dir}/persistent
 241         state database directory = ${_db_dir}/state
 242
 243 [failover]
 244         disabled = ${_disable_failover}
 245
 246 [event]
 247         debug script = debug-hung-script.sh
 248 EOF
 249         done
 250 }
 251
 252 local_daemons_ssh_usage ()
 253 {
 254         cat >&2 <<EOF
 255 usage: $0 <directory> ssh [ -n ] <ip> <command>
 256 EOF
 257
 258         exit 1
 259 }
 260
 261 local_daemons_ssh ()
 262 {
 263         if [ $# -lt 2 ] ; then
 264                 local_daemons_ssh_usage
 265         fi
 266
 267         # Only try to respect ssh -n option, others can't be used so discard them
 268         _close_stdin=false
 269         while getopts "nh?" _opt ; do
 270                 case "$_opt" in
 271                 n) _close_stdin=true ;;
 272                 \?|h) local_daemons_ssh_usage ;;
 273                 *) : ;;
 274                 esac
 275         done
 276         shift $((OPTIND - 1))
 277
 278         if [ $# -lt 2 ] ; then
 279                 local_daemons_ssh_usage
 280         fi
 281
 282         _nodes="${directory}/nodes"
 283
 284         # IP adress of node. onnode can pass hostnames but not in these tests
 285         _ip="$1" ; shift
 286         # "$*" is command
 287
 288
 289         # Determine the correct CTDB base directory
 290         _num=$(awk -v ip="$_ip" '$1 == ip { print NR }' "$_nodes")
 291         _node=$((_num - 1))
 292         export CTDB_BASE="${directory}/node.${_node}"
 293
 294         if [ ! -d "$CTDB_BASE" ] ; then
 295                 die "$0 ssh: Unable to find base for node ${_ip}"
 296         fi
 297
 298         if $_close_stdin ; then
 299                 exec sh -c "$*" <&-
 300         else
 301                 exec sh -c "$*"
 302         fi
 303 }
 304
 305 onnode_common ()
 306 {
 307         # onnode will execute this, which fakes ssh against local daemons
 308         export ONNODE_SSH="${0} ${directory} ssh"
 309
 310         # onnode just needs the nodes file, so use the common one
 311         export CTDB_BASE="$directory"
 312 }
 313
 314 local_daemons_generic_usage ()
 315 {
 316         cat >&2 <<EOF
 317 usage: $0 <directory> ${1} <nodes>
 318
 319 <nodes> can be  "all", a node number or any specification supported by onnode
 320 EOF
 321
 322         exit 1
 323 }
 324
 325 local_daemons_start_socket_wrapper ()
 326 {
 327         _so="${directory}/libsocket-wrapper.so"
 328         _d="${directory}/sw"
 329
 330         if [ -d "$_d" ] && [ -f "$_so" ] ; then
 331                 export SOCKET_WRAPPER_DIR="$_d"
 332                 export LD_PRELOAD="$_so"
 333         fi
 334 }
 335
 336 local_daemons_start ()
 337 {
 338         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 339                 local_daemons_generic_usage "start"
 340         fi
 341
 342         local_daemons_start_socket_wrapper
 343
 344         _nodes="$1"
 345
 346         onnode_common
 347
 348         onnode "$_nodes" "${VALGRIND:-} ctdbd &"
 349 }
 350
 351 local_daemons_stop ()
 352 {
 353         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 354                 local_daemons_generic_usage "stop"
 355         fi
 356
 357         _nodes="$1"
 358
 359         onnode_common
 360
 361         onnode -p "$_nodes" "${VALGRIND:-} ${CTDB:-ctdb} shutdown"
 362 }
 363
 364 local_daemons_onnode_usage ()
 365 {
 366         cat >&2 <<EOF
 367 usage: $0 <directory> onnode <nodes> <command>...
 368
 369 <nodes> can be  "all", a node number or any specification supported by onnode
 370 EOF
 371
 372         exit 1
 373 }
 374
 375 local_daemons_onnode ()
 376 {
 377         if [ $# -lt 2 ] || [ "$1" = "-h" ] ; then
 378                 local_daemons_onnode_usage
 379         fi
 380
 381         _nodes="$1"
 382         shift
 383
 384         onnode_common
 385
 386         onnode "$_nodes" "$@"
 387 }
 388
 389 local_daemons_print_socket ()
 390 {
 391         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 392                 local_daemons_generic_usage "print-socket"
 393         fi
 394
 395         _nodes="$1"
 396         shift
 397
 398         onnode_common
 399
 400         _path="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb-path"
 401         onnode -q "$_nodes" "${VALGRIND:-} ${_path} socket ctdbd"
 402 }
 403
 404 local_daemons_print_log ()
 405 {
 406         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 407                 local_daemons_generic_usage "print-log"
 408         fi
 409
 410         _nodes="$1"
 411         shift
 412
 413         onnode_common
 414
 415         # shellcheck disable=SC2016
 416         # $CTDB_BASE must only be expanded under onnode, not in top-level shell
 417         onnode -q "$_nodes" 'echo ${CTDB_BASE}/log.ctdb' |
 418         while IFS='' read -r _l ; do
 419                 _dir=$(dirname "$_l")
 420                 _node=$(basename "$_dir")
 421                 # Add fake hostname after date and time, which are the
 422                 # first 2 words on each line
 423                 sed -e "s|^\\([^ ][^ ]* [^ ][^ ]*\\)|\\1 ${_node}|" "$_l"
 424         done |
 425         sort
 426
 427 }
 428
 429 usage ()
 430 {
 431         cat <<EOF
 432 usage: $0 <directory> <command> [ <options>... ]
 433
 434 Commands:
 435   setup          Set up daemon configuration according to given options
 436   start          Start specified daemon(s)
 437   stop           Stop specified daemon(s)
 438   onnode         Run a command in the environment of specified daemon(s)
 439   print-socket   Print the Unix domain socket used by specified daemon(s)
 440   print-log      Print logs for specified daemon(s) to stdout
 441
 442 All commands use <directory> for daemon configuration
 443
 444 Run command with -h option to see per-command usage
 445 EOF
 446
 447         exit 1
 448 }
 449
 450 if [ $# -lt 2 ] ; then
 451         usage
 452 fi
 453
 454 directory="$1"
 455 command="$2"
 456 shift 2
 457
 458 case "$command" in
 459 setup) local_daemons_setup "$@" ;;
 460 ssh) local_daemons_ssh "$@" ;; # Internal, not shown by usage()
 461 start) local_daemons_start "$@" ;;
 462 stop) local_daemons_stop "$@" ;;
 463 onnode) local_daemons_onnode "$@" ;;
 464 print-socket) local_daemons_print_socket "$@" ;;
 465 print-log) local_daemons_print_log "$@" ;;
 466 *) usage ;;
 467 esac