ctdb/tests/local_daemons.sh

   1 #!/bin/sh
   2
   3 set -u
   4
   5 export CTDB_TEST_MODE="yes"
   6
   7 # Following 2 lines may be modified by installation script
   8 export CTDB_TESTS_ARE_INSTALLED=false
   9 export CTDB_TEST_DIR=$(dirname "$0")
  10
  11 export TEST_SCRIPTS_DIR="${CTDB_TEST_DIR}/scripts"
  12
  13 . "${TEST_SCRIPTS_DIR}/common.sh"
  14
  15 # common.sh will set TEST_SUBDIR to a stupid value when installed
  16 # because common.sh is usually sourced by a test.  TEST_SUBDIR needs
  17 # to be correctly set so setup_ctdb_base() finds the etc-ctdb/
  18 # subdirectory and the test event script is correctly installed, so
  19 # fix it.
  20 TEST_SUBDIR="$CTDB_TEST_DIR"
  21
  22 if ! $CTDB_TESTS_ARE_INSTALLED ; then
  23         hdir="$CTDB_SCRIPTS_HELPER_BINDIR"
  24         export CTDB_EVENTD="${hdir}/ctdb-eventd"
  25         export CTDB_EVENT_HELPER="${hdir}/ctdb-event"
  26         export CTDB_LOCK_HELPER="${hdir}/ctdb_lock_helper"
  27         export CTDB_RECOVERY_HELPER="${hdir}/ctdb_recovery_helper"
  28         export CTDB_TAKEOVER_HELPER="${hdir}/ctdb_takeover_helper"
  29         export CTDB_CLUSTER_MUTEX_HELPER="${hdir}/ctdb_mutex_fcntl_helper"
  30 fi
  31
  32 ########################################
  33
  34 # If the given IP is hosted then print 2 items: maskbits and iface
  35 have_ip ()
  36 {
  37         _addr="$1"
  38
  39         case "$_addr" in
  40         *:*) _bits=128 ;;
  41         *)   _bits=32  ;;
  42         esac
  43
  44         _t=$(ip addr show to "${_addr}/${_bits}")
  45         [ -n "$_t" ]
  46 }
  47
  48 setup_nodes ()
  49 {
  50         _num_nodes="$1"
  51         _use_ipv6="$2"
  52
  53         _have_all_ips=true
  54         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  55                 if $_use_ipv6 ; then
  56                         _j=$(printf "%04x" $((0x5f00 + 1 + _i)) )
  57                         _node_ip="fd00::5357:${_j}"
  58                         if have_ip "$_node_ip" ; then
  59                                 echo "$_node_ip"
  60                         else
  61                                 cat >&2 <<EOF
  62 ERROR: ${_node_ip} not on an interface, please add it
  63 EOF
  64                                 _have_all_ips=false
  65                         fi
  66                 else
  67                         _c=$(( _i / 100 ))
  68                         _d=$(( 1 + (_i % 100) ))
  69                         echo "127.0.${_c}.${_d}"
  70                 fi
  71         done
  72
  73         # Fail if we don't have all of the IPv6 addresses assigned
  74         $_have_all_ips
  75 }
  76
  77 setup_public_addresses ()
  78 {
  79         _num_nodes="$1"
  80         _node_no_ips="$2"
  81         _use_ipv6="$3"
  82
  83         for _i in $(seq 0 $((_num_nodes - 1)) ) ; do
  84                 if  [ "$_i" -eq "$_node_no_ips" ] ; then
  85                         continue
  86                 fi
  87
  88                 # 2 public addresses on most nodes, just to make
  89                 # things interesting
  90                 if $_use_ipv6 ; then
  91                         printf 'fc00:10::1:%x/64 lo\n' $((1 + _i))
  92                         printf 'fc00:10::2:%x/64 lo\n' $((1 + _i))
  93                 else
  94                         _c1=$(( 100 + (_i / 100) ))
  95                         _c2=$(( 200 + (_i / 100) ))
  96                         _d=$(( 1 + (_i % 100) ))
  97                         printf '192.168.%d.%d/24 lo\n' "$_c1" "$_d"
  98                         printf '192.168.%d.%d/24 lo\n' "$_c2" "$_d"
  99                 fi
 100         done
 101 }
 102
 103 setup_socket_wrapper ()
 104 {
 105         _socket_wrapper_so="$1"
 106
 107         _so="${directory}/libsocket-wrapper.so"
 108         if [ ! -f "$_socket_wrapper_so" ] ; then
 109                 die "$0 setup: Unable to find ${_socket_wrapper_so}"
 110         fi
 111
 112         # Find absoluate path if only relative is given
 113         case "$_socket_wrapper_so" in
 114         /*) : ;;
 115         *) _socket_wrapper_so="${PWD}/${_socket_wrapper_so}" ;;
 116         esac
 117
 118         rm -f "$_so"
 119         ln -s "$_socket_wrapper_so" "$_so"
 120
 121         _d="${directory}/sw"
 122         rm -rf "$_d"
 123         mkdir -p "$_d"
 124 }
 125
 126 local_daemons_setup_usage ()
 127 {
 128         cat >&2 <<EOF
 129 $0 <directory> setup [ <options>... ]
 130
 131 Options:
 132   -F            Disable failover (default: failover enabled)
 133   -N <file>     Nodes file (default: automatically generated)
 134   -n <num>      Number of nodes (default: 3)
 135   -P <file>     Public addresses file (default: automatically generated)
 136   -R            Use a command for the recovery lock (default: use a file)
 137   -S <library>  Socket wrapper shared library to preload (default: none)
 138   -6            Generate IPv6 IPs for nodes, public addresses (default: IPv4)
 139 EOF
 140
 141         exit 1
 142 }
 143
 144 local_daemons_setup ()
 145 {
 146         _disable_failover=false
 147         _nodes_file=""
 148         _num_nodes=3
 149         _public_addresses_file=""
 150         _recovery_lock_use_command=false
 151         _socket_wrapper=""
 152         _use_ipv6=false
 153
 154         set -e
 155
 156         while getopts "FN:n:P:RS:6h?" _opt ; do
 157                 case "$_opt" in
 158                 F) _disable_failover=true ;;
 159                 N) _nodes_file="$OPTARG" ;;
 160                 n) _num_nodes="$OPTARG" ;;
 161                 P) _public_addresses_file="$OPTARG" ;;
 162                 R) _recovery_lock_use_command=true ;;
 163                 S) _socket_wrapper="$OPTARG" ;;
 164                 6) _use_ipv6=true ;;
 165                 \?|h) local_daemons_setup_usage ;;
 166                 esac
 167         done
 168         shift $((OPTIND - 1))
 169
 170         mkdir -p "$directory"
 171
 172         _nodes_all="${directory}/nodes"
 173         if [ -n "$_nodes_file" ] ; then
 174                 cp "$_nodes_file" "$_nodes_all"
 175         else
 176                 setup_nodes "$_num_nodes" $_use_ipv6 >"$_nodes_all"
 177         fi
 178
 179         # If there are (strictly) greater than 2 nodes then we'll
 180         # "randomly" choose a node to have no public addresses
 181         _node_no_ips=-1
 182         if [ "$_num_nodes" -gt 2 ] ; then
 183                 _node_no_ips=$(($$ % _num_nodes))
 184         fi
 185
 186         _public_addresses_all="${directory}/public_addresses"
 187         if [ -n "$_public_addresses_file" ] ; then
 188                 cp "$_public_addresses_file" "$_public_addresses_all"
 189         else
 190                 setup_public_addresses "$_num_nodes" \
 191                                        $_node_no_ips \
 192                                        $_use_ipv6 >"$_public_addresses_all"
 193         fi
 194
 195         _recovery_lock="${directory}/rec.lock"
 196         if $_recovery_lock_use_command ; then
 197                 _helper="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb_mutex_fcntl_helper"
 198                 _recovery_lock="! ${_helper} ${_recovery_lock}"
 199         fi
 200
 201         if [ -n "$_socket_wrapper" ] ; then
 202                 setup_socket_wrapper "$_socket_wrapper"
 203         fi
 204
 205         for _n in $(seq 0 $((_num_nodes - 1))) ; do
 206                 setup_ctdb_base "$directory" "node.${_n}" \
 207                                 functions notify.sh debug-hung-script.sh
 208
 209                 cp "$_nodes_all" "${CTDB_BASE}/nodes"
 210
 211                 _public_addresses="${CTDB_BASE}/public_addresses"
 212
 213                 if  [ -z "$_public_addresses_file" ] && \
 214                             [ $_node_no_ips -eq "$_n" ] ; then
 215                         echo "Node ${_n} will have no public IPs."
 216                         : >"$_public_addresses"
 217                 else
 218                         cp "$_public_addresses_all" "$_public_addresses"
 219                 fi
 220
 221                 _node_ip=$(sed -n -e "$((_n + 1))p" "$_nodes_all")
 222
 223                 _db_dir="${CTDB_BASE}/db"
 224                 for _d in "volatile" "persistent" "state" ; do
 225                         mkdir -p "${_db_dir}/${_d}"
 226                 done
 227
 228                 cat >"${CTDB_BASE}/ctdb.conf" <<EOF
 229 [logging]
 230         location = file:${CTDB_BASE}/log.ctdb
 231         log level = INFO
 232
 233 [cluster]
 234         recovery lock = ${_recovery_lock}
 235         node address = ${_node_ip}
 236
 237 [database]
 238         volatile database directory = ${_db_dir}/volatile
 239         persistent database directory = ${_db_dir}/persistent
 240         state database directory = ${_db_dir}/state
 241
 242 [failover]
 243         disabled = ${_disable_failover}
 244
 245 [event]
 246         debug script = debug-hung-script.sh
 247 EOF
 248         done
 249 }
 250
 251 local_daemons_ssh_usage ()
 252 {
 253         cat >&2 <<EOF
 254 usage: $0 <directory> ssh [ -n ] <ip> <command>
 255 EOF
 256
 257         exit 1
 258 }
 259
 260 local_daemons_ssh ()
 261 {
 262         if [ $# -lt 2 ] ; then
 263                 local_daemons_ssh_usage
 264         fi
 265
 266         # Only try to respect ssh -n option, others can't be used so discard them
 267         _close_stdin=false
 268         while getopts "nh?" _opt ; do
 269                 case "$_opt" in
 270                 n) _close_stdin=true ;;
 271                 \?|h) local_daemons_ssh_usage ;;
 272                 *) : ;;
 273                 esac
 274         done
 275         shift $((OPTIND - 1))
 276
 277         if [ $# -lt 2 ] ; then
 278                 local_daemons_ssh_usage
 279         fi
 280
 281         _nodes="${directory}/nodes"
 282
 283         # IP adress of node. onnode can pass hostnames but not in these tests
 284         _ip="$1" ; shift
 285         # "$*" is command
 286
 287
 288         # Determine the correct CTDB base directory
 289         _num=$(awk -v ip="$_ip" '$1 == ip { print NR }' "$_nodes")
 290         _node=$((_num - 1))
 291         export CTDB_BASE="${directory}/node.${_node}"
 292
 293         if [ ! -d "$CTDB_BASE" ] ; then
 294                 die "$0 ssh: Unable to find base for node ${_ip}"
 295         fi
 296
 297         if $_close_stdin ; then
 298                 exec sh -c "$*" <&-
 299         else
 300                 exec sh -c "$*"
 301         fi
 302 }
 303
 304 onnode_common ()
 305 {
 306         # onnode will execute this, which fakes ssh against local daemons
 307         export ONNODE_SSH="${0} ${directory} ssh"
 308
 309         # onnode just needs the nodes file, so use the common one
 310         export CTDB_BASE="$directory"
 311 }
 312
 313 local_daemons_generic_usage ()
 314 {
 315         cat >&2 <<EOF
 316 usage: $0 <directory> ${1} <nodes>
 317
 318 <nodes> can be  "all", a node number or any specification supported by onnode
 319 EOF
 320
 321         exit 1
 322 }
 323
 324 local_daemons_start_socket_wrapper ()
 325 {
 326         _so="${directory}/libsocket-wrapper.so"
 327         _d="${directory}/sw"
 328
 329         if [ -d "$_d" ] && [ -f "$_so" ] ; then
 330                 export SOCKET_WRAPPER_DIR="$_d"
 331                 export LD_PRELOAD="$_so"
 332         fi
 333 }
 334
 335 local_daemons_start ()
 336 {
 337         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 338                 local_daemons_generic_usage "start"
 339         fi
 340
 341         local_daemons_start_socket_wrapper
 342
 343         _nodes="$1"
 344
 345         onnode_common
 346
 347         onnode "$_nodes" "${VALGRIND:-} ctdbd &"
 348 }
 349
 350 local_daemons_stop ()
 351 {
 352         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 353                 local_daemons_generic_usage "stop"
 354         fi
 355
 356         _nodes="$1"
 357
 358         onnode_common
 359
 360         onnode -p "$_nodes" "${VALGRIND:-} ${CTDB:-ctdb} shutdown"
 361 }
 362
 363 local_daemons_onnode_usage ()
 364 {
 365         cat >&2 <<EOF
 366 usage: $0 <directory> onnode <nodes> <command>...
 367
 368 <nodes> can be  "all", a node number or any specification supported by onnode
 369 EOF
 370
 371         exit 1
 372 }
 373
 374 local_daemons_onnode ()
 375 {
 376         if [ $# -lt 2 ] || [ "$1" = "-h" ] ; then
 377                 local_daemons_onnode_usage
 378         fi
 379
 380         _nodes="$1"
 381         shift
 382
 383         onnode_common
 384
 385         onnode "$_nodes" "$@"
 386 }
 387
 388 local_daemons_print_socket ()
 389 {
 390         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 391                 local_daemons_generic_usage "print-socket"
 392         fi
 393
 394         _nodes="$1"
 395         shift
 396
 397         onnode_common
 398
 399         _path="${CTDB_SCRIPTS_HELPER_BINDIR}/ctdb-path"
 400         onnode -q "$_nodes" "${VALGRIND:-} ${_path} socket ctdbd"
 401 }
 402
 403 local_daemons_print_log ()
 404 {
 405         if [ $# -ne 1 ] || [ "$1" = "-h" ] ; then
 406                 local_daemons_generic_usage "print-log"
 407         fi
 408
 409         _nodes="$1"
 410         shift
 411
 412         onnode_common
 413
 414         # shellcheck disable=SC2016
 415         # $CTDB_BASE must only be expanded under onnode, not in top-level shell
 416         onnode -q "$_nodes" 'echo ${CTDB_BASE}/log.ctdb' |
 417         while IFS='' read -r _l ; do
 418                 _dir=$(dirname "$_l")
 419                 _node=$(basename "$_dir")
 420                 # Add fake hostname after date and time, which are the
 421                 # first 2 words on each line
 422                 sed -e "s|^\\([^ ][^ ]* [^ ][^ ]*\\)|\\1 ${_node}|" "$_l"
 423         done |
 424         sort
 425
 426 }
 427
 428 usage ()
 429 {
 430         cat <<EOF
 431 usage: $0 <directory> <command> [ <options>... ]
 432
 433 Commands:
 434   setup          Set up daemon configuration according to given options
 435   start          Start specified daemon(s)
 436   stop           Stop specified daemon(s)
 437   onnode         Run a command in the environment of specified daemon(s)
 438   print-socket   Print the Unix domain socket used by specified daemon(s)
 439   print-log      Print logs for specified daemon(s) to stdout
 440
 441 All commands use <directory> for daemon configuration
 442
 443 Run command with -h option to see per-command usage
 444 EOF
 445
 446         exit 1
 447 }
 448
 449 if [ $# -lt 2 ] ; then
 450         usage
 451 fi
 452
 453 directory="$1"
 454 command="$2"
 455 shift 2
 456
 457 case "$command" in
 458 setup) local_daemons_setup "$@" ;;
 459 ssh) local_daemons_ssh "$@" ;; # Internal, not shown by usage()
 460 start) local_daemons_start "$@" ;;
 461 stop) local_daemons_stop "$@" ;;
 462 onnode) local_daemons_onnode "$@" ;;
 463 print-socket) local_daemons_print_socket "$@" ;;
 464 print-log) local_daemons_print_log "$@" ;;
 465 *) usage ;;
 466 esac