# utility functions for ctdb event scripts
-[ -z "$CTDB_VARDIR" ] && {
- if [ -d "/var/lib/ctdb" ] ; then
- export CTDB_VARDIR="/var/lib/ctdb"
- else
- export CTDB_VARDIR="/var/ctdb"
- fi
-}
-[ -z "$CTDB_ETCDIR" ] && {
- export CTDB_ETCDIR="/etc"
-}
+if [ -z "$CTDB_BASE" ] ; then
+ echo 'CTDB_BASE unset in CTDB functions file'
+ exit 1
+fi
+
+CTDB_VARDIR="/usr/local/var/lib/ctdb"
+ctdb_rundir="/usr/local/var/run/ctdb"
+
+# Only (and always) override these variables in test code
+
+if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
+ CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
+fi
+
+if [ -z "$CTDB_SYS_ETCDIR" ] ; then
+ CTDB_SYS_ETCDIR="/etc"
+fi
#######################################
# pull in a system config file, if any
+
+rewrite_ctdb_options ()
+{
+ case "$CTDB_DBDIR" in
+ tmpfs|tmpfs:*)
+ _opts_defaults="mode=700"
+ # Get any extra options specified after colon
+ if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
+ _opts=""
+ else
+ _opts="${CTDB_DBDIR#tmpfs:}"
+ fi
+ # This is an internal variable, only used by ctdbd_wrapper.
+ # It is OK to repeat mount options - last value wins
+ CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"
+
+ CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
+ ;;
+ *)
+ CTDB_DBDIR_TMPFS_OPTIONS=""
+ esac
+}
+
_loadconfig() {
if [ -z "$1" ] ; then
return
fi
- if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
- . $CTDB_ETCDIR/sysconfig/$1
- elif [ -f $CTDB_ETCDIR/default/$1 ]; then
- . $CTDB_ETCDIR/default/$1
+ if [ -f $CTDB_SYS_ETCDIR/sysconfig/$1 ]; then
+ . $CTDB_SYS_ETCDIR/sysconfig/$1
+ elif [ -f $CTDB_SYS_ETCDIR/default/$1 ]; then
+ . $CTDB_SYS_ETCDIR/default/$1
elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
. $CTDB_BASE/sysconfig/$1
fi
if [ -r "$_config" ] ; then
. "$_config"
fi
+ rewrite_ctdb_options
fi
}
if [ -n "$CTDB_LOGGING" ] ; then
_file="${CTDB_LOGGING#file:}"
else
- _file="/var/log/log.ctdb"
+ _file="/usr/local/var/log/log.ctdb"
fi
{
if [ -n "$*" ] ; then
$_nice /sbin/service "$_service_name" "$_op"
elif [ -x /usr/sbin/service ]; then
$_nice /usr/sbin/service "$_service_name" "$_op"
- elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
- $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
- elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
- $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
+ elif [ -x $CTDB_SYS_ETCDIR/init.d/$_service_name ]; then
+ $_nice $CTDB_SYS_ETCDIR/init.d/$_service_name "$_op"
+ elif [ -x $CTDB_SYS_ETCDIR/rc.d/init.d/$_service_name ]; then
+ $_nice $CTDB_SYS_ETCDIR/rc.d/init.d/$_service_name "$_op"
fi
}
# This sets $pnn - this avoid an unnecessary subprocess.
ctdb_get_pnn ()
{
- _pnn_file="$CTDB_VARDIR/state/my-pnn"
+ _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
if [ ! -f "$_pnn_file" ] ; then
ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
fi
done
}
-######################################################
-# Check that an RPC service is healthy -
-# this includes allowing a certain number of failures
-# before marking the NFS service unhealthy.
-#
-# usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
-#
-# each triple is a set of 3 arguments: an operator, a
-# fail count limit and an action string.
-#
-# For example:
-#
-# nfs_check_rpc_service "lockd" \
-# -ge 15 "verbose restart unhealthy" \
-# -eq 10 "restart:bs"
-#
-# says that if lockd is down for 15 iterations then do
-# a verbose restart of lockd and mark the node unhealthy.
-# Before this, after 10 iterations of failure, the
-# service is restarted silently in the background.
-# Order is important: the number of failures need to be
-# specified in reverse order because processing stops
-# after the first condition that is true.
-######################################################
-nfs_check_rpc_service ()
-{
- _prog_name="$1" ; shift
-
- if _nfs_check_rpc_common "$_prog_name" ; then
- return
- fi
-
- while [ -n "$3" ] ; do
- if _nfs_check_rpc_action "$1" "$2" "$3" ; then
- break
- fi
- shift 3
- done
-}
-
-# The new way of doing things...
-nfs_check_rpc_services ()
-{
- # Files must end with .check - avoids editor backups, RPM fu, ...
- for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
- _t="${_f%.check}"
- _prog_name="${_t##*/[0-9][0-9].}"
-
- # If $_prog_name contains '@' then the bit after it is the
- # address family.
- _family="${_prog_name#*@}"
- if [ "$_family" = "$_prog_name" ] ; then
- _family=""
- else
- _prog_name="${_prog_name%@*}"
- fi
-
- if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
- # This RPC service is up, check next service...
- continue
- fi
-
- # Check each line in the file in turn until one of the limit
- # checks is hit...
- while read _cmp _lim _rest ; do
- # Skip comments
- case "$_cmp" in
- \#*) continue ;;
- esac
-
- if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
- # Limit was hit on this line, no further checking...
- break
- fi
- done <"$_f"
- done
-}
-
-_nfs_check_rpc_common ()
-{
- _prog_name="$1"
- _family="$2"
-
- # Some platforms don't have separate programs for all services.
- case "$_prog_name" in
- statd)
- type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
- esac
-
- case "$_prog_name" in
- nfsd)
- _rpc_prog=nfs
- _version=3
- ;;
- mountd)
- _rpc_prog=mountd
- _version=1
- ;;
- rquotad)
- _rpc_prog=rquotad
- _version=1
- ;;
- lockd)
- _rpc_prog=nlockmgr
- _version=4
- ;;
- statd)
- _rpc_prog=status
- _version=1
- ;;
- *)
- echo "Internal error: unknown RPC program \"$_prog_name\"."
- exit 1
- esac
-
- _service_name="nfs_${_prog_name}${_family:+_}${_family}"
-
- if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
- ctdb_counter_init "$_service_name"
- return 0
- fi
-
- ctdb_counter_incr "$_service_name"
-
- return 1
-}
-
-_nfs_check_rpc_action ()
-{
- _cmp="$1"
- _limit="$2"
- _actions="$3"
-
- if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
- return 1
- fi
-
- for _action in $_actions ; do
- case "$_action" in
- verbose)
- echo "ERROR: $ctdb_check_rpc_out"
- ;;
- restart)
- _nfs_restart_rpc_service "$_prog_name"
- ;;
- restart:b)
- _nfs_restart_rpc_service "$_prog_name" true
- ;;
- unhealthy)
- exit 1
- ;;
- *)
- echo "Internal error: unknown action \"$_action\"."
- exit 1
- esac
- done
-
- return 0
-}
-
-_nfs_restart_rpc_service ()
-{
- _prog_name="$1"
- _background="${2:-false}"
-
- if $_background ; then
- _maybe_background="background_with_logging"
- else
- _maybe_background=""
- fi
-
- _p="rpc.${_prog_name}"
-
- case "$_prog_name" in
- nfsd)
- echo "Trying to restart NFS service"
- $_maybe_background startstop_nfs restart
- ;;
- mountd)
- echo "Trying to restart $_prog_name [${_p}]"
- killall -q -9 "$_p"
- nfs_dump_some_threads "$_p"
- $_maybe_background $_p $RPCMOUNTDOPTS \
- ${MOUNTD_PORT:+-p} $MOUNTD_PORT
- ;;
- rquotad)
- echo "Trying to restart $_prog_name [${_p}]"
- killall -q -9 "$_p"
- nfs_dump_some_threads "$_p"
- $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
- ;;
- lockd)
- echo "Trying to restart lock manager service"
- $_maybe_background startstop_nfslock restart
- ;;
- statd)
- echo "Trying to restart $_prog_name [${_p}]"
- killall -q -9 "$_p"
- nfs_dump_some_threads "$_p"
- $_maybe_background $_p \
- ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
- ${STATD_PORT:+-p} $STATD_PORT \
- ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
- ;;
- *)
- echo "Internal error: unknown RPC program \"$_prog_name\"."
- exit 1
- esac
-}
-
-######################################################
-# Check the health of NFS services
-#
-# Use .check files in given directory.
-# Default is "${CTDB_BASE}/nfs-checks.d/"
-######################################################
-nfs_check_services ()
-{
- _dir="${1:-${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}}"
-
- # Files must end with .check - avoids editor backups, RPM fu, ...
- for _f in "$_dir"/[0-9][0-9].*.check ; do
- _t="${_f%.check}"
- _progname="${_t##*/[0-9][0-9].}"
-
- nfs_check_service "$_progname" <"$_f"
- done
-}
-
-######################################################
-# Check the health of an NFS service
-#
-# $1 - progname, passed to rpcinfo (looked up in /etc/rpc)
-#
-# Reads variables from stdin
-#
-# Variables are:
-#
-# * family - "tcp" or "udp" or space separated list
-# default: tcp
-# * version - optional, RPC service version number
-# default is to omit to check for any version
-# * unhealthy_after - number of check fails before unhealthy
-# default: 1
-# * restart_every - number of check fails before restart
-# default: 0, meaning no restart
-# * service_stop_cmd - command to stop service
-# default: no default, must be provided if
-# restart_every > 0
-# * service_start_cmd - command to start service
-# default: no default, must be provided if
-# restart_every > 0
-# * service_debug_cmd - command to debug a service after trying to stop it;
-# for example, it can be useful to print stack
-# traces of threads that have not exited, since
-# they may be stuck doing I/O;
-# no default, see also function program_stack_traces()
-#
-# Quoting in values is not preserved
-#
-######################################################
-nfs_check_service ()
-{
- _progname="$1"
-
- (
- # Subshell to restrict scope variables...
-
- # Defaults
- family="tcp"
- version=""
- unhealthy_after=1
- restart_every=0
- service_stop_cmd=""
- service_start_cmd=""
- service_debug_cmd=""
-
- # Eval line-by-line. Expands variable references in values.
- # Also allows variable name checking, which seems useful.
- while read _line ; do
- case "$_line" in
- \#*|"") : ;; # Ignore comments, blank lines
-
- family=*|version=*|\
- unhealthy_after=*|restart_every=*|\
- service_stop_cmd=*|service_start_cmd=*|\
- service_debug_cmd=*)
-
- eval "$_line"
- ;;
- *)
- echo "ERROR: Unknown variable for ${_progname}: ${_line}"
- exit 1
- esac
- done
-
- _service_name="nfs_${_progname}"
-
- if nfs_check_rpcinfo \
- "$_progname" "$version" "$family" >/dev/null ; then
- if [ $unhealthy_after -ne 1 -o $restart_every -ne 0 ] ; then
- ctdb_counter_init "$_service_name"
- fi
- exit 0
- fi
-
- ctdb_counter_incr "$_service_name"
- _failcount=$(ctdb_counter_get "$_service_name")
-
- _unhealthy=false
- if [ $unhealthy_after -gt 0 ] ; then
- if [ $_failcount -ge $unhealthy_after ] ; then
- _unhealthy=true
- echo "ERROR: $ctdb_check_rpc_out"
- fi
- fi
-
- if [ $restart_every -gt 0 ] ; then
- if [ $(($_failcount % $restart_every)) -eq 0 ] ; then
- if ! $_unhealthy ; then
- echo "WARNING: $ctdb_check_rpc_out"
- fi
- nfs_restart_service
- fi
- fi
-
- if $_unhealthy ; then
- exit 1
- fi
-
- return 0
- ) || exit 1
-}
-
-# Uses: stop_service, start_service, debug_stuck_threads
-nfs_restart_service ()
-{
- if [ -z "$service_stop_cmd" -o -z "$service_start_cmd" ] ; then
- die "ERROR: Can not restart service \"${_progname}\" without corresponding service_start_cmd/service_stop_cmd settings"
- fi
-
- echo "Trying to restart service \"${_progname}\"..."
- # Using eval means variables can contain semicolon separated commands
- eval "$service_stop_cmd"
- if [ -n "$service_debug_cmd" ] ; then
- eval "$service_debug_cmd"
- fi
- background_with_logging eval "$service_start_cmd"
-}
-
-######################################################
-# Check an RPC service with rpcinfo
-######################################################
-ctdb_check_rpc ()
-{
- _progname="$1" # passed to rpcinfo (looked up in /etc/rpc)
- _version="$2" # optional, not passed if empty/unset
- _family="${3:-tcp}" # optional, default is "tcp"
-
- _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
-
- if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost \
- $_progname $_version 2>&1) ; then
- ctdb_check_rpc_out="$_progname failed RPC check:
-$ctdb_check_rpc_out"
- echo "$ctdb_check_rpc_out"
- return 1
- fi
-}
-
-nfs_check_rpcinfo ()
-{
- _progname="$1" # passed to rpcinfo (looked up in /etc/rpc)
- _versions="$2" # optional, space separated, not passed if empty/unset
- _families="${3:-tcp}" # optional, space separated, default is "tcp"
-
- for _family in $_families ; do
- if [ -n "$_versions" ] ; then
- for _version in $_versions ; do
- ctdb_check_rpc $_progname $_version $_family || return $?
- done
- else
- ctdb_check_rpc $_progname "" $_family || return $?
- fi
- done
-}
-
######################################################
# Ensure $service_name is set
assert_service_name ()
_ctdb_check_tcp_common ()
{
assert_service_name
- _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
+ _d="${CTDB_SCRIPT_VARDIR}/failcount"
+ _ctdb_service_started_file="${_d}/${service_name}.started"
}
ctdb_check_tcp_init ()
{print $4" "$5}'
}
-##################################################################
-# use statd-callout to update NFS lock info
-##################################################################
-nfs_update_lock_info ()
-{
- if [ -x "$CTDB_BASE/statd-callout" ] ; then
- "$CTDB_BASE/statd-callout" update
- fi
-}
-
-########################################################
-# start/stop the Ganesha nfs service
-########################################################
-startstop_ganesha()
-{
- _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
- case "$1" in
- start)
- service "$_service_name" start
- ;;
- stop)
- service "$_service_name" stop
- ;;
- restart)
- service "$_service_name" stop
- nfs_dump_some_threads "rpc.statd"
- service "$_service_name" start
- ;;
- esac
-}
-
-########################################################
-# start/stop the nfs service on different platforms
-########################################################
-startstop_nfs() {
- PLATFORM="unknown"
- [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
- PLATFORM="sles"
- }
- [ -x $CTDB_ETCDIR/init.d/nfslock -o \
- -r /usr/lib/systemd/system/nfs-lock.service ] && {
- PLATFORM="rhel"
- }
-
- case $PLATFORM in
- sles)
- case $1 in
- start)
- service nfsserver start
- ;;
- stop)
- service nfsserver stop > /dev/null 2>&1
- ;;
- restart)
- set_proc "fs/nfsd/threads" 0
- service nfsserver stop > /dev/null 2>&1
- pkill -9 nfsd
- nfs_dump_some_threads
- service nfsserver start
- ;;
- esac
- ;;
- rhel)
- case $1 in
- start)
- service nfslock start
- service nfs start
- ;;
- stop)
- service nfs stop
- service nfslock stop
- ;;
- restart)
- set_proc "fs/nfsd/threads" 0
- service nfs stop > /dev/null 2>&1
- service nfslock stop > /dev/null 2>&1
- pkill -9 nfsd
- nfs_dump_some_threads
- service nfslock start
- service nfs start
- ;;
- esac
- ;;
- *)
- echo "Unknown platform. NFS is not supported with ctdb"
- exit 1
- ;;
- esac
-}
-
-# Dump up to the configured number of nfsd thread backtraces.
-nfs_dump_some_threads ()
-{
- _prog="${1:-nfsd}"
-
- _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
- [ $_num -gt 0 ] || return 0
-
- program_stack_traces "$_prog" $_num
-}
-
-########################################################
-# start/stop the nfs lockmanager service on different platforms
-########################################################
-startstop_nfslock() {
- PLATFORM="unknown"
- [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
- PLATFORM="sles"
- }
- [ -x $CTDB_ETCDIR/init.d/nfslock -o \
- -r /usr/lib/systemd/system/nfs-lock.service ] && {
- PLATFORM="rhel"
- }
-
- case $PLATFORM in
- sles)
- # for sles there is no service for lockmanager
- # so we instead just shutdown/restart nfs
- case $1 in
- start)
- service nfsserver start
- ;;
- stop)
- service nfsserver stop > /dev/null 2>&1
- ;;
- restart)
- service nfsserver stop > /dev/null 2>&1
- service nfsserver start
- ;;
- esac
- ;;
- rhel)
- case $1 in
- start)
- service nfslock start
- ;;
- stop)
- service nfslock stop > /dev/null 2>&1
- ;;
- restart)
- service nfslock stop > /dev/null 2>&1
- service nfslock start
- ;;
- esac
- ;;
- *)
- echo "Unknown platform. NFS locking is not supported with ctdb"
- exit 1
- ;;
- esac
-}
-
########################################################
add_ip_to_iface ()
# Simple counters
_ctdb_counter_common () {
_service_name="${1:-${service_name:-${script_name}}}"
- _counter_file="$ctdb_fail_dir/$_service_name"
+ _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
mkdir -p "${_counter_file%/*}" # dirname
}
ctdb_counter_init () {
########################################################
-ctdb_status_dir="$CTDB_VARDIR/state/service_status"
-ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
-
ctdb_setup_service_state_dir ()
{
- service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
+ service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${1:-${service_name}}"
mkdir -p "$service_state_dir" || {
echo "Error creating state dir \"$service_state_dir\""
exit 1
########################################################
# Managed status history, for auto-start/stop
-ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
-
_ctdb_managed_common ()
{
- _ctdb_managed_file="$ctdb_managed_dir/$service_name"
+ _ctdb_managed_file="${CTDB_SCRIPT_VARDIR}/managed_history/${service_name}"
}
ctdb_service_managed ()
{
_ctdb_managed_common
- mkdir -p "$ctdb_managed_dir"
+ mkdir -p "${_ctdb_managed_file%/*}" # dirname
touch "$_ctdb_managed_file"
}
[ -f "$_ctdb_managed_file" ]
}
-########################################################
-# Check and set status
-
-log_status_cat ()
-{
- echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
-}
-
-ctdb_checkstatus ()
-{
- if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
- log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
- return 1
- elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
- log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
- return 2
- else
- return 0
- fi
-}
-
-ctdb_setstatus ()
-{
- d="$ctdb_status_dir/$script_name"
- case "$1" in
- unhealthy|banned)
- mkdir -p "$d"
- cat "$2" >"$d/$1"
- ;;
- *)
- for i in "banned" "unhealthy" ; do
- rm -f "$d/$i"
- done
- ;;
- esac
-}
-
##################################################################
# Reconfigure a service on demand
_ctdb_service_reconfigure_common ()
{
- _d="$ctdb_status_dir/${service_name}"
+ _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
mkdir -p "$_d"
_ctdb_service_reconfigure_flag="$_d/reconfigure"
}
ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
- ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
t=" $CTDB_MANAGED_SERVICES "
ctdb_standard_event_handler ()
{
- case "$1" in
- status)
- ctdb_checkstatus
- exit
- ;;
- setstatus)
- shift
- ctdb_setstatus "$@"
- exit
- ;;
- esac
+ :
}
iptables_wrapper ()
fi
# iptables doesn't like being re-entered, so flock-wrap it.
- flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
+ flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
}
# AIX (and perhaps others?) doesn't have mktemp
{
_port="$1"
- tickledir="$CTDB_VARDIR/state/tickles"
+ tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
mkdir -p "$tickledir"
ctdb_get_pnn
# Record connections to our public IPs in a temporary file
_my_connections="${tickledir}/${_port}.connections"
- rm -f "$_my_connections"
netstat -tn |
awk -v destpat="^${_ipschoice}:${_port}\$" \
'$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
# Record our current tickles in a temporary file
_my_tickles="${tickledir}/${_port}.tickles"
- rm -f "$_my_tickles"
for _i in $_ips ; do
ctdb -X gettickles $_i $_port |
awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'