ctdb/tests/eventscripts/scripts/60.nfs.sh

   1 setup_nfs ()
   2 {
   3         setup_ctdb
   4
   5         service_name="nfs"
   6
   7         export FAKE_RPCINFO_SERVICES=""
   8
   9         export CTDB_NFS_SKIP_SHARE_CHECK="no"
  10
  11         export RPCNFSDCOUNT
  12
  13         # This doesn't even need to exist
  14         export CTDB_NFS_EXPORTS_FILE="$EVENTSCRIPTS_TESTS_VAR_DIR/etc-exports"
  15
  16         # Reset the failcounts for nfs services.
  17         eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
  18
  19         if [ "$1" != "down" ] ; then
  20                 debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB"
  21
  22                 service "nfs" force-started
  23                 service "nfslock" force-started
  24
  25                 export CTDB_MANAGES_NFS="yes"
  26
  27                 rpc_services_up \
  28                         "portmapper" "nfs" "mountd" "rquotad" "nlockmgr" "status"
  29
  30                 nfs_setup_fake_threads "nfsd"
  31                 nfs_setup_fake_threads "rpc.foobar"  # Just set the variable to empty
  32         else
  33                 debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
  34
  35                 service "nfs" force-stopped
  36                 service "nfslock" force-stopped
  37
  38                 export CTDB_MANAGES_NFS=""
  39         fi
  40
  41         # This is really nasty.  However, when we test NFS we don't
  42         # actually test statd-callout. If we leave it there then left
  43         # over, backgrounded instances of statd-callout will do
  44         # horrible things with the "ctdb ip" stub and cause the actual
  45         # statd-callout tests that follow to fail.
  46         rm "${CTDB_BASE}/statd-callout"
  47 }
  48
  49 rpc_services_down ()
  50 {
  51         for _i ; do
  52             debug "Marking RPC service \"${_i}\" as unavailable"
  53             FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g")
  54         done
  55 }
  56
  57 rpc_services_up ()
  58 {
  59         for _i ; do
  60                 debug "Marking RPC service \"${_i}\" as available"
  61                 case "$_i" in
  62                 portmapper) _t="2:4" ;;
  63                 nfs)        _t="2:3" ;;
  64                 mountd)     _t="1:3" ;;
  65                 rquotad)    _t="1:2" ;;
  66                 nlockmgr)   _t="3:4" ;;
  67                 status)     _t="1:1" ;;
  68                 *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
  69                 esac
  70
  71                 FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
  72         done
  73 }
  74
  75 nfs_setup_fake_threads ()
  76 {
  77         _prog="$1" ; shift
  78
  79         case "$_prog" in
  80         nfsd)
  81                 export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
  82                 _threads="${PROCFS_PATH}/fs/nfsd/threads"
  83                 mkdir -p $(dirname "$_threads")
  84                 echo $# >"$_threads"
  85                 export FAKE_NFSD_THREAD_PIDS="$*"
  86                 ;;
  87         *)
  88                 export FAKE_RPC_THREAD_PIDS="$*"
  89                 ;;
  90         esac
  91 }
  92
  93 program_stack_traces ()
  94 {
  95         _prog="$1"
  96         _max="${2:-1}"
  97
  98         _count=1
  99         for _pid in ${FAKE_NFSD_THREAD_PIDS:-$FAKE_RPC_THREAD_PIDS} ; do
 100                 [ $_count -le $_max ] || break
 101
 102                 program_stack_trace "$_prog" "$_pid"
 103                 _count=$(($_count + 1))
 104         done
 105 }
 106
 107 guess_output ()
 108 {
 109         case "$1" in
 110         $CTDB_NFS_CALLOUT\ start\ nlockmgr)
 111                 echo "&Starting nfslock: OK"
 112                 ;;
 113         $CTDB_NFS_CALLOUT\ start\ nfs)
 114                 cat <<EOF
 115 &Starting nfslock: OK
 116 &Starting nfs: OK
 117 EOF
 118                 ;;
 119         *)
 120                 : # Nothing
 121         esac
 122 }
 123
 124 # Set the required result for a particular RPC program having failed
 125 # for a certain number of iterations.  This is probably still a work
 126 # in progress.  Note that we could hook aggressively
 127 # nfs_check_rpc_service() to try to implement this but we're better
 128 # off testing nfs_check_rpc_service() using independent code...  even
 129 # if it is incomplete and hacky.  So, if the 60.nfs eventscript
 130 # changes and the tests start to fail then it may be due to this
 131 # function being incomplete.
 132 rpc_set_service_failure_response ()
 133 {
 134         _rpc_service="$1"
 135         _numfails="${2:-1}" # default 1
 136
 137         # Default
 138         ok_null
 139         if [ $_numfails -eq 0 ] ; then
 140                 return
 141         fi
 142
 143         nfs_load_config
 144
 145         # A handy newline.  :-)
 146         _nl="
 147 "
 148
 149         _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
 150
 151         _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check")
 152         [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique"
 153
 154         _out=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
 155         _rc_file=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
 156
 157         (
 158                 # Subshell to restrict scope variables...
 159
 160                 # Defaults
 161                 family="tcp"
 162                 version=""
 163                 unhealthy_after=1
 164                 restart_every=0
 165                 service_stop_cmd=""
 166                 service_start_cmd=""
 167                 service_check_cmd=""
 168                 service_debug_cmd=""
 169
 170                 # Don't bother syntax checking, eventscript does that...
 171                 . "$_file"
 172
 173                 # Just use the first version, or use default.  This is
 174                 # dumb but handles all the cases that we care about
 175                 # now...
 176                 if [ -n "$version" ] ; then
 177                         _ver="${version%% *}"
 178                 else
 179                         case "$_rpc_service" in
 180                         portmapper) _ver="" ;;
 181                         *)          _ver=1  ;;
 182                         esac
 183                 fi
 184                 _rpc_check_out="\
 185 $_rpc_service failed RPC check:
 186 rpcinfo: RPC: Program not registered
 187 program $_rpc_service${_ver:+ version }${_ver} is not available"
 188
 189                 if [ $unhealthy_after -gt 0 -a $_numfails -ge $unhealthy_after ] ; then
 190                         _unhealthy=true
 191                         echo 1 >"$_rc_file"
 192                         echo "ERROR: ${_rpc_check_out}" >>"$_out"
 193                 else
 194                         _unhealthy=false
 195                         echo 0 >"$_rc_file"
 196                 fi
 197
 198                 if [ $restart_every -gt 0 ] && \
 199                            [ $(($_numfails % $restart_every)) -eq 0 ] ; then
 200                         if ! $_unhealthy ; then
 201                                 echo "WARNING: ${_rpc_check_out}" >>"$_out"
 202                         fi
 203
 204                         echo "Trying to restart service \"${_rpc_service}\"..." >>"$_out"
 205
 206                         if [ -n "$service_debug_cmd" ] ; then
 207                                 $service_debug_cmd 2>&1 >>"$_out"
 208                         fi
 209
 210                         guess_output "$service_start_cmd" >>"$_out"
 211                 fi
 212         )
 213
 214         read _rc <"$_rc_file"
 215         required_result $_rc <"$_out"
 216
 217         rm -f "$_out" "$_rc_file"
 218 }
 219
 220 # Run an NFS eventscript iteratively.
 221 #
 222 # - 1st argument is the number of iterations.
 223 #
 224 # - 2nd argument is the NFS/RPC service being tested
 225 #
 226 #   rpcinfo (or $service_check_cmd) is used on each iteration to test
 227 #   the availability of the service
 228 #
 229 #   If this is not set or null then no RPC service is checked and the
 230 #   required output is not reset on each iteration.  This is useful in
 231 #   baseline tests to confirm that the eventscript and test
 232 #   infrastructure is working correctly.
 233 #
 234 # - Subsequent arguments come in pairs: an iteration number and
 235 #   something to eval before that iteration.  Each time an iteration
 236 #   number is matched the associated argument is given to eval after
 237 #   the default setup is done.  The iteration numbers need to be given
 238 #   in ascending order.
 239 #
 240 #   These arguments can allow a service to be started or stopped
 241 #   before a particular iteration.
 242 #
 243 nfs_iterate_test ()
 244 {
 245         _repeats="$1"
 246         _rpc_service="$2"
 247         if [ -n "$2" ] ; then
 248                 shift 2
 249         else
 250                 shift
 251         fi
 252
 253         echo "Running $_repeats iterations of \"$script $event\" $args"
 254
 255         _iterate_failcount=0
 256         for _iteration in $(seq 1 $_repeats) ; do
 257                 # This is not a numerical comparison because $1 will
 258                 # often not be set.
 259                 if [ "$_iteration" = "$1" ] ; then
 260                         debug "##################################################"
 261                         eval "$2"
 262                         debug "##################################################"
 263                         shift 2
 264                 fi
 265                 if [ -n "$_rpc_service" ] ; then
 266                         _ok=false
 267                         if [ -n "$service_check_cmd" ] ; then
 268                                 if eval "$service_check_cmd" ; then
 269                                         _ok=true
 270                                 fi
 271                         else
 272                                 if rpcinfo -T tcp localhost "$_rpc_service" >/dev/null 2>&1 ; then
 273                                         _ok=true
 274                                 fi
 275                         fi
 276
 277                         if $_ok ; then
 278                                 _iterate_failcount=0
 279                         else
 280                                 _iterate_failcount=$(($_iterate_failcount + 1))
 281                         fi
 282                         rpc_set_service_failure_response "$_rpc_service" $_iterate_failcount
 283                 fi
 284                 _out=$(simple_test 2>&1)
 285                 _ret=$?
 286                 if "$TEST_VERBOSE" || [ $_ret -ne 0 ] ; then
 287                         echo "##################################################"
 288                         echo "Iteration ${_iteration}:"
 289                         echo "$_out"
 290                 fi
 291                 if [ $_ret -ne 0 ] ; then
 292                         exit $_ret
 293                 fi
 294         done
 295 }