2 # standalone test for ctdb_mutex_ceph_rados_helper
4 # Copyright (C) David Disseldorp 2016
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, see <http://www.gnu.org/licenses/>.
19 # XXX The following parameters may require configuration:
20 CLUSTER="ceph" # Name of the Ceph cluster under test
21 USER="client.admin" # Ceph user - a keyring must exist
22 POOL="rbd" # RADOS pool - must exist
23 OBJECT="ctdb_reclock" # RADOS object: target for lock requests
26 # - using ctdb_mutex_ceph_rados_helper, take a lock on the Ceph RADOS object at
27 # CLUSTER/$POOL/$OBJECT using the Ceph keyring for $USER
28 # + confirm that lock is obtained, via ctdb_mutex_ceph_rados_helper "0" output
29 # - check RADOS object lock state, using the "rados lock info" command
30 # - attempt to obtain the lock again, using ctdb_mutex_ceph_rados_helper
31 # + confirm that the lock is not successfully taken ("1" output=contention)
32 # - tell the first locker to drop the lock and exit, via SIGTERM
33 # - once the first locker has exited, attempt to get the lock again
34 # + confirm that this attempt succeeds
41 # this test requires the Ceph "rados" binary, and "jq" json parser
42 which jq > /dev/null || exit 1
43 which rados > /dev/null || exit 1
44 which ctdb_mutex_ceph_rados_helper || exit 1
46 TMP_DIR="$(mktemp --directory)" || exit 1
47 rados -p "$POOL" rm "$OBJECT"
49 # explicitly disable lock expiry (duration=0), to ensure that we don't get
50 # intermittent failures (due to renewal) from the lock state diff further down
51 (ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" 0 \
55 # TODO wait for ctdb_mutex_ceph_rados_helper to write one byte to stdout,
56 # indicating lock acquisition success/failure
59 first_out=$(cat ${TMP_DIR}/first)
60 [ "$first_out" == "0" ] \
61 || _fail "expected lock acquisition (0), but got $first_out"
63 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
64 > ${TMP_DIR}/lock_state_first
66 # echo "with lock: `cat ${TMP_DIR}/lock_state_first`"
68 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_first)"
69 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
70 || _fail "unexpected lock name: $LOCK_NAME"
71 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_first)"
72 [ "$LOCK_TYPE" == "exclusive" ] \
73 || _fail "unexpected lock type: $LOCK_TYPE"
75 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_first)"
76 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
77 LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_first)"
78 [ "$LOCKER_COOKIE" == "ctdb_reclock_mutex" ] \
79 || _fail "unexpected locker cookie: $LOCKER_COOKIE"
80 LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_first)"
81 [ "$LOCKER_DESC" == "CTDB recovery lock" ] \
82 || _fail "unexpected locker description: $LOCKER_DESC"
83 LOCKER_EXP="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_first)"
84 [ "$LOCKER_EXP" == "0.000000" ] \
85 || _fail "unexpected locker expiration: $LOCKER_EXP"
87 # second attempt while first is still holding the lock - expect failure
88 ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \
90 second_out=$(cat ${TMP_DIR}/second)
91 [ "$second_out" == "1" ] \
92 || _fail "expected lock contention (1), but got $second_out"
94 # confirm lock state didn't change
95 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
96 > ${TMP_DIR}/lock_state_second
98 diff ${TMP_DIR}/lock_state_first ${TMP_DIR}/lock_state_second \
99 || _fail "unexpected lock state change"
101 # tell first locker to drop the lock and terminate
102 kill $locker_pid || exit 1
104 wait $locker_pid &> /dev/null
106 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
107 > ${TMP_DIR}/lock_state_third
108 # echo "without lock: `cat ${TMP_DIR}/lock_state_third`"
110 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_third)"
111 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
112 || _fail "unexpected lock name: $LOCK_NAME"
113 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_third)"
114 [ "$LOCK_TYPE" == "exclusive" ] \
115 || _fail "unexpected lock type: $LOCK_TYPE"
117 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_third)"
118 [ $LOCK_COUNT -eq 0 ] \
119 || _fail "didn\'t expect any locks in rados state, got $LOCK_COUNT"
121 exec >${TMP_DIR}/third -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" &
126 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
127 > ${TMP_DIR}/lock_state_fourth
128 # echo "with lock again: `cat ${TMP_DIR}/lock_state_fourth`"
130 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fourth)"
131 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
132 || _fail "unexpected lock name: $LOCK_NAME"
133 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fourth)"
134 [ "$LOCK_TYPE" == "exclusive" ] \
135 || _fail "unexpected lock type: $LOCK_TYPE"
137 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fourth)"
138 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
139 LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_fourth)"
140 [ "$LOCKER_COOKIE" == "ctdb_reclock_mutex" ] \
141 || _fail "unexpected locker cookie: $LOCKER_COOKIE"
142 LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_fourth)"
143 [ "$LOCKER_DESC" == "CTDB recovery lock" ] \
144 || _fail "unexpected locker description: $LOCKER_DESC"
146 kill $locker_pid || exit 1
147 wait $locker_pid &> /dev/null
149 third_out=$(cat ${TMP_DIR}/third)
150 [ "$third_out" == "0" ] \
151 || _fail "expected lock acquisition (0), but got $third_out"
153 # test renew / expire behaviour using a 1s expiry (update period = 500ms)
154 exec >${TMP_DIR}/forth -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" \
155 "$POOL" "$OBJECT" 1 &
160 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
161 > ${TMP_DIR}/lock_state_fifth_a
162 #echo "with lock fifth: `cat ${TMP_DIR}/lock_state_fifth_a`"
164 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fifth_a)"
165 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
166 || _fail "unexpected lock name: $LOCK_NAME"
167 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fifth_a)"
168 [ "$LOCK_TYPE" == "exclusive" ] \
169 || _fail "unexpected lock type: $LOCK_TYPE"
170 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fifth_a)"
171 [ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT"
172 LOCKER_EXP_A="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_a)"
173 [ "$LOCKER_EXP_A" != "0.000000" ] \
174 || _fail "unexpected locker expiration: $LOCKER_EXP_A"
175 sleep 1 # sleep until renewal
176 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
177 > ${TMP_DIR}/lock_state_fifth_b
178 LOCKER_EXP_B="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_b)"
179 [ "$LOCKER_EXP_B" != "0.000000" ] \
180 || _fail "unexpected locker expiration: $LOCKER_EXP_B"
181 #echo "lock expiration before renewal $LOCKER_EXP_A, after renewal $LOCKER_EXP_B"
182 [ "$LOCKER_EXP_B" != "$LOCKER_EXP_A" ] \
183 || _fail "locker expiration matches: $LOCKER_EXP_B"
185 # no chance to drop the lock, rely on expiry
186 kill -KILL $locker_pid || exit 1
187 wait $locker_pid &> /dev/null
188 sleep 1 # sleep until lock expiry
190 rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \
191 > ${TMP_DIR}/lock_state_sixth
192 #echo "lock expiry sixth: `cat ${TMP_DIR}/lock_state_sixth`"
194 LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_sixth)"
195 [ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \
196 || _fail "unexpected lock name: $LOCK_NAME"
197 LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_sixth)"
198 [ "$LOCK_TYPE" == "exclusive" ] \
199 || _fail "unexpected lock type: $LOCK_TYPE"
200 LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_sixth)"
201 [ $LOCK_COUNT -eq 0 ] || _fail "expected 0 locks in rados state, got $LOCK_COUNT"
206 echo "$0: all tests passed"