3 # This must run as root as CTDB tool commands need to access CTDB socket
4 [ "$(id -u)" -eq 0 ] || exec sudo "$0" "$@"
6 # this script needs to be installed so that statd points to it with the -H
7 # command line argument. The easiest way to do that is to put something like this in
9 # STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
11 [ -n "$CTDB_BASE" ] || \
12 CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; dirname "$PWD")
14 . "${CTDB_BASE}/functions"
16 # Overwrite this so we get some logging
19 script_log "statd-callout" "$@"
26 [ -n "$NFS_HOSTNAME" ] || \
27 die "NFS_HOSTNAME is not configured. statd-callout failed"
33 ctdb_setup_service_state_dir "statd-callout"
35 cd "$service_state_dir" || \
36 die "Failed to change directory to \"${service_state_dir}\""
39 # Keep a single file to keep track of the last "add-client" or
40 # "del-client'. These get pushed to ctdb.tdb during "update",
41 # which will generally be run once each "monitor" cycle. In this
42 # way we avoid scalability problems with flood of persistent
43 # transactions after a "notify" when all the clients re-take their
47 # statd does not tell us to which IP the client connected so
48 # we must add it to all the IPs that we serve
52 # x is intentionally ignored
53 # shellcheck disable=SC2034
56 while IFS="|" read x sip node x ; do
57 [ "$node" = "$pnn" ] || continue # not us
58 key="statd-state@${sip}@${cip}"
59 echo "\"${key}\" \"${date}\"" >"$key"
64 # statd does not tell us from which IP the client disconnected
65 # so we must add it to all the IPs that we serve
68 # x is intentionally ignored
69 # shellcheck disable=SC2034
72 while IFS="|" read x sip node x ; do
73 [ "$node" = "$pnn" ] || continue # not us
74 key="statd-state@${sip}@${cip}"
75 echo "\"${key}\" \"\"" >"$key"
80 files=$(echo statd-state@*)
81 if [ "$files" = "statd-state@*" ] ; then
85 # Filter out lines for any IP addresses that are not currently
86 # hosted public IP addresses.
88 ctdb_ips=$($CTDB ip | tail -n +2)
89 sed_expr=$(echo "$ctdb_ips" |
90 awk -v pnn="$pnn" 'pnn == $2 { \
91 ip = $1; gsub(/\./, "\\.", ip); \
92 printf "/statd-state@%s@/p\n", ip }')
93 # Intentional multi-word expansion for multiple files
94 # shellcheck disable=SC2086
95 if cat $files | sed -n "$sed_expr" | $CTDB ptrans "ctdb.tdb" ; then
101 # we must restart the lockmanager (on all nodes) so that we get
102 # a clusterwide grace period (so other clients don't take out
103 # conflicting locks through other nodes before all locks have been
106 # we need these settings to make sure that no tcp connections survive
107 # across a very fast failover/failback
108 #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
109 #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
110 #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
112 # Delete the notification list for statd, we don't want it to
114 rm -f /var/lib/nfs/statd/sm/*
115 rm -f /var/lib/nfs/statd/sm.bak/*
117 # we must keep a monotonically increasing state variable for the entire
118 # cluster so state always increases when ip addresses fail from one
120 # We use epoch and hope the nodes are close enough in clock.
121 # Even numbers mean service is shut down, odd numbers mean
122 # service is started.
123 state_even=$(( $(date '+%s') / 2 * 2))
125 # We must also let some time pass between stopping and
126 # restarting the lock manager. Otherwise there is a window
127 # where the lock manager will respond "strangely" immediately
128 # after restarting it, which causes clients to fail to reclaim
130 "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
132 "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
134 # we now need to send out additional statd notifications to ensure
135 # that clients understand that the lockmanager has restarted.
136 # we have three cases:
137 # 1, clients that ignore the ip address the stat notification came from
138 # and ONLY care about the 'name' in the notify packet.
139 # these clients ONLY work with lock failover IFF that name
140 # can be resolved into an ipaddress that matches the one used
141 # to mount the share. (==linux clients)
142 # This is handled when starting lockmanager above, but those
143 # packets are sent from the "wrong" ip address, something linux
144 # clients are ok with, buth other clients will barf at.
145 # 2, Some clients only accept statd packets IFF they come from the
146 # 'correct' ip address.
147 # 2a,Send out the notification using the 'correct' ip address and also
148 # specify the 'correct' hostname in the statd packet.
149 # Some clients require both the correct source address and also the
150 # correct name. (these clients also ONLY work if the ip addresses
151 # used to map the share can be resolved into the name returned in
152 # the notify packet.)
153 # 2b,Other clients require that the source ip address of the notify
154 # packet matches the ip address used to take out the lock.
155 # I.e. that the correct source address is used.
156 # These clients also require that the statd notify packet contains
157 # the name as the ip address used when the lock was taken out.
159 # Both 2a and 2b are commonly used in lockmanagers since they maximize
160 # probability that the client will accept the statd notify packet and
161 # not just ignore it.
162 # For all IPs we serve, collect info and push to the config database
165 # Construct a sed expression to take catdb output and produce pairs of:
166 # server-IP client-IP
167 # but only for the server-IPs that are hosted on this node.
168 ctdb_all_ips=$($CTDB ip all | tail -n +2)
169 sed_expr=$(echo "$ctdb_all_ips" |
170 awk -v pnn="$pnn" 'pnn == $2 { \
171 ip = $1; gsub(/\./, "\\.", ip); \
172 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
174 statd_state=$($CTDB catdb ctdb.tdb | sed -n "$sed_expr" | sort)
175 [ -n "$statd_state" ] || exit 0
177 smnotify="${CTDB_HELPER_BINDIR}/smnotify"
179 echo "$statd_state" | {
180 # This all needs to be in the same command group at the
181 # end of the pipe so it doesn't get lost when the loop
184 while read sip cip ; do
185 # Collect item to delete from the DB
186 key="statd-state@${sip}@${cip}"
187 item="\"${key}\" \"\""
188 items="${items}${items:+${nl}}${item}"
190 # NOTE: Consider optimising smnotify to read all the
191 # data from stdin and then run it in the background.
193 # Reset stateval for each serverip
194 [ "$sip" = "$prev" ] || stateval="$state_even"
195 # Send notifies for server shutdown
196 "$smnotify" --client="$cip" --ip="$sip" \
197 --server="$sip" --stateval="$stateval"
198 "$smnotify" --client="$cip" --ip="$sip" \
199 --server="$NFS_HOSTNAME" --stateval="$stateval"
200 # Send notifies for server startup
201 stateval=$((stateval + 1))
202 "$smnotify" --client="$cip" --ip="$sip" \
203 --server="$sip" --stateval="$stateval"
204 "$smnotify" --client="$cip" --ip="$sip" \
205 --server="$NFS_HOSTNAME" --stateval="$stateval"
208 echo "$items" | $CTDB ptrans "ctdb.tdb"
211 # Remove any stale touch files (i.e. for IPs not currently
212 # hosted on this node and created since the last "update").
213 # There's nothing else we can do with them at this stage.
214 echo "$ctdb_all_ips" |
215 awk -v pnn="$pnn" 'pnn != $2 { print $1 }' |
217 rm -f "statd-state@${sip}@"*