ctdb-scripts: Change statd-callout to be more scalable
authorMartin Schwenke <martin@meltin.net>
Fri, 13 Feb 2015 09:55:43 +0000 (20:55 +1100)
committerAmitay Isaacs <amitay@samba.org>
Wed, 4 Mar 2015 09:42:27 +0000 (10:42 +0100)
Updating ctdb.tdb on each add-client, del-client and each delete
during notify was too ambitious.  Persistent transactions do not
perform well enough to do this.

Revert to having add-client and del-client create touch files.  Each
monitor event calls "statd-callout update" to convert touch files into
ctdb.tdb records.

Update testcases to do the "update" and add an extra test.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Pair-programmed-with: Amitay Isaacs <amitay@gmail.com>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
ctdb/config/events.d/60.ganesha
ctdb/config/events.d/60.nfs
ctdb/config/functions
ctdb/config/statd-callout
ctdb/tests/eventscripts/statd-callout.001.sh
ctdb/tests/eventscripts/statd-callout.002.sh
ctdb/tests/eventscripts/statd-callout.003.sh
ctdb/tests/eventscripts/statd-callout.004.sh
ctdb/tests/eventscripts/statd-callout.005.sh
ctdb/tests/eventscripts/statd-callout.006.sh
ctdb/tests/eventscripts/statd-callout.007.sh [new file with mode: 0755]

index 150be1f15942f33bd070a6788dd8331ed05b7a25..fc170983dc5a79bc815bb7a95f255cc2554069a9 100755 (executable)
@@ -221,6 +221,7 @@ case "$1" in
        } || exit $?
 
        update_tickles 2049
+       nfs_update_lock_info
 
        # check that statd responds to rpc requests
        # if statd is not running we try to restart it
index c4e798e2ca9ba4a202c49fe13c5650368a2259de..d570cd784b70ab3978f2af6d17976cb58facb064 100755 (executable)
@@ -91,6 +91,7 @@ case "$1" in
        } || exit $?
 
        update_tickles 2049
+       nfs_update_lock_info
 
        nfs_check_rpc_services
 
index 8c5e472f1a44fdb2f3c474fd44f0ea4dfde6d23e..da726222cbd5af61941c451300d16207f6d5f6be 100755 (executable)
@@ -690,6 +690,16 @@ get_tcp_connections_for_ip ()
         {print $4" "$5}'
 }
 
+##################################################################
+# use statd-callout to update NFS lock info
+##################################################################
+nfs_update_lock_info ()
+{
+    if [ -x "$CTDB_BASE/statd-callout" ] ; then
+       "$CTDB_BASE/statd-callout" update
+    fi
+}
+
 ########################################################
 # start/stop the Ganesha nfs service
 ########################################################
index b92942557f3e7fde985b0980bc038ad85d2f7f7f..4a331aca2cf267bebe533f58100eb9b80d45f06d 100755 (executable)
@@ -30,7 +30,19 @@ loadconfig nfs
 nl="
 "
 
+ctdb_setup_service_state_dir "statd-callout"
+
+cd "$service_state_dir" || \
+    die "Failed to change directory to \"${service_state_dir}\""
+
 case "$1" in
+    # Keep a single file to keep track of the last "add-client" or
+    # "del-client'.  These get pushed to ctdb.tdb during "update",
+    # which will generally be run once each "monitor" cycle.  In this
+    # way we avoid scalability problems with flood of persistent
+    # transactions after a "notify" when all the clients re-take their
+    # locks.
+
     add-client)
        # statd does not tell us to which IP the client connected so
        # we must add it to all the IPs that we serve
@@ -38,42 +50,47 @@ case "$1" in
        pnn=$(ctdb xpnn | sed -e 's/.*://')
        date=$(date '+%s')
        ctdb ip -X |
-       tail -n +2 | {
-           # This all needs to be in the end of the pipe so it
-           # doesn't get lost
-           items=""
-           while IFS="|" read x sip node x ; do
-               [ "$node" = "$pnn" ] || continue # not us
-               key="statd-state@${sip}@${cip}"
-               item="\"${key}\" \"${date}\""
-               items="${items}${items:+${nl}}${item}"
-           done
-           if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
-               die "Failed to add clients"
-           fi
-       }
+       tail -n +2 |
+       while IFS="|" read x sip node x ; do
+           [ "$node" = "$pnn" ] || continue # not us
+           key="statd-state@${sip}@${cip}"
+           echo "\"${key}\" \"${date}\"" >"$key"
+       done
        ;;
-  del-client)
+
+    del-client)
        # statd does not tell us from which IP the client disconnected
        # so we must add it to all the IPs that we serve
        cip="$2"
        pnn=$(ctdb xpnn | sed -e 's/.*://')
        ctdb ip -X |
-       tail -n +2 | {
-           # This all needs to be in the end of the pipe so it
-           # doesn't get lost
-           items=""
-           while IFS="|" read x sip node x ; do
-               [ "$node" = "$pnn" ] || continue # not us
-               key="statd-state@${sip}@${cip}"
-               item="\"${key}\" \"\""
-               items="${items}${items:+${nl}}${item}"
-           done
-           if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then
-               die "Failed to delete clients"
-           fi
-       }
+       tail -n +2 |
+       while IFS="|" read x sip node x ; do
+           [ "$node" = "$pnn" ] || continue # not us
+           key="statd-state@${sip}@${cip}"
+           echo "\"${key}\" \"\"" >"$key"
+       done
        ;;
+
+    update)
+        files=$(echo statd-state@*)
+       if [ "$files" = "statd-state@*" ] ; then
+           # No files!
+           exit 0
+       fi
+       # Filter out lines for any IP addresses that are not currently
+       # hosted public IP addresses.
+       pnn=$(ctdb xpnn | sed -e 's/.*://')
+       ctdb_ips=$(ctdb ip | tail -n +2)
+       sed_expr=$(echo "$ctdb_ips" |
+           awk -v pnn=$pnn 'pnn == $2 { \
+                ip = $1; gsub(/\./, "\\.", ip); \
+                printf "/statd-state@%s@/p\n", ip }')
+        if cat $files | sed -n "$sed_expr" | ctdb ptrans "ctdb.tdb" ; then
+            rm $files
+       fi
+        ;;
+
     notify)
        # we must restart the lockmanager (on all nodes) so that we get
        # a clusterwide grace period (so other clients dont take out
@@ -144,7 +161,8 @@ case "$1" in
        # Construct a sed expression to take catdb output and produce pairs of:
        #   server-IP client-IP
        # but only for the server-IPs that are hosted on this node.
-       sed_expr=$(ctdb ip | tail -n +2 |
+       ctdb_all_ips=$(ctdb ip -n all | tail -n +2)
+       sed_expr=$(echo "$ctdb_all_ips" |
            awk -v pnn=$pnn 'pnn == $2 { \
                 ip = $1; gsub(/\./, "\\.", ip); \
                 printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
@@ -152,34 +170,42 @@ case "$1" in
        statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
        [ -n "$statd_state" ] || exit 0
 
-       # The following is dangerous if this script times out before
-       # all of the smnotify commands are run.  Revert to individual
-       # pdelete commands for now and consider optimising smnotify to
-       # read all the data from stdin and then run it in the
-       # background.
-       #
-       # Delete all the items from the TDB
-       #if ! echo "$statd_state" | \
-       #    awk '{ printf "\"statd-state@%s@%s\" \"\"\n", $1, $2 }') | \
-       #    ctdb ptrans ctdb.tdb ; then
+       prev=""
+       echo "$statd_state" | {
+           # This all needs to be in the same command group at the
+           # end of the pipe so it doesn't get lost when the loop
+           # completes.
+           items=""
+           while read sip cip ; do
+               # Collect item to delete from the DB
+               key="statd-state@${sip}@${cip}"
+               item="\"${key}\" \"\""
+               items="${items}${items:+${nl}}${item}"
 
-       #    die "Yikes!"
-       #fi
+               # NOTE: Consider optimising smnotify to read all the
+               # data from stdin and then run it in the background.
+
+               # Reset stateval for each serverip
+               [ "$sip" = "$prev" ] || stateval="$state_even"
+               # Send notifies for server shutdown
+               smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
+               smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
+               # Send notifies for server startup
+               stateval=$(($stateval + 1))
+               smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
+               smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
+           done
 
-       prev=""
-       echo "$statd_state" |
-       while read sip cip ; do
-           # Delete the entry from the DB
-           ctdb pdelete ctdb.tdb "statd-state@${sip}@${cip}"
-           # Reset stateval for each serverip
-           [ "$sip" = "$prev" ] || stateval="$state_even"
-           # Send notifies for server shutdown
-           smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
-           smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
-           # Send notifies for server startup
-           stateval=$(($stateval + 1))
-           smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval
-           smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
-       done
+           echo "$items" | ctdb ptrans "ctdb.tdb"
+       }
+
+       # Remove any stale touch files (i.e. for IPs not currently
+       # hosted on this node and created since the last "update").
+       # There's nothing else we can do with them at this stage.
+       echo "$ctdb_all_ips" |
+           awk -v pnn=$pnn 'pnn != $2 { print $1 }' |
+           while read sip ; do
+               rm -f "statd-state@${sip}@"*
+           done
        ;;
 esac
index 5f7b7e281be2ff6149e92e06f060e9e22a2be3df..29b9fbc8bfc70e24c0df353529b2555a581d3f89 100755 (executable)
@@ -10,5 +10,6 @@ FAKE_DATE_OUTPUT="1234565789"
 
 ok_null
 simple_test_event "add-client" "192.168.123.45"
+simple_test_event "update"
 
 check_ctdb_tdb_statd_state "192.168.123.45"
index f8778f7e98ad2933a37a9afbf70c1b3ad090cf3e..009da1b0d9d453adbcf31d37f463e6ddee29de48 100755 (executable)
@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "2 x add-client"
+define_test "2 x add-client, update"
 
 setup_ctdb
 
@@ -11,5 +11,6 @@ FAKE_DATE_OUTPUT="1234565789"
 ok_null
 simple_test_event "add-client" "192.168.123.45"
 simple_test_event "add-client" "192.168.123.46"
+simple_test_event "update"
 
 check_ctdb_tdb_statd_state "192.168.123.45" "192.168.123.46"
index 1319ee409961ae8ec4a1c132c9e5006c76ef8b3e..ed28de683e030205f34b776071c229727970106e 100755 (executable)
@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "add-client, del-client"
+define_test "add-client, update, del-client, update"
 
 setup_ctdb
 
@@ -10,6 +10,9 @@ FAKE_DATE_OUTPUT="1234565789"
 
 ok_null
 simple_test_event "add-client" "192.168.123.45"
+simple_test_event "update"
+
 simple_test_event "del-client" "192.168.123.45"
+simple_test_event "update"
 
 check_ctdb_tdb_statd_state
index 5702b85290c7d924a47ad384ba9fa738dda5cf82..011ced9578f774cebe0539afd305765885fbd527 100755 (executable)
@@ -10,6 +10,7 @@ FAKE_DATE_OUTPUT="1234565789"
 
 ok_null
 simple_test_event "add-client" "192.168.123.45"
+simple_test_event "update"
 
 check_ctdb_tdb_statd_state "192.168.123.45"
 
index 65e291d08846217b1d523e7105f2a594dc121959..ceb4445fb759d1502f26478659e2c5b92094ca3b 100755 (executable)
@@ -10,11 +10,13 @@ FAKE_DATE_OUTPUT="1234565789"
 
 ok_null
 simple_test_event "add-client" "192.168.123.45"
+simple_test_event "update"
 
 FAKE_CTDB_PNN=1
 
 ok_null
 simple_test_event "add-client" "192.168.123.46"
+simple_test_event "update"
 
 FAKE_CTDB_PNN=0
 
index df8af8876bb9066946d3c3b3aa3061ca5e520b44..0db86fee0ff13da17c18487016b46f0a88e66cfc 100755 (executable)
@@ -10,11 +10,13 @@ FAKE_DATE_OUTPUT="1234565789"
 
 ok_null
 simple_test_event "add-client" "192.168.123.45"
+simple_test_event "update"
 
 FAKE_CTDB_PNN=1
 
 ok_null
 simple_test_event "add-client" "192.168.123.46"
+simple_test_event "update"
 
 FAKE_CTDB_PNN=0
 
diff --git a/ctdb/tests/eventscripts/statd-callout.007.sh b/ctdb/tests/eventscripts/statd-callout.007.sh
new file mode 100755 (executable)
index 0000000..32339cd
--- /dev/null
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "add-client, del-client, update"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+simple_test_event "del-client" "192.168.123.45"
+simple_test_event "update"
+
+check_ctdb_tdb_statd_state