From dac075129b93dd0ecf0c59189f8e8ef294ab65c4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 9 Jun 2017 14:34:56 +1000 Subject: [PATCH] ctdb-scripts: Compact server-end TCP connection killing output When thousands of connections are being killed the logs are flooded with information about connections that should be killed. When some connections are not killed then the number not killed is printed. This is the wrong way around! When debugging "fail-back" problems, it is important to know details of connections that were *not* killed. It is almost never important to know the full list of all connections that were *supposed* to be killed. Instead, print a summary showing how many connections of the total were killed. If any were not killed then print a list of remaining connections. Update unit tests: infrastructure for fake TCP connections, existing, test cases, add new test cases. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs --- ctdb/config/functions | 22 +++++++---- .../10.interface.releaseip.010.sh | 24 ++++-------- .../10.interface.releaseip.011.sh | 39 +++++++------------ .../10.interface.releaseip.012.sh | 30 ++++++++++++++ .../10.interface.releaseip.013.sh | 35 +++++++++++++++++ ctdb/tests/eventscripts/scripts/local.sh | 39 ++++++++++++++++++- ctdb/tests/eventscripts/stubs/ss | 4 ++ 7 files changed, 141 insertions(+), 52 deletions(-) create mode 100755 ctdb/tests/eventscripts/10.interface.releaseip.012.sh create mode 100755 ctdb/tests/eventscripts/10.interface.releaseip.013.sh diff --git a/ctdb/config/functions b/ctdb/config/functions index f4539685137..2c630bd11be 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -414,7 +414,6 @@ kill_tcp_connections () 139|445) __oneway=true ;; esac - echo "Killing TCP connection $_src $_dst" _connections="${_connections}${_nl}${_src} ${_dst}" if ! $__oneway ; then _connections="${_connections}${_nl}${_dst} ${_src}" @@ -433,15 +432,22 @@ kill_tcp_connections () return } - _remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l) - - if [ "$_remaining" -eq 0 ] ; then - echo "Killed $_killcount TCP connections to released IP $_ip" - return + _connections=$(get_tcp_connections_for_ip "$_ip") + if [ -z "$_connections" ] ; then + _remaining=0 + else + _remaining=$(echo "$_connections" | wc -l) fi - _t="${_remaining}/${_killcount}" - echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)" + _actually_killed=$((_killcount - _remaining)) + + _t="${_actually_killed}/${_killcount}" + echo "Killed ${_t} TCP connections to released IP $_ip" + + if [ -n "$_connections" ] ; then + echo "Remaining connections:" + echo "$_connections" | sed -e 's|^| |' + fi } } diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.010.sh b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh index b6d9c7a8bd6..095e85c9b7c 100755 --- a/ctdb/tests/eventscripts/10.interface.releaseip.010.sh +++ b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh @@ -8,25 +8,15 @@ setup_ctdb ctdb_get_1_public_address | while read dev ip bits ; do - ip addr add "${ip}/${bits}" dev "$dev" + ip addr add "${ip}/${bits}" dev "$dev" - # Setup 10 fake connections... - count=10 - out="" - nl=" -" - i=0 - while [ $i -lt $count ] ; do - echo "${ip}:445 10.254.254.1:1230${i}" - # Expected output for killing this connection - out="${out}${out:+${nl}}Killing TCP connection 10.254.254.1:1230${i} ${ip}:445" - i=$(($i + 1)) - done >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" + count=10 + setup_tcp_connections $count \ + "$ip" 445 10.254.254.0 12300 - ok <"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" - - # Note that the fake TCP killing done by the "ctdb killtcp" stub - # can only kill conections in the file, so killing this connection - # will never succeed so it will look like a time out. - FAKE_NETSTAT_TCP_ESTABLISHED="${ip}:445|10.254.254.1:43210" - - ok <"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" +} + +setup_tcp_connections_unkillable () +{ + # These connections are listed by the "ss" stub but are not + # killed by the "ctdb killtcp" stub. So killing these + # connections will never succeed... and will look like a time + # out. + _t=$(_tcp_connections "$@" | sed -e 's/ /|/g') + export FAKE_NETSTAT_TCP_ESTABLISHED="$_t" +} + shares_missing () { _fmt="$1" ; shift diff --git a/ctdb/tests/eventscripts/stubs/ss b/ctdb/tests/eventscripts/stubs/ss index e8d804481df..1a3db0d0467 100755 --- a/ctdb/tests/eventscripts/stubs/ss +++ b/ctdb/tests/eventscripts/stubs/ss @@ -77,6 +77,10 @@ ss_tcp_established () echo 0 0 "$src" "$dst" fi done + + if [ -z "$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" ] ; then + return + fi while read src dst ; do if filter_socket "$srcs" "$sports" "$src" ; then echo 0 0 "$src" "$dst" -- 2.34.1