scripts: Add helper script to log locking information using /proc/locks

This finds any processes locking tdb databases used by CTDB and logs
stack trace for each process.

Includes this fix from 1.2.40 branch:

  scripts: Fix the variable name for sed expressions

Signed-off-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 0e99ca1cdf28a7043554afb78bd439f727ab4f95)
(cherry picked from commit 9fbd13ea7d3da5e297827e7763f336f484262f47)

diff --git a/Makefile.in b/Makefile.in
index 9f55278a245e7bbac3e81a76457d455ccbc08ac2..298bd7e8b8a2b8e4ee78f30f39d0c6ffe67658f4 100755 (executable)
--- a/Makefile.in
+++ b/Makefile.in
@@ -246,6 +246,7 @@ install: all
        ${INSTALLCMD} -m 644 config/functions $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 755 config/interface_modify.sh $(DESTDIR)$(etcdir)/ctdb
+       ${INSTALLCMD} -m 755 config/debug_locks.sh $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts
        ${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt
        ${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d

diff --git a/config/debug_locks.sh b/config/debug_locks.sh
new file mode 100755 (executable)
index 0000000..91cb405
--- /dev/null
+++ b/config/debug_locks.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Create sed expression to convert inodes to names
+sed_cmd=$( ls -li /var/ctdb/*.tdb.* /var/ctdb/persistent/*.tdb.* |
+          sed -e "s#/var/ctdb[/persistent]*/\(.*\)#\1#" |
+          awk '{printf "s#[0-9]*:[0-9]*:%s #%s #\n", $1, $10}' )
+
+# Parse /proc/locks and extract following information
+#    pid process_name tdb_name offsets [W]
+out=$( cat /proc/locks |
+    grep -F "POSIX  ADVISORY  WRITE" |
+    awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' |
+    while read pid rest ; do
+       pname=$(readlink /proc/$pid/exe)
+       echo $pid $pname $rest
+    done | sed -e "$sed_cmd" | grep "\.tdb" )
+
+if [ -n "$out" ]; then
+    # Log information about locks
+    echo "$out" | logger -t "debug-lock"
+
+    # Find processes that are waiting for locks
+    dbs=$(echo "$out" | grep "W$" | awk '{print $3}')
+    all_pids=""
+    for db in $dbs ; do
+       pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}')
+       all_pids="$all_pids $pids"
+    done
+    pids=$(echo $all_pids | sort -u)
+
+    # For each process waiting, log stack trace
+    for pid in $pids ; do
+       gstack $pid | logger -t "debug-lock $pid"
+#      gcore -o /var/log/core-deadlock-ctdb $pid
+    done
+fi
+
+exit 0

diff --git a/packaging/RPM/ctdb.spec.in b/packaging/RPM/ctdb.spec.in
index 848a3692a81d35a1f57e53fd847fe20781110e10..703e983167f4f684289afccd584e65c9d9eaf98e 100644 (file)
--- a/packaging/RPM/ctdb.spec.in
+++ b/packaging/RPM/ctdb.spec.in
@@ -113,6 +113,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_sysconfdir}/ctdb/events.d/91.lvs
 %{_sysconfdir}/ctdb/statd-callout
 %{_sysconfdir}/ctdb/interface_modify.sh
+%{_sysconfdir}/ctdb/debug_locks.sh
 %{_sbindir}/ctdbd
 %{_bindir}/ctdb
 %{_bindir}/smnotify