2 # a script to test the basic setup of a CTDB/Samba install
3 # tridge@samba.org September 2007
4 # martin@meltin.net August 2010
9 Usage: ctdb_diagnostics [OPTION] ...
11 -n <nodes> Comma separated list of nodes to operate on
12 -c Ignore comment lines (starting with '#') in file comparisons
13 -w Ignore whitespace in file comparisons
14 --no-ads Do not use commands that assume an Active Directory Server
20 nodes=$(ctdb listnodes -Y | cut -d: -f2)
26 temp=$(getopt -n "ctdb_diagnostics" -o "n:cwh" -l no-ads,help -- "$@")
34 -n) nodes=$(echo "$2" | sed -e 's@,@ @g') ; shift 2 ;;
35 -c) diff_opts="${diff_opts} -I ^#.*" ; shift ;;
36 -w) diff_opts="${diff_opts} -w" ; shift ;;
37 --no-ads) no_ads=true ; shift ;;
48 nodes_comma=$(echo $nodes | sed -e 's@[[:space:]]@,@g')
50 PATH="$PATH:/sbin:/usr/sbin:/usr/lpp/mmfs/bin"
52 # list of config files that must exist and that we check are the same
54 CONFIG_FILES_MUST="/etc/krb5.conf /etc/hosts /etc/ctdb/nodes /etc/sysconfig/ctdb /etc/resolv.conf /etc/nsswitch.conf /etc/sysctl.conf /etc/samba/smb.conf /etc/fstab /etc/multipath.conf /etc/pam.d/system-auth /etc/sysconfig/nfs /etc/exports /etc/vsftpd/vsftpd.conf"
56 # list of config files that may exist and should be checked that they
57 # are the same on the nodes
58 CONFIG_FILES_MAY="/etc/ctdb/public_addresses /etc/ctdb/static-routes"
63 --------------------------------------------------------------------
64 ctdb_diagnostics starting. This script will gather information about
65 your ctdb cluster. You should send the output of this script along
66 with any ctdb or clustered Samba bug reports.
67 --------------------------------------------------------------------
75 NUM_ERRORS=`expr $NUM_ERRORS + 1`
76 echo " ERROR[$NUM_ERRORS]: $msg" >> $ERRORS
81 echo " ================================"
83 echo " `ls -l $fname 2>&1`"
84 cat "$fname" 2>&1 | sed 's/^/ /'
85 echo " ================================"
89 echo "running $1 on nodes $nodes_comma"
90 onnode $nodes_comma "hostname; date; $1 2>&1 | sed 's/^/ /'" 2>&1
93 show_and_compare_files () {
104 onnode $n [ -r "$f" ] || {
105 msg=$(printf "$fmt" "$f" $n)
110 fstf=/tmp/`basename $f`.node$n
111 onnode $n cat $f > $fstf 2>&1
113 echo " ================================"
114 echo " File (on node $n): $f"
115 echo " `onnode $n ls -l $f 2>&1`"
116 cat "$fstf" | sed 's/^/ /'
117 echo " ================================"
120 echo "Testing for same config file $f on node $n"
121 tmpf=/tmp/`basename $f`.node$n
122 onnode $n cat $f > $tmpf 2>&1
123 diff $diff_opts $fstf $tmpf >/dev/null 2>&1 || {
124 error "File $f is different on node $n"
125 diff -u $diff_opts $fstf $tmpf
135 ERRORS="/tmp/diag_err.$$"
139 Diagnosis started on these nodes:
142 For reference, here is the nodes file on the current node...
144 show_file /etc/ctdb/nodes
148 --------------------------------------------------------------------
149 Comping critical config files on nodes $nodes_comma
152 show_and_compare_files \
153 "%s is missing on node %d" \
156 show_and_compare_files \
157 "Optional file %s is not present on node %d" \
161 --------------------------------------------------------------------
162 Checking for clock drift
166 t2=`onnode $i date +%s`
168 if [ $d -gt 30 -o $d -lt -30 ]; then
169 error "time on node $i differs by $d seconds"
174 --------------------------------------------------------------------
175 Showing software versions
179 show_all "rpm -qa | egrep 'samba|ctdb|gpfs'"
181 [ -x /usr/bin/dpkg-query ] && {
182 show_all "/usr/bin/dpkg-query --show 'ctdb'"
183 show_all "/usr/bin/dpkg-query --show 'samba'"
184 #show_all "/usr/bin/dpkg-query --show 'gpfs'"
189 --------------------------------------------------------------------
190 Showing ctdb status and recent log entries
192 show_all "ctdb status; ctdb ip"
193 show_all "ctdb statistics"
194 show_all "ctdb uptime"
196 echo "Showing log.ctdb"
197 show_all "test -f /var/log/log.ctdb && tail -100 /var/log/log.ctdb"
199 echo "Showing log.ctdb"
200 show_all "test -f /var/log/log.ctdb && tail -100 /var/log/log.ctdb"
202 show_all "tail -200 /var/log/messages"
203 show_all "tail -200 /etc/ctdb/state/vacuum.log"
204 show_all "ls -lRs /var/ctdb"
205 show_all "ls -lRs /etc/ctdb"
209 --------------------------------------------------------------------
210 Showing system and process status
218 show_all "/sbin/lspci"
220 show_all "cat /proc/partitions"
221 show_all "cat /proc/cpuinfo"
222 show_all "cat /proc/scsi/scsi"
223 show_all "/sbin/ifconfig -a"
224 show_all "/sbin/ifconfig -a"
225 show_all "/sbin/ip addr list"
226 show_all "/sbin/route -n"
227 show_all "netstat -s"
229 show_all "crontab -l"
231 show_all "iptables -L -n"
232 show_all "iptables -L -n -t nat"
233 show_all "/usr/sbin/rpcinfo -p"
234 show_all "/usr/sbin/showmount -a"
235 show_all "/usr/sbin/showmount -e"
236 show_all "/usr/sbin/nfsstat -v"
237 [ -x /sbin/multipath ] && {
238 show_all "/sbin/multipath -ll"
240 [ -x /sbin/chkconfig ] && {
241 show_all "/sbin/chkconfig --list"
243 [ -x /usr/sbin/getenforce ] && {
244 show_all "/usr/sbin/getenforce"
246 [ -d /proc/net/bonding ] && {
247 for f in /proc/net/bonding/*; do
252 [ -d /usr/lpp/mmfs ] && {
254 --------------------------------------------------------------------
255 Showing GPFS status and recent log entries
257 show_all "tail -100 /var/adm/ras/mmfs.log.latest"
258 show_all "/usr/lpp/mmfs/bin/mmlsconfig"
259 show_all "/usr/lpp/mmfs/bin/mmlsfs all"
260 show_all "/usr/lpp/mmfs/bin/mmlsnsd"
261 show_all "/usr/lpp/mmfs/bin/mmlsnsd -X"
262 show_all "/usr/lpp/mmfs/bin/mmfsadm dump version"
263 show_all "/usr/lpp/mmfs/bin/mmfsadm dump waiters"
264 show_all "/usr/lpp/mmfs/bin/mmlsmount all"
265 show_all "/usr/lpp/mmfs/bin/mmlsquota"
266 show_all "/usr/lpp/mmfs/bin/mmlscluster"
267 show_all "/usr/lpp/mmfs/bin/mmlsmgr"
268 devlist=`mmlsfs all|grep ^File.system.attributes | cut -d/ -f3 | cut -d: -f1`
269 for d in $devlist; do
271 show_all "mmlsdisk $d"
272 show_all "mmlsfileset $d"
273 show_all "mmlspolicy $d"
274 show_all "mmlssnapshot $d"
276 fslist=`mount|grep type.gpfs|awk '{print $1}'`
277 for fs in $fslist; do
278 show_all "/usr/lpp/mmfs/bin/mmlssnapshot $fs"
279 show_all "/usr/lpp/mmfs/bin/mmlsdisk $fs"
280 show_all "/usr/lpp/mmfs/bin/mmlsfileset $fs"
285 --------------------------------------------------------------------
288 show_all "smbstatus -n -B"
291 echo "Skipping \"net ads testjoin\" as requested"
294 show_all "net ads testjoin"
296 show_all "net conf list"
297 show_all "lsof -n | grep smbd"
298 show_all "lsof -n | grep ctdbd"
299 show_all "netstat -tan"
302 echo "Skipping \"net ads info\" as requested"
305 show_all "net ads info"
308 show_all "smbclient -U% -L 127.0.0.1"
309 WORKGROUP=`testparm -s --parameter-name=WORKGROUP 2> /dev/null`
310 show_all id "$WORKGROUP/Administrator"
312 show_all "wbinfo --online-status"
316 echo "Diagnostics finished with $NUM_ERRORS errors"