${INSTALLCMD} -m 755 config/events.d/10.interface $(DESTDIR)$(etcdir)/ctdb/events.d
${INSTALLCMD} -m 755 config/events.d/11.natgw $(DESTDIR)$(etcdir)/ctdb/events.d
${INSTALLCMD} -m 755 config/events.d/11.routing $(DESTDIR)$(etcdir)/ctdb/events.d
- ${INSTALLCMD} -m 755 config/events.d/20.multipathd $(DESTDIR)$(etcdir)/ctdb/events.d
+ ${INSTALLCMD} -m 644 config/events.d/20.multipathd $(DESTDIR)$(etcdir)/ctdb/events.d
${INSTALLCMD} -m 644 config/events.d/31.clamd $(DESTDIR)$(etcdir)/ctdb/events.d
${INSTALLCMD} -m 755 config/events.d/40.vsftpd $(DESTDIR)$(etcdir)/ctdb/events.d
${INSTALLCMD} -m 755 config/events.d/41.httpd $(DESTDIR)$(etcdir)/ctdb/events.d
#include "../include/ctdb_private.h"
#include "lib/util/dlinklist.h"
+pid_t ctdbd_pid;
+
/*
allocate a packet for use in client<->daemon communication
*/
{
struct ctdb_client_control_state *state = talloc_get_type(private_data, struct ctdb_client_control_state);
- DEBUG(DEBUG_ERR,("control timed out. reqid:%d opcode:%d dstnode:%d\n", state->reqid, state->c->opcode, state->c->hdr.destnode));
+ DEBUG(DEBUG_ERR,(__location__ " control timed out. reqid:%u opcode:%u "
+ "dstnode:%u\n", state->reqid, state->c->opcode,
+ state->c->hdr.destnode));
state->state = CTDB_CONTROL_TIMEOUT;
timeout, false, data,
NULL, NULL,
NULL) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " ctdb_control to disable node failed\n"));
+ DEBUG(DEBUG_ERR, (__location__ " Unable to update nodeflags on remote nodes\n"));
talloc_free(tmp_ctx);
return -1;
return ret;
}
+/**
+ * check whether a transaction is active on a given db on a given node
+ */
+static int32_t ctdb_ctrl_transaction_active(struct ctdb_context *ctdb,
+ uint32_t destnode,
+ uint32_t db_id)
+{
+ int32_t status;
+ int ret;
+ TDB_DATA indata;
+
+ indata.dptr = (uint8_t *)&db_id;
+ indata.dsize = sizeof(db_id);
+
+ ret = ctdb_control(ctdb, destnode, 0,
+ CTDB_CONTROL_TRANS2_ACTIVE,
+ 0, indata, NULL, NULL, &status,
+ NULL, NULL);
+
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " ctdb control for transaction_active failed\n"));
+ return -1;
+ }
+
+ return status;
+}
+
+
struct ctdb_transaction_handle {
struct ctdb_db_context *ctdb_db;
bool in_replay;
- /* we store the reads and writes done under a transaction one
- list stores both reads and writes, the other just writes
- */
+ /*
+ * we store the reads and writes done under a transaction:
+ * - one list stores both reads and writes (m_all),
+ * - the other just writes (m_write)
+ */
struct ctdb_marshall_buffer *m_all;
struct ctdb_marshall_buffer *m_write;
};
int ret;
struct ctdb_db_context *ctdb_db = h->ctdb_db;
pid_t pid;
+ int32_t status;
key.dptr = discard_const(keyname);
key.dsize = strlen(keyname);
rh = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, NULL);
if (rh == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to fetch_lock database\n"));
+ DEBUG(DEBUG_ERR,(__location__ " Failed to fetch_lock database\n"));
talloc_free(tmp_ctx);
return -1;
}
+
+ status = ctdb_ctrl_transaction_active(ctdb_db->ctdb,
+ CTDB_CURRENT_NODE,
+ ctdb_db->db_id);
+ if (status == 1) {
+ unsigned long int usec = (1000 + random()) % 100000;
+ DEBUG(DEBUG_NOTICE, (__location__ " transaction is active "
+ "on db_id[0x%08x]. waiting for %lu "
+ "microseconds\n",
+ ctdb_db->db_id, usec));
+ talloc_free(tmp_ctx);
+ usleep(usec);
+ goto again;
+ }
+
/*
* store the pid in the database:
* it is not enough that the node is dmaster...
&timeout, NULL);
if (ret != 0 || status != 0) {
tdb_transaction_cancel(h->ctdb_db->ltdb->tdb);
+ DEBUG(DEBUG_WARNING, (__location__ " transaction commit%s failed"
+ ", retrying after 1 second...\n",
+ (retries==0)?"":"retry "));
sleep(1);
if (ret != 0) {
struct ctdb_context *ctdb;
struct ctdb_partial partial; /* partial input packet */
struct ctdb_queue_pkt *out_queue;
+ uint32_t out_queue_length;
struct fd_event *fde;
int fd;
size_t alignment;
int ctdb_queue_length(struct ctdb_queue *queue)
{
- int i;
- struct ctdb_queue_pkt *pkt;
-
- for(i=0, pkt=queue->out_queue;pkt;i++,pkt=pkt->next);
-
- return i;
+ return queue->out_queue_length;
}
/*
if (pkt->length != pkt->full_length) {
/* partial packet sent - we have to drop it */
DLIST_REMOVE(queue->out_queue, pkt);
+ queue->out_queue_length--;
talloc_free(pkt);
}
talloc_free(queue->fde);
}
DLIST_REMOVE(queue->out_queue, pkt);
+ queue->out_queue_length--;
talloc_free(pkt);
}
}
DLIST_ADD_END(queue->out_queue, pkt, struct ctdb_queue_pkt *);
+ queue->out_queue_length++;
if (queue->ctdb->tunable.verbose_memory_names != 0) {
struct ctdb_req_header *hdr = (struct ctdb_req_header *)pkt->data;
check_persistent_databases || return $?
+ if [ yes == "$CTDB_SUPPRESS_COREFILE" ]; then
+ ulimit -c 0
+ else
+ ulimit -c unlimited
+ fi
+
case $init_style in
valgrind)
valgrind -q --log-file=/var/log/ctdb_valgrind \
# the default is ERR
CTDB_DEBUGLEVEL=ERR
+# whether to suppress core files. Default is no.
+# CTDB_SUPPRESS_COREFILE=yes
+
# Write debug messages to syslog instead of logfile?
# The default is not to use syslog.
# CTDB_SYSLOG=no
+# Should the 99.timeout monitor event script be run?
+# This event script just sleeps long enough to trigger the
+# event script timeout. Might be useful for debugging.
+# The default is "no".
+# CTDB_RUN_TIMEOUT_MONITOR=no
+
# set any default tuning options for ctdb
# use CTDB_SET_XXXX=value where XXXX is the name of the tuning
# variable
# CTDB_SET_TRAVERSETIMEOUT=60
# you can get a list of variables using "ctdb listvars"
-
# any other options you might want. Run ctdbd --help for a list
# CTDB_OPTIONS=
case $IFACE in
ethX*|bond*)
IFACE=`echo $IFACE |sed -e 's/\....$//'`
+ grep -q 'Currently Active Slave: None' /proc/net/bonding/$IFACE && {
+ echo "ERROR: No active slaves for bond device $IFACE"
+ exit 1
+ }
grep -q '^MII Status: up' /proc/net/bonding/$IFACE || {
echo "ERROR: public network interface $IFACE is down"
exit 1
--- /dev/null
+#!/bin/sh
+#
+# Event script to just sleep longer than the timeout
+# in the monitor action. The purpose is to trigger
+# the event timeout mechanism.
+
+. $CTDB_BASE/functions
+loadconfig ctdb
+
+[ "x$CTDB_RUN_TIMEOUT_MONITOR" = "xyes" ] || exit 0
+
+cmd="$1"
+shift
+
+case $cmd in
+ monitor)
+ TIMEOUT=$(ctdb listvars | grep EventScriptTimeout | awk '{print $3}')
+ echo "sleeping for $((TIMEOUT * 2)) seconds..."
+ sleep $((TIMEOUT * 2))
+ ;;
+esac
+
+exit 0
+
echo "Killing TCP connection $srcip:$srcport $destip:$destport"
ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
case $destport in
- # we only do one-way killtcp for NFS and CIFS
- 139|445|2049) : ;;
+ # we only do one-way killtcp for CIFS
+ 139|445) : ;;
# for all others we do 2-way
*)
ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
.\" Title: ctdb
.\" Author:
.\" Generator: DocBook XSL Stylesheets v1.73.2 <http://docbook.sf.net/>
-.\" Date: 08/13/2009
+.\" Date: 10/29/2009
.\" Manual:
.\" Source:
.\"
-.TH "CTDB" "1" "08/13/2009" "" ""
+.TH "CTDB" "1" "10/29/2009" "" ""
.\" disable hyphenation
.nh
.\" disable justification (adjust text to left margin only)
This command shows the pnn of the node which is currently the recmaster\.
.SS "uptime"
.PP
-This command shows the uptime for the ctdb daemon\. When the last recovery completed and how long the last recovery took\. If the "duration" is shown as a negative number, this indicates that there is a recovery in progress and it started that many seconds ago\.
+This command shows the uptime for the ctdb daemon\. When the last recovery or ip\-failover completed and how long it took\. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago\.
.PP
Example: ctdb uptime
.PP
.sp
.RS 4
.nf
-Current time of node : Tue Mar 24 18:27:54 2009
-Ctdbd start time : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Time of last recovery : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Duration of last recovery : 0\.000000 seconds
+Current time of node : Thu Oct 29 10:38:54 2009
+Ctdbd start time : (000 16:54:28) Wed Oct 28 17:44:26 2009
+Time of last recovery/failover: (000 16:53:31) Wed Oct 28 17:45:23 2009
+Duration of last recovery/failover: 2\.248552 seconds
.fi
.RE
-<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdb</title><meta name="generator" content="DocBook XSL Stylesheets V1.73.2"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="ctdb.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdb — clustered tdb database management utility</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdb [ OPTIONS ] COMMAND ...</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdb</code> [-n <node>] [-Y] [-t <timeout>] [-T <timelimit>] [-? --help] [--usage] [-d --debug=<INTEGER>] [--socket=<filename>]</p></div></div><div class="refsect1" lang="en"><a name="id2478
257"></a><h2>DESCRIPTION</h2><p>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdb</title><meta name="generator" content="DocBook XSL Stylesheets V1.73.2"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="ctdb.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdb — clustered tdb database management utility</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdb [ OPTIONS ] COMMAND ...</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdb</code> [-n <node>] [-Y] [-t <timeout>] [-T <timelimit>] [-? --help] [--usage] [-d --debug=<INTEGER>] [--socket=<filename>]</p></div></div><div class="refsect1" lang="en"><a name="id2478
395"></a><h2>DESCRIPTION</h2><p>
ctdb is a utility to view and manage a ctdb cluster.
- </p></div><div class="refsect1" lang="en"><a name="id2478
267"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-n <pnn></span></dt><dd><p>
+ </p></div><div class="refsect1" lang="en"><a name="id2478
405"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-n <pnn></span></dt><dd><p>
This specifies the physical node number on which to execute the
command. Default is to run the command on the deamon running on
the local host.
You only need to specify this parameter if you run multiple ctdb
daemons on the same physical host and thus can not use the default
name for the domain socket.
- </p></dd></dl></div></div><div class="refsect1" lang="en"><a name="id247
8404"></a><h2>Administrative Commands</h2><p>
+ </p></dd></dl></div></div><div class="refsect1" lang="en"><a name="id247
9674"></a><h2>Administrative Commands</h2><p>
These are commands used to monitor and administrate a CTDB cluster.
- </p><div class="refsect2" lang="en"><a name="id247
8413"></a><h3>pnn</h3><p>
+ </p><div class="refsect2" lang="en"><a name="id247
9683"></a><h3>pnn</h3><p>
This command displays the pnn of the current node.
- </p></div><div class="refsect2" lang="en"><a name="id247
8423"></a><h3>status</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id247
9693"></a><h3>status</h3><p>
This command shows the current status of the ctdb node.
- </p><div class="refsect3" lang="en"><a name="id247
8621"></a><h4>node status</h4><p>
+ </p><div class="refsect3" lang="en"><a name="id247
9702"></a><h4>node status</h4><p>
Node status reflects the current status of the node. There are five possible states:
</p><p>
OK - This node is fully functional.
RECMASTER or NATGW.
This node does not perticipate in the CTDB cluster but can still be
communicated with. I.e. ctdb commands can be sent to it.
- </p></div><div class="refsect3" lang="en"><a name="id247
8682"></a><h4>generation</h4><p>
+ </p></div><div class="refsect3" lang="en"><a name="id247
9763"></a><h4>generation</h4><p>
The generation id is a number that indicates the current generation
of a cluster instance. Each time a cluster goes through a
reconfiguration or a recovery its generation id will be changed.
All nodes start with generation "INVALID" and are not assigned a real
generation id until they have successfully been merged with a cluster
through a recovery.
- </p></div><div class="refsect3" lang="en"><a name="id247
8717"></a><h4>VNNMAP</h4><p>
+ </p></div><div class="refsect3" lang="en"><a name="id247
9798"></a><h4>VNNMAP</h4><p>
The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records.
Only nodes that are participating in the vnnmap can become lmaster or dmaster for a database record.
- </p></div><div class="refsect3" lang="en"><a name="id247
8731"></a><h4>Recovery mode</h4><p>
+ </p></div><div class="refsect3" lang="en"><a name="id247
9812"></a><h4>Recovery mode</h4><p>
This is the current recovery mode of the cluster. There are two possible modes:
</p><p>
NORMAL - The cluster is fully operational.
have been recovered, the node mode will change into NORMAL mode
and the databases will be "thawed", allowing samba to access the
databases again.
- </p></div><div class="refsect3" lang="en"><a name="id2
478773"></a><h4>Recovery master</h4><p>
+ </p></div><div class="refsect3" lang="en"><a name="id2
528501"></a><h4>Recovery master</h4><p>
This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired.
</p><p>
Only one node at a time can be the designated recovery master. Which
hash:3 lmaster:3
Recovery mode:NORMAL (0)
Recovery master:0
- </pre></div><div class="refsect2" lang="en"><a name="id2
478810"></a><h3>recmaster</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2
528538"></a><h3>recmaster</h3><p>
This command shows the pnn of the node which is currently the recmaster.
- </p></div><div class="refsect2" lang="en"><a name="id2
478821"></a><h3>uptime</h3><p>
- This command shows the uptime for the ctdb daemon. When the last recovery completed and how long the last recovery took. If the "duration" is shown as a negative number, this indicates that there is a recovery in progress and it started that many seconds ago.
+ </p></div><div class="refsect2" lang="en"><a name="id2
528549"></a><h3>uptime</h3><p>
+ This command shows the uptime for the ctdb daemon. When the last recovery or ip-failover completed and how long it took. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago.
</p><p>
Example: ctdb uptime
</p><p>Example output:</p><pre class="screen">
-Current time of node : Tue Mar 24 18:27:54 2009
-Ctdbd start time : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Time of last recovery : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Duration of last recovery : 0.000000 seconds
- </pre></div><div class="refsect2" lang="en"><a name="id2
478851"></a><h3>listnodes</h3><p>
+Current time of node : Thu Oct 29 10:38:54 2009
+Ctdbd start time : (000 16:54:28) Wed Oct 28 17:44:26 2009
+Time of last recovery/failover: (000 16:53:31) Wed Oct 28 17:45:23 2009
+Duration of last recovery/failover: 2.248552 seconds
+ </pre></div><div class="refsect2" lang="en"><a name="id2
528580"></a><h3>listnodes</h3><p>
This command shows lists the ip addresses of all the nodes in the cluster.
</p><p>
Example: ctdb listnodes
10.0.0.72
10.0.0.73
10.0.0.74
- </pre></div><div class="refsect2" lang="en"><a name="id2
478874"></a><h3>ping</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2
528604"></a><h3>ping</h3><p>
This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly.
</p><p>
Example: ctdb ping
response from 1 time=0.000144 sec (2 clients)
response from 2 time=0.000105 sec (2 clients)
response from 3 time=0.000114 sec (2 clients)
- </pre></div><div class="refsect2" lang="en"><a name="id2528
491"></a><h3>ip</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
630"></a><h3>ip</h3><p>
This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip. By default this command will ONLY show those public addresses that are known to the node itself. To see the full list of all public ips across the cluster you must use "ctdb ip -n all".
</p><p>
Example: ctdb ip
12.1.1.2 1
12.1.1.3 2
12.1.1.4 3
- </pre></div><div class="refsect2" lang="en"><a name="id2528
519"></a><h3>scriptstatus</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
658"></a><h3>scriptstatus</h3><p>
This command displays which scripts where run in the previous monitoring cycle and the result of each script. If a script failed with an error, causing the node to become unhealthy, the output from that script is also shown.
</p><p>
Example: ctdb scriptstatus
41.httpd Status:OK Duration:0.039 Tue Mar 24 18:56:57 2009
50.samba Status:ERROR Duration:0.082 Tue Mar 24 18:56:57 2009
OUTPUT:ERROR: Samba tcp port 445 is not responding
- </pre></div><div class="refsect2" lang="en"><a name="id2528
555"></a><h3>disablescript <script></h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
694"></a><h3>disablescript <script></h3><p>
This command is used to disable an eventscript.
</p><p>
This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'.
- </p></div><div class="refsect2" lang="en"><a name="id2528
572"></a><h3>enablescript <script></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
711"></a><h3>enablescript <script></h3><p>
This command is used to enable an eventscript.
</p><p>
This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'.
- </p></div><div class="refsect2" lang="en"><a name="id2528
588"></a><h3>getvar <name></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
727"></a><h3>getvar <name></h3><p>
Get the runtime value of a tuneable variable.
</p><p>
Example: ctdb getvar MaxRedirectCount
Example output:
</p><pre class="screen">
MaxRedirectCount = 3
- </pre></div><div class="refsect2" lang="en"><a name="id2528
611"></a><h3>setvar <name> <value></h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
750"></a><h3>setvar <name> <value></h3><p>
Set the runtime value of a tuneable variable.
</p><p>
Example: ctdb setvar MaxRedirectCount 5
- </p></div><div class="refsect2" lang="en"><a name="id2528
626"></a><h3>listvars</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
765"></a><h3>listvars</h3><p>
List all tuneable variables.
</p><p>
Example: ctdb listvars
EventScriptTimeout = 20
RecoveryGracePeriod = 60
RecoveryBanPeriod = 300
- </pre></div><div class="refsect2" lang="en"><a name="id2528
654"></a><h3>lvsmaster</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
793"></a><h3>lvsmaster</h3><p>
This command shows which node is currently the LVSMASTER. The
LVSMASTER is the node in the cluster which drives the LVS system and
which receives all incoming traffic from clients.
evenly onto the other nodes in the cluster. This is an alternative to using
public ip addresses. See the manpage for ctdbd for more information
about LVS.
- </p></div><div class="refsect2" lang="en"><a name="id2528
676"></a><h3>lvs</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
815"></a><h3>lvs</h3><p>
This command shows which nodes in the cluster are currently active in the
LVS configuration. I.e. which nodes we are currently loadbalancing
the single ip address across.
</p><pre class="screen">
2:10.0.0.13
3:10.0.0.14
- </pre></div><div class="refsect2" lang="en"><a name="id2528
706"></a><h3>getcapabilities</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
845"></a><h3>getcapabilities</h3><p>
This command shows the capabilities of the current node.
Please see manpage for ctdbd for a full list of all capabilities and
more detailed description.
RECMASTER: YES
LMASTER: YES
LVS: NO
- </pre></div><div class="refsect2" lang="en"><a name="id2528
742"></a><h3>statistics</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
881"></a><h3>statistics</h3><p>
Collect statistics from the CTDB daemon about how many calls it has served.
</p><p>
Example: ctdb statistics
max_hop_count 0
max_call_latency 4.948321 sec
max_lockwait_latency 0.000000 sec
- </pre></div><div class="refsect2" lang="en"><a name="id2528
786"></a><h3>statisticsreset</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
925"></a><h3>statisticsreset</h3><p>
This command is used to clear all statistics counters in a node.
</p><p>
Example: ctdb statisticsreset
- </p></div><div class="refsect2" lang="en"><a name="id2528
800"></a><h3>getreclock</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
939"></a><h3>getreclock</h3><p>
This command is used to show the filename of the reclock file that is used.
</p><p>
Example output:
</p><pre class="screen">
Reclock file:/gpfs/.ctdb/shared
- </pre></div><div class="refsect2" lang="en"><a name="id2528
820"></a><h3>setreclock [filename]</h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2528
959"></a><h3>setreclock [filename]</h3><p>
This command is used to modify, or clear, the file that is used as the reclock file at runtime. When this command is used, the reclock file checks are disabled. To re-enable the checks the administrator needs to activate the "VerifyRecoveryLock" tunable using "ctdb setvar".
</p><p>
If run with no parameter this will remove the reclock file completely. If run with a parameter the parameter specifies the new filename to use for the recovery lock.
</p><p>
This command only affects the runtime settings of a ctdb node and will be lost when ctdb is restarted. For persistent changes to the reclock file setting you must edit /etc/sysconfig/ctdb.
- </p></div><div class="refsect2" lang="en"><a name="id2528
846"></a><h3>getdebug</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2528
985"></a><h3>getdebug</h3><p>
Get the current debug level for the node. the debug level controls what information is written to the log file.
</p><p>
The debug levels are mapped to the corresponding syslog levels.
The list of debug levels from highest to lowest are :
</p><p>
EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG
- </p></div><div class="refsect2" lang="en"><a name="id252
8872"></a><h3>setdebug <debuglevel></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9011"></a><h3>setdebug <debuglevel></h3><p>
Set the debug level of a node. This controls what information will be logged.
</p><p>
The debuglevel is one of EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG
- </p></div><div class="refsect2" lang="en"><a name="id252
8888"></a><h3>getpid</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9027"></a><h3>getpid</h3><p>
This command will return the process id of the ctdb daemon.
- </p></div><div class="refsect2" lang="en"><a name="id252
8898"></a><h3>disable</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9037"></a><h3>disable</h3><p>
This command is used to administratively disable a node in the cluster.
A disabled node will still participate in the cluster and host
clustered TDB records but its public ip address has been taken over by
a different node and it no longer hosts any services.
- </p></div><div class="refsect2" lang="en"><a name="id252
8912"></a><h3>enable</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9051"></a><h3>enable</h3><p>
Re-enable a node that has been administratively disabled.
- </p></div><div class="refsect2" lang="en"><a name="id252
8922"></a><h3>stop</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9061"></a><h3>stop</h3><p>
This command is used to administratively STOP a node in the cluster.
A STOPPED node is connected to the cluster but will not host any
public ip addresse, nor does it participate in the VNNMAP.
The difference between a DISABLED node and a STOPPED node is that
a STOPPED node does not host any parts of the database which means
that a recovery is required to stop/continue nodes.
- </p></div><div class="refsect2" lang="en"><a name="id252
8937"></a><h3>continue</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9076"></a><h3>continue</h3><p>
Re-start a node that has been administratively stopped.
- </p></div><div class="refsect2" lang="en"><a name="id252
8948"></a><h3>addip <public_ip/mask> <iface></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9087"></a><h3>addip <public_ip/mask> <iface></h3><p>
This command is used to add a new public ip to a node during runtime.
This allows public addresses to be added to a cluster without having
to restart the ctdb daemons.
</p><p>
Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re-read.
If you want this change to be permanent you must also update the public addresses file manually.
- </p></div><div class="refsect2" lang="en"><a name="id252
8968"></a><h3>delip <public_ip></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9107"></a><h3>delip <public_ip></h3><p>
This command is used to remove a public ip from a node during runtime.
If this public ip is currently hosted by the node it being removed from, the ip will first be failed over to another node, if possible, before it is removed.
</p><p>
Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re-read.
If you want this change to be permanent you must also update the public addresses file manually.
- </p></div><div class="refsect2" lang="en"><a name="id252
8989"></a><h3>moveip <public_ip> <node></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id252
9132"></a><h3>moveip <public_ip> <node></h3><p>
This command can be used to manually fail a public ip address to a
specific node.
</p><p>
DeterministicIPs = 0
</p><p>
NoIPFailback = 1
- </p></div><div class="refsect2" lang="en"><a name="id2529
014"></a><h3>shutdown</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
157"></a><h3>shutdown</h3><p>
This command will shutdown a specific CTDB daemon.
- </p></div><div class="refsect2" lang="en"><a name="id2529
024"></a><h3>recover</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
167"></a><h3>recover</h3><p>
This command will trigger the recovery daemon to do a cluster
recovery.
- </p></div><div class="refsect2" lang="en"><a name="id2529
034"></a><h3>ipreallocate</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
178"></a><h3>ipreallocate</h3><p>
This command will force the recovery master to perform a full ip reallocation process and redistribute all ip addresses. This is useful to "reset" the allocations back to its default state if they have been changed using the "moveip" command. While a "recover" will also perform this reallocation, a recovery is much more hevyweight since it will also rebuild all the databases.
- </p></div><div class="refsect2" lang="en"><a name="id2529
050"></a><h3>setlmasterrole <on|off></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
193"></a><h3>setlmasterrole <on|off></h3><p>
This command is used ot enable/disable the LMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an LMASTER for records in the database. A node that does not have the LMASTER capability will not show up in the vnnmap.
</p><p>
Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command.
Once this setting has been enabled/disabled, you need to perform a recovery for it to take effect.
</p><p>
See also "ctdb getcapabilities"
- </p></div><div class="refsect2" lang="en"><a name="id2529
079"></a><h3>setrecmasterrole <on|off></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
222"></a><h3>setrecmasterrole <on|off></h3><p>
This command is used ot enable/disable the RECMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an RECMASTER for the cluster. A node that does not have the RECMASTER capability can not win a recmaster election. A node that already is the recmaster for the cluster when the capability is stripped off the node will remain the recmaster until the next cluster election.
</p><p>
Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command.
</p><p>
See also "ctdb getcapabilities"
- </p></div><div class="refsect2" lang="en"><a name="id2529
105"></a><h3>killtcp <srcip:port> <dstip:port></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
249"></a><h3>killtcp <srcip:port> <dstip:port></h3><p>
This command will kill the specified TCP connection by issuing a
TCP RST to the srcip:port endpoint. This is a command used by the
ctdb eventscripts.
- </p></div><div class="refsect2" lang="en"><a name="id2529
118"></a><h3>gratiousarp <ip> <interface></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
261"></a><h3>gratiousarp <ip> <interface></h3><p>
This command will send out a gratious arp for the specified interface
through the specified interface. This command is mainly used by the
ctdb eventscripts.
- </p></div><div class="refsect2" lang="en"><a name="id2529
130"></a><h3>reloadnodes</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
274"></a><h3>reloadnodes</h3><p>
This command is used when adding new nodes, or removing existing nodes from an existing cluster.
</p><p>
Procedure to add a node:
</p><p>
5, Use 'ctdb status' on all nodes and verify that the deleted node no longer shows up in the list..
</p><p>
- </p></div><div class="refsect2" lang="en"><a name="id2529
214"></a><h3>tickle <srcip:port> <dstip:port></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
357"></a><h3>tickle <srcip:port> <dstip:port></h3><p>
This command will will send a TCP tickle to the source host for the
specified TCP connection.
A TCP tickle is a TCP ACK packet with an invalid sequence and
TCP connection has been disrupted and that the client will need
to reestablish. This greatly speeds up the time it takes for a client
to detect and reestablish after an IP failover in the ctdb cluster.
- </p></div><div class="refsect2" lang="en"><a name="id2529
239"></a><h3>gettickles <ip></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
382"></a><h3>gettickles <ip></h3><p>
This command is used to show which TCP connections are registered with
CTDB to be "tickled" if there is a failover.
- </p></div><div class="refsect2" lang="en"><a name="id2529
250"></a><h3>repack [max_freelist]</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
394"></a><h3>repack [max_freelist]</h3><p>
Over time, when records are created and deleted in a TDB, the TDB list of free space will become fragmented. This can lead to a slowdown in accessing TDB records.
This command is used to defragment a TDB database and pruning the freelist.
</p><p>
Example: ctdb repack 1000
</p><p>
By default, this operation is issued from the 00.ctdb event script every 5 minutes.
- </p></div><div class="refsect2" lang="en"><a name="id2529
296"></a><h3>vacuum [max_records]</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
440"></a><h3>vacuum [max_records]</h3><p>
Over time CTDB databases will fill up with empty deleted records which will lead to a progressive slow down of CTDB database access.
This command is used to prune all databases and delete all empty records from the cluster.
</p><p>
Example: ctdb vacuum
</p><p>
By default, this operation is issued from the 00.ctdb event script every 5 minutes.
- </p></div><div class="refsect2" lang="en"><a name="id2529
332"></a><h3>backupdb <database> <file></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
476"></a><h3>backupdb <database> <file></h3><p>
This command can be used to copy the entire content of a database out to a file. This file can later be read back into ctdb using the restoredb command.
This is mainly useful for backing up persistent databases such as secrets.tdb and similar.
- </p></div><div class="refsect2" lang="en"><a name="id2529
346"></a><h3>restoredb <file></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
490"></a><h3>restoredb <file></h3><p>
This command restores a persistent database that was previously backed up using backupdb.
- </p></div></div><div class="refsect1" lang="en"><a name="id2529
358"></a><h2>Debugging Commands</h2><p>
+ </p></div></div><div class="refsect1" lang="en"><a name="id2529
502"></a><h2>Debugging Commands</h2><p>
These commands are primarily used for CTDB development and testing and
should not be used for normal administration.
- </p><div class="refsect2" lang="en"><a name="id2529
368"></a><h3>process-exists <pid></h3><p>
+ </p><div class="refsect2" lang="en"><a name="id2529
512"></a><h3>process-exists <pid></h3><p>
This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not.
- </p></div><div class="refsect2" lang="en"><a name="id2529
381"></a><h3>getdbmap</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2529
524"></a><h3>getdbmap</h3><p>
This command lists all clustered TDB databases that the CTDB daemon has attached to. Some databases are flagged as PERSISTENT, this means that the database stores data persistently and the data will remain across reboots. One example of such a database is secrets.tdb where information about how the cluster was joined to the domain is stored.
</p><p>
Most databases are not persistent and only store the state information that the currently running samba daemons need. These databases are always wiped when ctdb/samba starts and when a node is rebooted.
dbid:0xb775fff6 name:secrets.tdb path:/var/ctdb/persistent/secrets.tdb.0 PERSISTENT
dbid:0xe98e08b6 name:group_mapping.tdb path:/var/ctdb/persistent/group_mapping.tdb.0 PERSISTENT
dbid:0x7bbbd26c name:passdb.tdb path:/var/ctdb/persistent/passdb.tdb.0 PERSISTENT
- </pre></div><div class="refsect2" lang="en"><a name="id2529
427"></a><h3>catdb <dbname></h3><p>
+ </pre></div><div class="refsect2" lang="en"><a name="id2529
558"></a><h3>catdb <dbname></h3><p>
This command will dump a clustered TDB database to the screen. This is a debugging command.
- </p></div><div class="refsect2" lang="en"><a name="id2
529438"></a><h3>getmonmode</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476137"></a><h3>getmonmode</h3><p>
This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continously monitor that all other nodes that are expected are in fact connected and that they respond to commands.
</p><p>
ACTIVE - This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster.
</p><p>
DISABLED - This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery.
- </p></div><div class="refsect2" lang="en"><a name="id2
529468"></a><h3>setmonmode <0|1></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476168"></a><h3>setmonmode <0|1></h3><p>
This command can be used to explicitely disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards.
- </p></div><div class="refsect2" lang="en"><a name="id2
529484"></a><h3>attach <dbname></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476183"></a><h3>attach <dbname></h3><p>
This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it.
- </p></div><div class="refsect2" lang="en"><a name="id2
529495"></a><h3>dumpmemory</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476195"></a><h3>dumpmemory</h3><p>
This is a debugging command. This command will make the ctdb
daemon to write a fill memory allocation map to standard output.
- </p></div><div class="refsect2" lang="en"><a name="id2
529507"></a><h3>rddumpmemory</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476206"></a><h3>rddumpmemory</h3><p>
This is a debugging command. This command will dump the talloc memory
allocation tree for the recovery daemon to standard output.
- </p></div><div class="refsect2" lang="en"><a name="id2
529518"></a><h3>freeze</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476218"></a><h3>freeze</h3><p>
This command will lock all the local TDB databases causing clients
that are accessing these TDBs such as samba3 to block until the
databases are thawed.
This is primarily used by the recovery daemon to stop all samba
daemons from accessing any databases while the database is recovered
and rebuilt.
- </p></div><div class="refsect2" lang="en"><a name="id2
529537"></a><h3>thaw</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476236"></a><h3>thaw</h3><p>
Thaw a previously frozen node.
- </p></div><div class="refsect2" lang="en"><a name="id2
529546"></a><h3>eventscript <arguments></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476246"></a><h3>eventscript <arguments></h3><p>
This is a debugging command. This command can be used to manually
invoke and run the eventscritps with arbitrary arguments.
- </p></div><div class="refsect2" lang="en"><a name="id2
529558"></a><h3>ban <bantime|0></h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2
476257"></a><h3>ban <bantime|0></h3><p>
Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned.
</p><p>
A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by an other node and no services are hosted.
</p><p>
Nodes are automatically banned if they are the cause of too many
cluster recoveries.
- </p></div><div class="refsect2" lang="en"><a name="id2476
137"></a><h3>unban</h3><p>
+ </p></div><div class="refsect2" lang="en"><a name="id2476
281"></a><h3>unban</h3><p>
This command is used to unban a node that has either been
administratively banned using the ban command or has been automatically
banned by the recovery daemon.
- </p></div></div><div class="refsect1" lang="en"><a name="id2476
150"></a><h2>SEE ALSO</h2><p>
+ </p></div></div><div class="refsect1" lang="en"><a name="id2476
294"></a><h2>SEE ALSO</h2><p>
ctdbd(1), onnode(1)
<a class="ulink" href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a>
- </p></div><div class="refsect1" lang="en"><a name="id2476
163"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br>
+ </p></div><div class="refsect1" lang="en"><a name="id2476
307"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br>
Copyright (C) Andrew Tridgell 2007<br>
Copyright (C) Ronnie sahlberg 2007<br>
<br>
<refsect2><title>uptime</title>
<para>
- This command shows the uptime for the ctdb daemon. When the last recovery completed and how long the last recovery took. If the "duration" is shown as a negative number, this indicates that there is a recovery in progress and it started that many seconds ago.
+ This command shows the uptime for the ctdb daemon. When the last recovery or ip-failover completed and how long it took. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago.
</para>
<para>
</para>
<para>Example output:</para>
<screen format="linespecific">
-Current time of node : Tue Mar 24 18:27:54 2009
-Ctdbd start time : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Time of last recovery : (000 00:00:05) Tue Mar 24 18:27:49 2009
-Duration of last recovery : 0.000000 seconds
+Current time of node : Thu Oct 29 10:38:54 2009
+Ctdbd start time : (000 16:54:28) Wed Oct 28 17:44:26 2009
+Time of last recovery/failover: (000 16:53:31) Wed Oct 28 17:45:23 2009
+Duration of last recovery/failover: 2.248552 seconds
</screen>
</refsect2>
*/
#define CTDB_SRVID_TAKEOVER_RUN_RESPONSE 0xFD00000000000000LL
+/* A port reserved for samba (top 32 bits)
+ */
+#define CTDB_SRVID_SAMBA_NOTIFY 0xFE00000000000000LL
+
/* used on the domain socket, send a pdu to the local daemon */
#define CTDB_CURRENT_NODE 0xF0000001
/* send a broadcast to all nodes in the cluster, active or not */
} async;
};
+struct ctdb_client_notify_register {
+ uint64_t srvid;
+ uint32_t len;
+ uint8_t notify_data[1];
+};
+
+struct ctdb_client_notify_deregister {
+ uint64_t srvid;
+};
struct event_context;
start the ctdb protocol
*/
int ctdb_start(struct ctdb_context *ctdb);
-int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork);
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog);
/*
attach to a ctdb database
uint64_t srvid;
};
+/*
+ * pid of the ctdbd daemon
+ */
+extern pid_t ctdbd_pid;
+
/*
a tcp connection description
*/
int32_t status, TDB_DATA data,
const char *errormsg,
void *private_data);
-
/*
structure describing a connected client in the daemon
*/
struct ctdb_tcp_list *tcp_list;
uint32_t db_id;
uint32_t num_persistent_updates;
+ struct ctdb_client_notify_list *notify;
};
TALLOC_CTX *eventscripts_ctx; /* a context to hold data for the RUN_EVENTSCRIPTS control */
uint32_t *recd_ping_count;
TALLOC_CTX *release_ips_ctx; /* a context used to automatically drop all IPs if we fail to recover the node */
- TALLOC_CTX *script_monitoring_ctx; /* a context where we store results while running the monitor event */
- TALLOC_CTX *last_monitoring_ctx;
+ TALLOC_CTX *script_monitor_ctx; /* a context where we store results while running the monitor event */
+ TALLOC_CTX *last_monitor_ctx;
+ TALLOC_CTX *event_script_ctx; /* non-monitoring events */
TALLOC_CTX *banning_ctx;
};
CTDB_CONTROL_SET_DB_PRIORITY = 111,
CTDB_CONTROL_GET_DB_PRIORITY = 112,
CTDB_CONTROL_TRANSACTION_CANCEL = 113,
+ CTDB_CONTROL_REGISTER_NOTIFY = 114,
+ CTDB_CONTROL_DEREGISTER_NOTIFY = 115,
+ CTDB_CONTROL_TRANS2_ACTIVE = 116,
};
/*
struct ctdb_req_control *c);
int32_t ctdb_control_trans2_error(struct ctdb_context *ctdb,
struct ctdb_req_control *c);
+int32_t ctdb_control_trans2_active(struct ctdb_context *ctdb,
+ struct ctdb_req_control *c,
+ uint32_t db_id);
char *ctdb_addr_to_str(ctdb_sock_addr *addr);
unsigned ctdb_addr_to_port(ctdb_sock_addr *addr);
int32_t ctdb_control_get_ban_state(struct ctdb_context *ctdb, TDB_DATA *outdata);
int32_t ctdb_control_set_db_priority(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_register_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata);
+
+int32_t ctdb_control_deregister_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata);
+
+int start_syslog_daemon(struct ctdb_context *ctdb);
+
#endif
*/
#include "includes.h"
+#include "system/time.h"
#include "system/filesys.h"
#include "system/network.h"
#include "lib/util/dlinklist.h"
#include "lib/events/events_internal.h"
#include <sys/epoll.h>
+extern pid_t ctdbd_pid;
+
struct epoll_event_context {
/* a pointer back to the generic event_context */
struct event_context *ev;
*/
static int epoll_event_loop_wait(struct event_context *ev)
{
+ static time_t t=0;
+ time_t new_t;
struct epoll_event_context *epoll_ev = talloc_get_type(ev->additional_data,
struct epoll_event_context);
while (epoll_ev->num_fd_events) {
if (epoll_event_loop_once(ev) != 0) {
break;
}
+ if (getpid() == ctdbd_pid) {
+ new_t=time(NULL);
+ if (t != 0) {
+ if (t > new_t) {
+ DEBUG(0,("ERROR Time skipped backward by %d seconds\n", (int)(t-new_t)));
+ }
+ /* We assume here that we get at least one event every 5 seconds */
+ if (new_t > (t+5)) {
+ DEBUG(0,("ERROR Time jumped forward by %d seconds\n", (int)(new_t-t)));
+ }
+ }
+ t=new_t;
+ }
}
return 0;
*/
#include "includes.h"
+#include "system/time.h"
#include "system/filesys.h"
#include "system/select.h"
#include "lib/util/dlinklist.h"
#include "lib/events/events.h"
#include "lib/events/events_internal.h"
+extern pid_t ctdbd_pid;
+
struct select_event_context {
/* a pointer back to the generic event_context */
struct event_context *ev;
*/
static int select_event_loop_wait(struct event_context *ev)
{
+ static time_t t=0;
+ time_t new_t;
struct select_event_context *select_ev = talloc_get_type(ev->additional_data,
struct select_event_context);
select_ev->exit_code = 0;
if (select_event_loop_once(ev) != 0) {
break;
}
+ if (getpid() == ctdbd_pid) {
+ new_t=time(NULL);
+ if (t != 0) {
+ if (t > new_t) {
+ DEBUG(0,("ERROR Time skipped backward by %d seconds\n", (int)(t-new_t)));
+ }
+ /* We assume here that we get at least one event every 5 seconds */
+ if (new_t > (t+5)) {
+ DEBUG(0,("ERROR Time jumped forward by %d seconds\n", (int)(new_t-t)));
+ }
+ }
+ t=new_t;
+ }
}
return select_ev->exit_code;
*/
#include "includes.h"
+#include "system/time.h"
#include "system/filesys.h"
#include "system/network.h"
#include "system/select.h" /* needed for HAVE_EVENTS_EPOLL */
#include "lib/events/events.h"
#include "lib/events/events_internal.h"
+extern pid_t ctdbd_pid;
+
struct std_event_context {
/* a pointer back to the generic event_context */
struct event_context *ev;
*/
static int std_event_loop_wait(struct event_context *ev)
{
+ static time_t t=0;
+ time_t new_t;
struct std_event_context *std_ev = talloc_get_type(ev->additional_data,
struct std_event_context);
std_ev->exit_code = 0;
if (std_event_loop_once(ev) != 0) {
break;
}
+ if (getpid() == ctdbd_pid) {
+ new_t=time(NULL);
+ if (t != 0) {
+ if (t > new_t) {
+ DEBUG(0,("ERROR Time skipped backward by %d seconds\n", (int)(t-new_t)));
+ }
+ /* We assume here that we get at least one event every 5 seconds */
+ if (new_t > (t+5)) {
+ DEBUG(0,("ERROR Time jumped forward by %d seconds\n", (int)(new_t-t)));
+ }
+ }
+ t=new_t;
+ }
}
return std_ev->exit_code;
Vendor: Samba Team
Packager: Samba Team <samba@samba.org>
Name: ctdb
-Version: 1.0.98
-Release: 1
+Version: 1.0.104
+Release: 1GITHASH
Epoch: 0
License: GNU GPL version 3
Group: System Environment/Daemons
Source: ctdb-%{version}.tar.gz
-Prereq: /sbin/chkconfig /bin/mktemp /usr/bin/killall
+Prereq: /bin/mktemp /usr/bin/killall
Prereq: fileutils sed /etc/init.d
Provides: ctdb = %{version}
%clean
rm -rf $RPM_BUILD_ROOT
-%post
-[ -x /sbin/chkconfig ] && /sbin/chkconfig --add ctdb
-
-%preun
-if [ $1 = 0 ] ; then
- [ -x /sbin/chkconfig ] && /sbin/chkconfig --del ctdb
-fi
-exit 0
-
#######################################################################
## Files section ##
%{_libdir}/pkgconfig/ctdb.pc
%changelog
+* Fri Nov 6 2009 : Version 1.0.104
+ - Suggestion from Metze, we can now use killtcp to kill local connections
+ for nfs so change the killtcp script to kill both directions of an NFS
+ connection.
+ We used to deliberately only kill one direction in these cases due to
+ limitations.
+ - Suggestion from christian Ambach, when using natgw, try to avoid using a
+ UNHEALTHY node as the natgw master.
+ - From Michael Adam: Fix a SEGV bug in the recent change to the eventscripts
+ to allow the timeout to apply to each individual script.
+ - fix a talloc bug in teh vacuuming code that produced nasty valgrind
+ warnings.
+ - From Rusty: Set up ulimit to create core files for ctdb, and spawned
+ processes by default. This is useful for debugging and testing but can be
+ disabled by setting CTDB_SUPRESS_COREFILE=yes in the sysconfig file.
+ - Remove the wbinfo -t check from the startup check that winbindd is happy.
+ - Enhance the test for bond devices so we also check if the sysadmin have
+ disabled all slave devices using "ifdown".
+* Tue Nov 3 2009 : Version 1.0.103
+ - Dont use vacuuming on persistent databases
+ - Michael A : transaction updates to persistent databases
+ - Dont activate service automatically when installing the RPM. Leave this to the admin.
+ - Create a child process to send all log messages to, to prevent a hung/slow syslogd
+ from blocking the main daemon. In this case, discard log messages instead and let the child
+ process block.
+ - Michael A: updates to log messages
+* Thu Oct 29 2009 : Version 1.0.102
+ - Wolfgang: fix for the vacuuming code
+ - Wolfgang: stronger tests for persistent database filename tests
+ - Improve the log message when we refuse to startup since wbinfo -t fails
+ to make it easier to spot in the log.
+ - Update the uptime command output and the man page to indicate that
+ "time since last ..." if from either the last recovery OR the last failover
+ - Michael A: transaction updates
+* Wed Oct 28 2009 : Version 1.0.101
+ - create a separate context for non-monitoring events so they dont interfere with the monitor event
+ - make sure to return status 0 in teh callback when we abort an event
+* Wed Oct 28 2009 : Version 1.0.100
+ - Change eventscript handling to allow EventScriptTimeout for each individual script instead of for all scripts as a whole.
+ - Enhanced logging from the eventscripts, log the name and the duration for each script as it finishes.
+ - Add a check to use wbinfo -t for the startup event of samba
+ - TEMP: allow clients to attach to databases even when teh node is in recovery mode
+ - dont run the monitor event as frequently after an event has failed
+ - DEBUG: in the eventloops, check the local time and warn if the time changes backward or rapidly forward
+ - From Metze, fix a bug where recovery master becoming unhealthy did not trigger an ip failover.
+ - Disable the multipath script by default
+ - Automatically re-activate the reclock checking if the reclock file is specified at runtime. Update manpage to reflect this.
+ - Add a mechanism where samba can register a SRVID and if samba unexpectedly disconnects, a message will be broadcasted to all other samba daemons.
+ - Log the pstree on hung scripts to a file in /tmp isntead of /var/log/messages
+ - change ban count before unhealthy/banned to 10
+* Thu Oct 22 2009 : Version 1.0.99
+ - Fix a SEGV in the new db priority code.
+ - From Wolfgang : eliminate a ctdb_fatal() if there is a dmaster violation detected.
+ - During testing we often add/delete eventscripts at runtime. This could cause an eventscript to fail and mark the node unhealthy if an eventscript was deleted while we were listing the names. Handle the errorcode and make sure the node does not becomne unhealthy in this case.
+ - Lower the debuglevel for the messages when ctdb creates a filedescruiptor so we dont spam the logs with these messages.
+ - Dont have the RPM automatically restart ctdb
+ - Volker : add a missing transaction_cancel() in the handling of persistent databases
+ - Treat interfaces with the anme ethX* as bond devices in 10.interfaces so we do the correct test for if they are up or not.
* Tue Oct 20 2009 : Version 1.0.98
- Fix for the vacuuming database from Wolfgang M
- Create a directory where the test framework can put temporary overrides
SRCDIR=`rpm --eval %_sourcedir`
SPECFILE="ctdb.spec"
+SPECFILE_IN="ctdb.spec.in"
RPMBUILD="rpmbuild"
+GITHASH=".$(git log --pretty=format:%h -1)"
+
+if test "x$USE_GITHASH" = "xno" ; then
+ GITHASH=""
+fi
+
+sed -e s/GITHASH/${GITHASH}/g \
+ < ${DIRNAME}/${SPECFILE_IN} \
+ > ${DIRNAME}/${SPECFILE}
+
VERSION=$(grep ^Version ${DIRNAME}/${SPECFILE} | sed -e 's/^Version:\ \+//')
RELEASE=$(grep ^Release ${DIRNAME}/${SPECFILE} | sed -e 's/^Release:\ \+//')
ctdb_db->db_id, hdr->generation, ctdb->vnn_map->generation,
(unsigned long long)c->rsn, (unsigned long long)header.rsn, c->hdr.reqid,
(key.dsize >= 4)?(*(uint32_t *)key.dptr):0));
- /*
- * with the new vacuuming code there are conditions where a node has outdated
- * information about the real dmaster
- * since here we are lmaster and always know who is the real dmaster
- * we don't need to exit with a fatal error and we even don't have
- * to initiate a recovery
- */
+ if (header.rsn != 0 || header.dmaster != ctdb->pnn) {
+ ctdb_fatal(ctdb, "ctdb_req_dmaster from non-master");
+ return;
+ }
}
if (header.rsn > c->rsn) {
case CTDB_CONTROL_TRANS2_FINISHED:
return ctdb_control_trans2_finished(ctdb, c);
+ case CTDB_CONTROL_TRANS2_ACTIVE:
+ CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+ return ctdb_control_trans2_active(ctdb, c, *(uint32_t *)indata.dptr);
+
case CTDB_CONTROL_RECD_PING:
CHECK_CONTROL_DATA_SIZE(0);
return ctdb_control_recd_ping(ctdb);
}
if (indata.dsize > 0) {
ctdb->recovery_lock_file = talloc_strdup(ctdb, discard_const(indata.dptr));
+ ctdb->tunable.verify_recovery_lock = 1;
}
return 0;
CHECK_CONTROL_DATA_SIZE(0);
return ctdb_control_transaction_cancel(ctdb);
+ case CTDB_CONTROL_REGISTER_NOTIFY:
+ return ctdb_control_register_notify(ctdb, client_id, indata);
+
+ case CTDB_CONTROL_DEREGISTER_NOTIFY:
+ CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_client_notify_deregister));
+ return ctdb_control_deregister_notify(ctdb, client_id, indata);
+
default:
DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode));
return -1;
static void daemon_incoming_packet(void *, struct ctdb_req_header *);
-
static void print_exit_message(void)
{
DEBUG(DEBUG_NOTICE,("CTDB daemon shutting down\n"));
}
-
/* called when the "startup" event script has finished */
static void ctdb_start_transport(struct ctdb_context *ctdb)
{
/*
start the protocol going as a daemon
*/
-int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork)
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
{
int res, ret = -1;
struct fd_event *fde;
}
block_signal(SIGPIPE);
+ ctdbd_pid = getpid();
+
+
+ DEBUG(DEBUG_ERR, ("Starting CTDBD as pid : %u\n", ctdbd_pid));
+
if (ctdb->do_setsched) {
/* try to set us up as realtime */
ctdb_set_scheduler(ctdb);
DEBUG(DEBUG_CRIT,("Failed to set up signal handler for SIGCHLD\n"));
exit(1);
}
+
+ if (use_syslog) {
+ if (start_syslog_daemon(ctdb)) {
+ DEBUG(DEBUG_CRIT, ("Failed to start syslog daemon\n"));
+ exit(10);
+ }
+ }
+
/* go into a wait loop to allow other nodes to complete */
event_loop_wait(ctdb->ev);
return 0;
}
+
+
+struct ctdb_client_notify_list {
+ struct ctdb_client_notify_list *next, *prev;
+ struct ctdb_context *ctdb;
+ uint64_t srvid;
+ TDB_DATA data;
+};
+
+
+static int ctdb_client_notify_destructor(struct ctdb_client_notify_list *nl)
+{
+ int ret;
+
+ DEBUG(DEBUG_ERR,("Sending client notify message for srvid:%llu\n", (unsigned long long)nl->srvid));
+
+ ret = ctdb_daemon_send_message(nl->ctdb, CTDB_BROADCAST_CONNECTED, (unsigned long long)nl->srvid, nl->data);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to send client notify message\n"));
+ }
+
+ return 0;
+}
+
+int32_t ctdb_control_register_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata)
+{
+ struct ctdb_client_notify_register *notify = (struct ctdb_client_notify_register *)indata.dptr;
+ struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+ struct ctdb_client_notify_list *nl;
+
+ DEBUG(DEBUG_ERR,("Register srvid %llu for client %d\n", (unsigned long long)notify->srvid, client_id));
+
+ if (indata.dsize < offsetof(struct ctdb_client_notify_register, notify_data)) {
+ DEBUG(DEBUG_ERR,(__location__ " Too little data in control : %d\n", (int)indata.dsize));
+ return -1;
+ }
+
+ if (indata.dsize != (notify->len + offsetof(struct ctdb_client_notify_register, notify_data))) {
+ DEBUG(DEBUG_ERR,(__location__ " Wrong amount of data in control. Got %d, expected %d\n", (int)indata.dsize, (int)(notify->len + offsetof(struct ctdb_client_notify_register, notify_data))));
+ return -1;
+ }
+
+
+ if (client == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " Could not find client parent structure. You can not send this control to a remote node\n"));
+ return -1;
+ }
+
+ for(nl=client->notify; nl; nl=nl->next) {
+ if (nl->srvid == notify->srvid) {
+ break;
+ }
+ }
+ if (nl != NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " Notification for srvid:%llu already exists for this client\n", (unsigned long long)notify->srvid));
+ return -1;
+ }
+
+ nl = talloc(client, struct ctdb_client_notify_list);
+ CTDB_NO_MEMORY(ctdb, nl);
+ nl->ctdb = ctdb;
+ nl->srvid = notify->srvid;
+ nl->data.dsize = notify->len;
+ nl->data.dptr = talloc_size(nl, nl->data.dsize);
+ CTDB_NO_MEMORY(ctdb, nl->data.dptr);
+ memcpy(nl->data.dptr, notify->notify_data, nl->data.dsize);
+
+ DLIST_ADD(client->notify, nl);
+ talloc_set_destructor(nl, ctdb_client_notify_destructor);
+
+ return 0;
+}
+
+int32_t ctdb_control_deregister_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata)
+{
+ struct ctdb_client_notify_deregister *notify = (struct ctdb_client_notify_deregister *)indata.dptr;
+ struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+ struct ctdb_client_notify_list *nl;
+
+ DEBUG(DEBUG_ERR,("Deregister srvid %llu for client %d\n", (unsigned long long)notify->srvid, client_id));
+
+ if (client == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " Could not find client parent structure. You can not send this control to a remote node\n"));
+ return -1;
+ }
+
+ for(nl=client->notify; nl; nl=nl->next) {
+ if (nl->srvid == notify->srvid) {
+ break;
+ }
+ }
+ if (nl == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " No notification for srvid:%llu found for this client\n", (unsigned long long)notify->srvid));
+ return -1;
+ }
+
+ DLIST_REMOVE(client->notify, nl);
+ talloc_set_destructor(nl, NULL);
+ talloc_free(nl);
+
+ return 0;
+}
#include "includes.h"
#include "lib/events/events.h"
+#include "../include/ctdb.h"
#include "../include/ctdb_private.h"
#include "system/syslog.h"
#include "system/time.h"
#include "system/filesys.h"
+static bool syslogd_is_started;
+
+struct syslog_message {
+ uint32_t level;
+ uint32_t len;
+ char message[1];
+};
+
+
+
+/*
+ * this is for the syslog daemon, we can not use DEBUG here
+ */
+int start_syslog_daemon(struct ctdb_context *ctdb)
+{
+ pid_t child;
+ int syslog_fd = -1;
+ struct sockaddr_in syslog_sin;
+
+ child = fork();
+ if (child == (pid_t)-1) {
+ printf("Failed to create syslog child process\n");
+ return -1;
+ }
+
+ syslogd_is_started = 1;
+
+ if (child != 0) {
+ return 0;
+ }
+
+ syslog_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+ if (syslog_fd == -1) {
+ printf("Failed to create syslog socket\n");
+ return -1;
+ }
+
+ syslog_sin.sin_family = AF_INET;
+ syslog_sin.sin_port = htons(CTDB_PORT);
+ syslog_sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+
+ if (bind(syslog_fd, &syslog_sin, sizeof(syslog_sin)) == -1) {
+ if (errno == EADDRINUSE) {
+ /* this is ok, we already have a syslog daemon */
+ _exit(0);
+ }
+ printf("syslog daemon failed to bind to socket. errno:%d(%s)\n", errno, strerror(errno));
+ _exit(10);
+ }
+
+ /* just loop over the messages */
+ while (1) {
+ int count;
+ char str[1024];
+ struct syslog_message *msg;
+
+ count = recv(syslog_fd, str, sizeof(str), 0);
+ if (count < sizeof(struct syslog_message)) {
+ continue;
+ }
+ msg = (struct syslog_message *)str;
+
+ syslog(msg->level, "%s", msg->message);
+ }
+ _exit(10);
+}
+
struct ctdb_log_state {
int fd, pfd;
char buf[1024];
*/
static void ctdb_syslog_log(const char *format, va_list ap)
{
+ struct syslog_message *msg;
int level = LOG_DEBUG;
+ char *s = NULL;
+ int len, ret;
+ int syslog_fd;
+ struct sockaddr_in syslog_sin;
+
+ ret = vasprintf(&s, format, ap);
+ if (ret == -1) {
+ return;
+ }
+
switch (this_log_level) {
case DEBUG_EMERG:
level = LOG_EMERG;
level = LOG_DEBUG;
break;
}
- vsyslog(level, format, ap);
+
+ len = offsetof(struct syslog_message, message) + strlen(s) + 1;
+ msg = malloc(len);
+ if (msg == NULL) {
+ free(s);
+ return;
+ }
+ msg->level = level;
+ msg->len = strlen(s);
+ strcpy(msg->message, s);
+
+ if (syslogd_is_started == 0) {
+ syslog(msg->level, "%s", msg->message);
+ } else {
+ syslog_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+ if (syslog_fd == -1) {
+ printf("Failed to create syslog socket\n");
+ free(s);
+ free(msg);
+ return;
+ }
+
+ syslog_sin.sin_family = AF_INET;
+ syslog_sin.sin_port = htons(CTDB_PORT);
+ syslog_sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+
+
+ ret = sendto(syslog_fd, msg, len, 0, &syslog_sin, sizeof(syslog_sin));
+ /* no point in checking here since we cant log an error */
+
+ close(syslog_fd);
+ }
+
+ free(s);
+ free(msg);
}
}
}
+
+
/*
choose the logfile location
*/
int ctdb_set_logfile(struct ctdb_context *ctdb, const char *logfile, bool use_syslog)
{
int ret;
+
ctdb->log = talloc_zero(ctdb, struct ctdb_log_state);
if (ctdb->log == NULL) {
printf("talloc_zero failed\n");
#include "../include/ctdb_private.h"
#include "db_wrap.h"
#include "lib/util/dlinklist.h"
+#include <ctype.h>
/*
this is the dummy null procedure that all databases support
ret = ctdb_vacuum_init(ctdb_db);
if (ret != 0) {
- DEBUG(DEBUG_CRIT,("Failed to setup vacuuming for database '%s'\n", ctdb_db->db_name));
+ DEBUG(DEBUG_CRIT,("Failed to setup vacuuming for "
+ "database '%s'\n", ctdb_db->db_name));
talloc_free(ctdb_db);
return -1;
}
{
const char *db_name = (const char *)indata.dptr;
struct ctdb_db_context *db;
+#if 0
struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
+#endif
/* the client can optionally pass additional tdb flags, but we
only allow a subset of those on the database in ctdb. Note
srvid to the attach control */
tdb_flags &= TDB_NOSYNC;
- /* If the node is inactive it is not part of the cluster
- and we should not allow clients to attach to any
- databases
- */
- if (node->flags & NODE_FLAGS_INACTIVE) {
- DEBUG(DEBUG_ERR,("DB Attach to database %s refused since node is inactive (disconnected or banned)\n", db_name));
- return -1;
- }
-
-
/* see if we already have this name */
db = ctdb_db_handle(ctdb, db_name);
- if (db) {
+ if (db != NULL) {
outdata->dptr = (uint8_t *)&db->db_id;
outdata->dsize = sizeof(db->db_id);
tdb_add_flags(db->ltdb->tdb, tdb_flags);
return 0;
}
+#if 0
+ /* If the node is inactive it is not part of the cluster
+ and we should not allow clients to attach to any new
+ databases
+ */
+ if (node->flags & NODE_FLAGS_INACTIVE) {
+ DEBUG(DEBUG_ERR,("DB Attach to database %s refused since node is inactive (disconnected or banned)\n", db_name));
+ return -1;
+ }
+#endif
+
if (ctdb_local_attach(ctdb, db_name, persistent) != 0) {
return -1;
}
}
while ((de=readdir(d))) {
- char *p, *s;
+ char *p, *s, *q;
size_t len = strlen(de->d_name);
uint32_t node;
+ int invalid_name = 0;
s = talloc_strdup(ctdb, de->d_name);
CTDB_NO_MEMORY(ctdb, s);
- /* ignore names ending in .bak */
- p = strstr(s, ".bak");
- if (p != NULL) {
- continue;
- }
-
/* only accept names ending in .tdb */
p = strstr(s, ".tdb.");
if (len < 7 || p == NULL) {
talloc_free(s);
continue;
}
- if (sscanf(p+5, "%u", &node) != 1 || node != ctdb->pnn) {
+
+ /* only accept names ending with .tdb. and any number of digits */
+ q = p+5;
+ while (*q != 0 && invalid_name == 0) {
+ if (!isdigit(*q++)) {
+ invalid_name = 1;
+ }
+ }
+ if (invalid_name == 1 || sscanf(p+5, "%u", &node) != 1 || node != ctdb->pnn) {
+ DEBUG(DEBUG_ERR,("Ignoring persistent database '%s'\n", de->d_name));
talloc_free(s);
continue;
}
if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
DEBUG(DEBUG_NOTICE,("monitor event failed - disabling node\n"));
node->flags |= NODE_FLAGS_UNHEALTHY;
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
if (ctdb->tunable.disable_when_unhealthy != 0) {
DEBUG(DEBUG_INFO, ("DISABLING node since it became unhealthy\n"));
node->flags |= NODE_FLAGS_DISABLED;
} else if (status == 0 && (node->flags & NODE_FLAGS_UNHEALTHY)) {
DEBUG(DEBUG_NOTICE,("monitor event OK - node re-enabled\n"));
node->flags &= ~NODE_FLAGS_UNHEALTHY;
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
ctdb_run_notification_script(ctdb, "healthy");
} else if (status == 0) {
DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
ctdb->done_startup = true;
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
ctdb_run_notification_script(ctdb, "startup");
}
struct timeval t, void *private_data)
{
struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
- int ret;
+ int ret = 0;
if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
(ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->done_startup)) {
if (!ctdb->done_startup) {
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
ctdb->monitor->monitor_context, ctdb_startup_callback,
ctdb, "startup");
} else {
break;
}
}
- if (skip_monitoring) {
+ if (skip_monitoring != 0) {
event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
timeval_current_ofs(ctdb->monitor->next_interval, 0),
ctdb_check_health, ctdb);
return;
} else {
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
- ctdb->monitor->monitor_context, ctdb_health_callback,
- ctdb, "monitor");
+ timeval_set(ctdb->tunable.script_timeout, 0),
+ ctdb->monitor->monitor_context, ctdb_health_callback,
+ ctdb, "monitor");
}
}
if (ret != 0) {
DEBUG(DEBUG_ERR,("Unable to launch monitor event script\n"));
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
- timeval_current_ofs(1, 0),
- ctdb_check_health, ctdb);
- }
+ timeval_current_ofs(5, 0),
+ ctdb_check_health, ctdb);
+ }
}
/*
void ctdb_enable_monitoring(struct ctdb_context *ctdb)
{
ctdb->monitor->monitoring_mode = CTDB_MONITORING_ACTIVE;
- ctdb->monitor->next_interval = 2;
+ ctdb->monitor->next_interval = 5;
DEBUG(DEBUG_INFO,("Monitoring has been enabled\n"));
}
ctdb->monitor->monitor_context = NULL;
ctdb->monitor->monitoring_mode = CTDB_MONITORING_DISABLED;
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
DEBUG(DEBUG_NOTICE,("Monitoring has been stopped\n"));
}
ctdb->monitor = talloc(ctdb, struct ctdb_monitor_state);
CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor);
- ctdb->monitor->next_interval = 1;
+ ctdb->monitor->next_interval = 5;
ctdb->monitor->monitor_context = talloc_new(ctdb->monitor);
CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor->monitor_context);
node = ctdb->nodes[c->pnn];
old_flags = node->flags;
- c->old_flags = node->flags;
+ if (c->pnn != ctdb->pnn) {
+ c->old_flags = node->flags;
+ }
node->flags = c->new_flags & ~NODE_FLAGS_DISCONNECTED;
node->flags |= (c->old_flags & NODE_FLAGS_DISCONNECTED);
ctdb_db = find_ctdb_db(ctdb, m->db_id);
if (ctdb_db == NULL) {
DEBUG(DEBUG_ERR,(__location__ " ctdb_control_trans2_commit: "
- "Unknown database 0x%08x\n", m->db_id));
+ "Unknown database db_id[0x%08x]\n", m->db_id));
return -1;
}
switch (c->opcode) {
case CTDB_CONTROL_PERSISTENT_STORE:
if (ctdb_db->transaction_active) {
- DEBUG(DEBUG_ERR, (__location__ " trans2_commit client db_id[%d] transaction active - refusing persistent store\n",
- client->db_id));
+ DEBUG(DEBUG_ERR, (__location__ " trans2_commit: a "
+ "transaction is active on database "
+ "db_id[0x%08x] - refusing persistent "
+ " store for client id[0x%08x]\n",
+ ctdb_db->db_id, client->client_id));
return -1;
}
if (client->num_persistent_updates > 0) {
break;
case CTDB_CONTROL_TRANS2_COMMIT:
if (ctdb_db->transaction_active) {
- DEBUG(DEBUG_ERR,(__location__ " trans2_commit: client "
- "already has a transaction commit "
- "active on db_id[%d]\n",
- client->db_id));
+ DEBUG(DEBUG_ERR,(__location__ " trans2_commit: there is"
+ " already a transaction commit "
+ "active on db_id[0x%08x] - forbidding "
+ "client_id[0x%08x] to commit\n",
+ ctdb_db->db_id, client->client_id));
return -1;
}
if (client->db_id != 0) {
DEBUG(DEBUG_ERR,(__location__ " ERROR: trans2_commit: "
- "client-db_id[%d] != 0\n",
- client->db_id));
+ "client-db_id[0x%08x] != 0 "
+ "(client_id[0x%08x])\n",
+ client->db_id, client->client_id));
return -1;
}
client->num_persistent_updates++;
ctdb_db->transaction_active = true;
client->db_id = m->db_id;
+ DEBUG(DEBUG_DEBUG, (__location__ " client id[0x%08x] started to"
+ " commit transaction on db id[0x%08x]\n",
+ client->client_id, client->db_id));
break;
case CTDB_CONTROL_TRANS2_COMMIT_RETRY:
/* already updated from the first commit */
if (client->db_id != m->db_id) {
DEBUG(DEBUG_ERR,(__location__ " ERROR: trans2_commit "
- "retry: client-db_id[%d] != db_id[%d]"
- "\n", client->db_id, m->db_id));
+ "retry: client-db_id[0x%08x] != "
+ "db_id[0x%08x] (client_id[0x%08x])\n",
+ client->db_id,
+ m->db_id, client->client_id));
return -1;
}
+ DEBUG(DEBUG_DEBUG, (__location__ " client id[0x%08x] started "
+ "transaction commit retry on "
+ "db_id[0x%08x]\n",
+ client->client_id, client->db_id));
break;
}
}
client->num_persistent_updates--;
+ DEBUG(DEBUG_DEBUG, (__location__ " client id[0x%08x] finished "
+ "transaction commit db_id[0x%08x]\n",
+ client->client_id, ctdb_db->db_id));
+
return 0;
}
client->num_persistent_updates--;
}
- DEBUG(DEBUG_ERR,(__location__ " Forcing recovery\n"));
+ DEBUG(DEBUG_ERR,(__location__ " An error occurred during transaction on"
+ " db_id[0x%08x] - forcing recovery\n",
+ ctdb_db->db_id));
client->ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
return 0;
}
+/**
+ * Tell whether a transaction is active on this node on the give DB.
+ */
+int32_t ctdb_control_trans2_active(struct ctdb_context *ctdb,
+ struct ctdb_req_control *c,
+ uint32_t db_id)
+{
+ struct ctdb_db_context *ctdb_db;
+ struct ctdb_client *client = ctdb_reqid_find(ctdb, c->client_id, struct ctdb_client);
+
+ ctdb_db = find_ctdb_db(ctdb, db_id);
+ if (!ctdb_db) {
+ DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", db_id));
+ return -1;
+ }
+
+ if (client->db_id == db_id) {
+ return 0;
+ }
+
+ if (ctdb_db->transaction_active) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
/*
backwards compatibility:
ctdb_disable_monitoring(ctdb);
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state,
ctdb_end_recovery_callback,
state, "recovered");
ctdb_disable_monitoring(ctdb);
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state,
ctdb_start_recovery_callback,
state, "startrecovery");
ctdb_disable_monitoring(ctdb);
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state,
ctdb_stop_node_callback,
state, "stopped");
for (callers=rec->reallocate_callers; callers; callers=callers->next) {
DEBUG(DEBUG_INFO,("Sending ip reallocate reply message to "
- "%u:%lu\n", (unsigned)callers->rd->pnn,
- (long unsigned)callers->rd->srvid));
+ "%u:%llu\n", (unsigned)callers->rd->pnn,
+ (unsigned long long)callers->rd->srvid));
ret = ctdb_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to send ip reallocate reply "
- "message to %u:%lu\n",
+ "message to %u:%llu\n",
(unsigned)callers->rd->pnn,
- (long unsigned)callers->rd->srvid));
+ (unsigned long long)callers->rd->srvid));
}
}
{
int ret;
struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)data.dptr;
+ struct ctdb_node_map *nodemap=NULL;
+ TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+ uint32_t recmaster;
+ uint32_t *nodes;
- ret = ctdb_ctrl_modflags(ctdb, CONTROL_TIMEOUT(), c->pnn, c->new_flags, ~c->new_flags);
+ /* find the recovery master */
+ ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to update nodeflags on remote nodes\n"));
+ DEBUG(DEBUG_ERR, (__location__ " Unable to get recmaster from local node\n"));
+ talloc_free(tmp_ctx);
+ return;
+ }
+
+ /* read the node flags from the recmaster */
+ ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), recmaster, tmp_ctx, &nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to get nodemap from node %u\n", c->pnn));
+ talloc_free(tmp_ctx);
+ return;
}
+ if (c->pnn >= nodemap->num) {
+ DEBUG(DEBUG_ERR,(__location__ " Nodemap from recmaster does not contain node %d\n", c->pnn));
+ talloc_free(tmp_ctx);
+ return;
+ }
+
+ /* send the flags update to all connected nodes */
+ nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
+
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_MODIFY_FLAGS,
+ nodes, 0, CONTROL_TIMEOUT(),
+ false, data,
+ NULL, NULL,
+ NULL) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " ctdb_control to modify node flags failed\n"));
+
+ talloc_free(tmp_ctx);
+ return;
+ }
+
+ talloc_free(tmp_ctx);
}
vnn->iface));
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state, takeover_ip_callback, state,
"takeip %s %s %u",
vnn->iface,
state->vnn = vnn;
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state, release_ip_callback, state,
"releaseip %s %s %u",
vnn->iface,
DLIST_REMOVE(ctdb->vnn, vnn);
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
mem_ctx, delete_ip_callback, mem_ctx,
"releaseip %s %s %u",
vnn->iface,
{ "TakeoverTimeout", 5, offsetof(struct ctdb_tunable, takeover_timeout) },
{ "MonitorInterval", 15, offsetof(struct ctdb_tunable, monitor_interval) },
{ "TickleUpdateInterval",20, offsetof(struct ctdb_tunable, tickle_update_interval) },
- { "EventScriptTimeout", 60, offsetof(struct ctdb_tunable, script_timeout) },
- { "EventScriptBanCount", 5, offsetof(struct ctdb_tunable, script_ban_count) },
+ { "EventScriptTimeout", 30, offsetof(struct ctdb_tunable, script_timeout) },
+ { "EventScriptBanCount", 10, offsetof(struct ctdb_tunable, script_ban_count) },
{ "EventScriptUnhealthyOnTimeout", 0, offsetof(struct ctdb_tunable, script_unhealthy_on_timeout) },
{ "RecoveryGracePeriod", 120, offsetof(struct ctdb_tunable, recovery_grace_period) },
{ "RecoveryBanPeriod", 300, offsetof(struct ctdb_tunable, recovery_ban_period) },
hash = ctdb_hash(&key);
if (trbt_lookup32(vdata->delete_tree, hash)) {
- DEBUG(DEBUG_INFO, (__location__ " Hash collission when vacuuming, skipping this record.\n"));
+ DEBUG(DEBUG_DEBUG, (__location__ " Hash collission when vacuuming, skipping this record.\n"));
}
else {
struct delete_record_data *dd;
* there might be hash collisions so we have to compare the keys here to be sure
*/
if (kd && kd->key.dsize == key.dsize && memcmp(kd->key.dptr, key.dptr, key.dsize) == 0) {
- vdata->vacuumed++;
- return 0;
+ struct ctdb_ltdb_header *hdr = (struct ctdb_ltdb_header *)data.dptr;
+ /*
+ * we have to check if the record hasn't changed in the meantime in order to
+ * savely remove it from the database
+ */
+ if (data.dsize == sizeof(struct ctdb_ltdb_header) &&
+ hdr->dmaster == kd->ctdb->pnn &&
+ ctdb_lmaster(kd->ctdb, &(kd->key)) == kd->ctdb->pnn &&
+ kd->hdr.rsn == hdr->rsn) {
+ vdata->vacuumed++;
+ return 0;
+ }
}
}
if (tdb_store(vdata->dest_db, key, data, TDB_INSERT) != 0) {
* decide if a repack is necessary
*/
if (size < repack_limit && vdata->delete_count < vacuum_limit) {
- talloc_free(vdata);
update_tuning_db(ctdb_db, vdata, size);
+ talloc_free(vdata);
return 0;
}
*/
int ctdb_vacuum_init(struct ctdb_db_context *ctdb_db)
{
+ if (ctdb_db->persistent != 0) {
+ DEBUG(DEBUG_ERR,("Vacuuming is disabled for persistent database %s\n", ctdb_db->db_name));
+ return 0;
+ }
+
ctdb_db->vacuum_handle = talloc(ctdb_db, struct ctdb_vacuum_handle);
CTDB_NO_MEMORY(ctdb_db->ctdb, ctdb_db->vacuum_handle);
}
/* start the protocol running (as a child) */
- return ctdb_start_daemon(ctdb, interactive?False:True);
+ return ctdb_start_daemon(ctdb, interactive?False:True, options.use_syslog);
}
*/
#include "includes.h"
+#include <time.h>
#include "system/filesys.h"
#include "system/wait.h"
#include "system/dir.h"
const char *script_running;
} child_state;
+static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p);
+
/*
ctdbd sends us a SIGTERM when we should time out the current script
*/
static void sigterm(int sig)
{
- FILE *p;
+ char tbuf[100], buf[200];
+ time_t t;
DEBUG(DEBUG_ERR,("Timed out running script '%s' after %.1f seconds pid :%d\n",
child_state.script_running, timeval_elapsed(&child_state.start), getpid()));
- p = popen("pstree -p", "r");
- if (p == NULL) {
- DEBUG(DEBUG_ERR,("Failed popen to collect pstree for hung script\n"));
- } else {
- char buf[256];
- int count;
-
- DEBUG(DEBUG_ERR,("PSTREE:\n"));
- while(!feof(p)){
- count=fread(buf, 1, 255, p);
- if (count == EOF) {
- break;
- }
- if (count < 0) {
- break;
- }
- if (count == 0) {
- break;
- }
- buf[count] = 0;
- DEBUG(DEBUG_ERR,("%s", buf));
- }
- DEBUG(DEBUG_ERR,("END OF PSTREE OUTPUT\n"));
- pclose(p);
- }
+ t = time(NULL);
+
+ strftime(tbuf, sizeof(tbuf)-1, "%Y%m%d%H%M%S", localtime(&t));
+ sprintf(buf, "pstree -p >/tmp/ctdb.event.%s.%d", tbuf, getpid());
+ system(buf);
+
+ DEBUG(DEBUG_ERR,("Logged timedout eventscript : %s\n", buf));
/* all the child processes will be running in the same process group */
kill(-getpgrp(), SIGKILL);
- exit(1);
+ _exit(1);
}
struct ctdb_event_script_state {
int fd[2];
void *private_data;
const char *options;
+ struct timed_event *te;
+ struct timeval timeout;
};
char *output;
};
-struct ctdb_monitoring_status {
+struct ctdb_monitor_status {
struct timeval start;
struct timeval finished;
int32_t status;
struct ctdb_monitor_script_status *scripts;
+ struct ctdb_event_script_state *state;
};
*/
int ctdb_log_event_script_output(struct ctdb_context *ctdb, char *str, uint16_t len)
{
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
struct ctdb_monitor_script_status *script;
if (monitoring_status == NULL) {
*/
int32_t ctdb_control_event_script_init(struct ctdb_context *ctdb)
{
- struct ctdb_monitoring_status *monitoring_status;
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
DEBUG(DEBUG_INFO, ("event script init called\n"));
- if (ctdb->script_monitoring_ctx != NULL) {
- talloc_free(ctdb->script_monitoring_ctx);
- ctdb->script_monitoring_ctx = NULL;
- }
- monitoring_status = talloc_zero(ctdb, struct ctdb_monitoring_status);
if (monitoring_status == NULL) {
- DEBUG(DEBUG_ERR, (__location__ " ERROR: Failed to talloc script_monitoring context\n"));
- return -1;
+ DEBUG(DEBUG_ERR,(__location__ " Init called when context is NULL\n"));
+ return 0;
}
- ctdb->script_monitoring_ctx = monitoring_status;
monitoring_status->start = timeval_current();
return 0;
int32_t ctdb_control_event_script_start(struct ctdb_context *ctdb, TDB_DATA indata)
{
const char *name = (const char *)indata.dptr;
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
+ struct ctdb_event_script_state *state;
struct ctdb_monitor_script_status *script;
DEBUG(DEBUG_INFO, ("event script start called : %s\n", name));
script->start = timeval_current();
monitoring_status->scripts = script;
+ state = monitoring_status->state;
+ if (state != NULL) {
+ /* reset the timeout for the next eventscript */
+ if (!timeval_is_zero(&state->timeout)) {
+ if (state->te != NULL) {
+ talloc_free(state->te);
+ state->te = NULL;
+ }
+ state->te = event_add_timed(ctdb->ev, state, timeval_current_ofs(state->timeout.tv_sec, state->timeout.tv_usec), ctdb_event_script_timeout, state);
+ }
+
+ }
+
return 0;
}
int32_t ctdb_control_event_script_stop(struct ctdb_context *ctdb, TDB_DATA indata)
{
int32_t res = *((int32_t *)indata.dptr);
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
struct ctdb_monitor_script_status *script;
- DEBUG(DEBUG_INFO, ("event script stop called : %d\n", (int)res));
-
if (monitoring_status == NULL) {
DEBUG(DEBUG_ERR,(__location__ " script_status is NULL when script finished.\n"));
return -1;
script->finished = timeval_current();
script->status = res;
+ DEBUG(DEBUG_INFO, ("event script stop called for script:%s duration:%.1f status:%d\n", script->name, timeval_elapsed(&script->start), (int)res));
+
return 0;
}
int32_t ctdb_control_event_script_disabled(struct ctdb_context *ctdb, TDB_DATA indata)
{
const char *name = (const char *)indata.dptr;
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
struct ctdb_monitor_script_status *script;
DEBUG(DEBUG_INFO, ("event script disabed called for script %s\n", name));
*/
int32_t ctdb_control_event_script_finished(struct ctdb_context *ctdb)
{
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
DEBUG(DEBUG_INFO, ("event script finished called\n"));
monitoring_status->finished = timeval_current();
monitoring_status->status = MONITOR_SCRIPT_OK;
- if (ctdb->last_monitoring_ctx) {
- talloc_free(ctdb->last_monitoring_ctx);
+
+ if (ctdb->last_monitor_ctx) {
+ talloc_free(ctdb->last_monitor_ctx);
+ ctdb->last_monitor_ctx = NULL;
}
- ctdb->last_monitoring_ctx = ctdb->script_monitoring_ctx;
- ctdb->script_monitoring_ctx = NULL;
+ ctdb->last_monitor_ctx = talloc_steal(ctdb, ctdb->script_monitor_ctx);
+ ctdb->script_monitor_ctx = NULL;
return 0;
}
int32_t ctdb_control_get_event_script_status(struct ctdb_context *ctdb, TDB_DATA *outdata)
{
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->last_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->last_monitor_ctx;
struct ctdb_monitoring_wire *monitoring_scripts;
if (monitoring_status == NULL) {
{
struct ctdb_event_script_state *state =
talloc_get_type(p, struct ctdb_event_script_state);
- void (*callback)(struct ctdb_context *, int, void *) = state->callback;
- void *private_data = state->private_data;
struct ctdb_context *ctdb = state->ctdb;
signed char rt = -1;
DEBUG(DEBUG_INFO,(__location__ " Eventscript %s finished with state %d\n", state->options, rt));
+ if (state->callback) {
+ state->callback(ctdb, rt, state->private_data);
+ state->callback = NULL;
+ }
+
talloc_set_destructor(state, NULL);
talloc_free(state);
- callback(ctdb, rt, private_data);
-
ctdb->event_script_timeouts = 0;
}
struct timeval t, void *p)
{
struct ctdb_event_script_state *state = talloc_get_type(p, struct ctdb_event_script_state);
- void (*callback)(struct ctdb_context *, int, void *) = state->callback;
void *private_data = state->private_data;
struct ctdb_context *ctdb = state->ctdb;
char *options;
- struct ctdb_monitoring_status *monitoring_status =
- talloc_get_type(ctdb->script_monitoring_ctx,
- struct ctdb_monitoring_status);
+ struct ctdb_monitor_status *monitoring_status = (struct ctdb_monitor_status *)ctdb->script_monitor_ctx;
+
+ state->te = NULL;
DEBUG(DEBUG_ERR,("Event script timed out : %s count : %u pid : %d\n", state->options, ctdb->event_script_timeouts, state->child));
if (kill(state->child, 0) != 0) {
DEBUG(DEBUG_ERR,("Event script child process already dead, errno %s(%d)\n", strerror(errno), errno));
- callback(ctdb, 0, private_data);
-
+ if (state->callback) {
+ state->callback(ctdb, 0, private_data);
+ state->callback = NULL;
+ }
talloc_set_destructor(state, NULL);
talloc_free(state);
return;
options = talloc_strdup(ctdb, state->options);
CTDB_NO_MEMORY_VOID(ctdb, options);
- talloc_free(state);
if (!strcmp(options, "monitor")) {
/* if it is a monitor event, we allow it to "hang" a few times
before we declare it a failure and ban ourself (and make
if (ctdb->event_script_timeouts > ctdb->tunable.script_ban_count) {
if (ctdb->tunable.script_unhealthy_on_timeout != 0) {
DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_ban_count));
- callback(ctdb, -ETIME, private_data);
+ if (state->callback) {
+ state->callback(ctdb, -ETIME, private_data);
+ state->callback = NULL;
+ }
} else {
ctdb->event_script_timeouts = 0;
DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Banning self for %d seconds\n", ctdb->tunable.script_ban_count, ctdb->tunable.recovery_ban_period));
ctdb_ban_self(ctdb, ctdb->tunable.recovery_ban_period);
- callback(ctdb, -1, private_data);
+ if (state->callback) {
+ state->callback(ctdb, -1, private_data);
+ state->callback = NULL;
+ }
}
} else {
- callback(ctdb, 0, private_data);
+ if (state->callback) {
+ state->callback(ctdb, 0, private_data);
+ state->callback = NULL;
+ }
}
} else if (!strcmp(options, "startup")) {
DEBUG(DEBUG_ERR, (__location__ " eventscript for startup event timedout.\n"));
- callback(ctdb, -1, private_data);
+ if (state->callback) {
+ state->callback(ctdb, -1, private_data);
+ state->callback = NULL;
+ }
} else {
/* if it is not a monitor event we ban ourself immediately */
DEBUG(DEBUG_ERR, (__location__ " eventscript for NON-monitor/NON-startup event timedout. Immediately banning ourself for %d seconds\n", ctdb->tunable.recovery_ban_period));
ctdb_ban_self(ctdb, ctdb->tunable.recovery_ban_period);
- callback(ctdb, -1, private_data);
+ if (state->callback) {
+ state->callback(ctdb, -1, private_data);
+ state->callback = NULL;
+ }
}
- if (monitoring_status != NULL) {
+ if ((!strcmp(options, "monitor")) && (monitoring_status != NULL)) {
struct ctdb_monitor_script_status *script;
script = monitoring_status->scripts;
script->timedout = 1;
}
monitoring_status->status = MONITOR_SCRIPT_TIMEOUT;
- if (ctdb->last_monitoring_ctx) {
- talloc_free(ctdb->last_monitoring_ctx);
- ctdb->last_monitoring_ctx = ctdb->script_monitoring_ctx;
- ctdb->script_monitoring_ctx = NULL;
+ if (ctdb->last_monitor_ctx) {
+ talloc_free(ctdb->last_monitor_ctx);
+ ctdb->last_monitor_ctx = NULL;
}
+ ctdb->last_monitor_ctx = talloc_steal(ctdb, ctdb->script_monitor_ctx);
+ ctdb->script_monitor_ctx = NULL;
}
+ talloc_free(state);
talloc_free(options);
}
{
DEBUG(DEBUG_ERR,(__location__ " Sending SIGTERM to child pid:%d\n", state->child));
+ if (state->callback) {
+ state->callback(state->ctdb, 0, state->private_data);
+ state->callback = NULL;
+ }
+
if (kill(state->child, SIGTERM) != 0) {
DEBUG(DEBUG_ERR,("Failed to kill child process for eventscript, errno %s(%d)\n", strerror(errno), errno));
}
*/
static int ctdb_event_script_callback_v(struct ctdb_context *ctdb,
struct timeval timeout,
- TALLOC_CTX *mem_ctx,
void (*callback)(struct ctdb_context *, int, void *),
void *private_data,
const char *fmt, va_list ap)
{
+ struct ctdb_monitor_status *monitoring_status;
struct ctdb_event_script_state *state;
int ret;
- state = talloc(mem_ctx, struct ctdb_event_script_state);
- CTDB_NO_MEMORY(ctdb, state);
+ if (!strcmp(fmt, "monitor")) {
+ if (ctdb->script_monitor_ctx != NULL) {
+ talloc_free(ctdb->script_monitor_ctx);
+ ctdb->script_monitor_ctx = NULL;
+ }
+ monitoring_status = talloc_zero(ctdb, struct ctdb_monitor_status);
+ } else {
+ if (ctdb->event_script_ctx == NULL) {
+ ctdb->event_script_ctx = talloc_zero(ctdb, struct ctdb_monitor_status);
+ }
+ monitoring_status = ctdb->event_script_ctx;
+ }
+
+ if (monitoring_status == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " ERROR: Failed to talloc script_monitoring context\n"));
+ return -1;
+ }
+
+ state = talloc(monitoring_status, struct ctdb_event_script_state);
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " could not allocate state\n"));
+ return -1;
+ }
+ monitoring_status->state = state;
state->ctdb = ctdb;
state->callback = callback;
state->private_data = private_data;
state->options = talloc_vasprintf(state, fmt, ap);
- CTDB_NO_MEMORY(ctdb, state->options);
+ state->timeout = timeout;
+ state->te = NULL;
+ if (state->options == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " could not allocate state->options\n"));
+ talloc_free(state);
+ return -1;
+ }
DEBUG(DEBUG_INFO,(__location__ " Starting eventscript %s\n", state->options));
rt = ctdb_event_script_v(ctdb, state->options);
while ((ret = write(state->fd[1], &rt, sizeof(rt))) != sizeof(rt)) {
- sleep(1);
+ write(state->fd[1], &rt, sizeof(rt));
+ usleep(100000);
}
_exit(rt);
}
talloc_set_destructor(state, event_script_destructor);
+ if (!strcmp(fmt, "monitor")) {
+ ctdb->script_monitor_ctx = monitoring_status;
+ } else {
+ ctdb->event_script_ctx = monitoring_status;
+ }
close(state->fd[1]);
set_close_on_exec(state->fd[0]);
event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
ctdb_event_script_handler, state);
- if (!timeval_is_zero(&timeout)) {
- event_add_timed(ctdb->ev, state, timeout, ctdb_event_script_timeout, state);
+ if (!timeval_is_zero(&state->timeout)) {
+ state->te = event_add_timed(ctdb->ev, state, timeval_current_ofs(state->timeout.tv_sec, state->timeout.tv_usec), ctdb_event_script_timeout, state);
} else {
DEBUG(DEBUG_ERR, (__location__ " eventscript %s called with no timeout\n", state->options));
}
int ret;
va_start(ap, fmt);
- ret = ctdb_event_script_callback_v(ctdb, timeout, mem_ctx, callback, private_data, fmt, ap);
+ ret = ctdb_event_script_callback_v(ctdb, timeout, callback, private_data, fmt, ap);
va_end(ap);
return ret;
{
va_list ap;
int ret;
- TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
struct callback_status status;
va_start(ap, fmt);
ret = ctdb_event_script_callback_v(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
- tmp_ctx, event_script_callback, &status, fmt, ap);
+ timeval_set(ctdb->tunable.script_timeout, 0),
+ event_script_callback, &status, fmt, ap);
va_end(ap);
if (ret != 0) {
- talloc_free(tmp_ctx);
return ret;
}
while (status.done == false && event_loop_once(ctdb->ev) == 0) /* noop */;
- talloc_free(tmp_ctx);
-
return status.status;
}
ctdb_disable_monitoring(ctdb);
ret = ctdb_event_script_callback(ctdb,
- timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+ timeval_set(ctdb->tunable.script_timeout, 0),
state, run_eventscripts_callback, state,
"%s", (const char *)indata.dptr);
setsockopt(ctcp->listen_fd,SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(one));
if (bind(ctcp->listen_fd, (struct sockaddr * )&sock, sock_size) == 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to bind() to socket. %s(%d)\n", strerror(errno), errno));
break;
}
+
+ if (errno == EADDRNOTAVAIL) {
+ DEBUG(DEBUG_DEBUG,(__location__ " Failed to bind() to socket. %s(%d)\n",
+ strerror(errno), errno));
+ } else {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to bind() to socket. %s(%d)\n",
+ strerror(errno), errno));
+ }
}
if (i == ctdb->num_nodes) {
ret = ctdb_transaction_commit(h);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
- //exit(1);
+ exit(1);
}
if (verbose) printf("transaction committed\n");
} else {
#include "../common/rb_tree.h"
#include "db_wrap.h"
-
#define ERR_TIMEOUT 20 /* timed out trying to reach node */
#define ERR_NONODE 21 /* node does not exist */
#define ERR_DISNODE 22 /* node is disconnected */
}
if (options.machinereadable){
- printf(":Current Node Time:Ctdb Start Time:Last Recovery Time:Last Recovery Duration:\n");
+ printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
printf(":%u:%u:%u:%lf\n",
(unsigned int)uptime->current_time.tv_sec,
(unsigned int)uptime->ctdbd_start_time.tv_sec,
return 0;
}
- printf("Current time of node : %s", ctime(&uptime->current_time.tv_sec));
+ printf("Current time of node : %s", ctime(&uptime->current_time.tv_sec));
tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
seconds = tmp%60;
hours = tmp%24;
tmp /= 24;
days = tmp;
- printf("Ctdbd start time : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
+ printf("Ctdbd start time : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
seconds = tmp%60;
hours = tmp%24;
tmp /= 24;
days = tmp;
- printf("Time of last recovery : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
+ printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
- printf("Duration of last recovery : %lf seconds\n",
+ printf("Duration of last recovery/failover: %lf seconds\n",
timeval_delta(&uptime->last_recovery_finished,
&uptime->last_recovery_started));
i++;
}
- /* print the natgw master
- * we dont allow STOPPED or DELETED nodes to become the natgwmaster
+ /* pick a node to be natgwmaster
+ * we dont allow STOPPED, DELETED, BANNED or UNHEALTHY nodes to become the natgwmaster
*/
for(i=0;i<nodemap->num;i++){
- if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
+ if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY))) {
printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
break;
}
}
+ /* we couldnt find any healthy node, try unhealthy ones */
+ if (i == nodemap->num) {
+ for(i=0;i<nodemap->num;i++){
+ if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
+ printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
+ break;
+ }
+ }
+ }
/* unless all nodes are STOPPED, when we pick one anyway */
if (i == nodemap->num) {
for(i=0;i<nodemap->num;i++){
show_all "free"
show_all "crontab -l"
show_all "sysctl -a"
-show_all "/sbin/iptables -L -n"
-show_all "/sbin/iptables -L -n -t nat"
+show_all "iptables -L -n"
+show_all "iptables -L -n -t nat"
show_all "/usr/sbin/rpcinfo -p"
show_all "/usr/sbin/showmount -a"
show_all "/usr/sbin/showmount -e"
git.samba.org / metze / ctdb / wip.git / commitdiff