*/
#include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
#include "system/filesys.h"
#include "system/time.h"
#include "system/network.h"
struct ip_reallocate_list *reallocate_callers;
TALLOC_CTX *ip_check_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
+ TALLOC_CTX *deferred_rebalance_ctx;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
#define MONITOR_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_interval, 0)
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event *te, struct timeval t, void *private_data);
/*
ban a node for a period of time
ZERO_STRUCT(call);
call.call_id = CTDB_NULL_FUNC;
call.flags = CTDB_IMMEDIATE_MIGRATION;
+ call.flags |= CTDB_CALL_FLAG_VACUUM_MIGRATION;
r = v->r;
v->r = (struct ctdb_rec_data *)(r->length + (uint8_t *)r);
hdr = (struct ctdb_ltdb_header *)data.dptr;
if (!params->persistent) {
hdr->dmaster = params->ctdb->pnn;
+ hdr->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
}
/* add the record to the blob ready to send to the nodes */
return -1;
}
- if (rec->ip_check_disable_ctx == NULL) {
- if (verify_remote_ip_allocation(ctdb, ctdb->nodes[j]->known_public_ips)) {
- DEBUG(DEBUG_ERR,("Node %d has inconsistent public ip allocation and needs update.\n", ctdb->nodes[j]->pnn));
- rec->need_takeover_run = true;
+ if (ctdb->tunable.disable_ip_failover == 0) {
+ if (rec->ip_check_disable_ctx == NULL) {
+ if (verify_remote_ip_allocation(ctdb, ctdb->nodes[j]->known_public_ips)) {
+ DEBUG(DEBUG_ERR,("Node %d has inconsistent public ip allocation and needs update.\n", ctdb->nodes[j]->pnn));
+ rec->need_takeover_run = true;
+ }
}
}
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
+ rec->need_takeover_run = true;
return -1;
}
rec->need_takeover_run = false;
ret = ctdb_takeover_run(ctdb, nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses\n"));
- return -1;
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. ctdb_takeover_run() failed.\n"));
+ rec->need_takeover_run = true;
}
- DEBUG(DEBUG_NOTICE, (__location__ " Recovery - takeip finished\n"));
/* execute the "recovered" event script on all nodes */
ret = run_recovered_eventscript(ctdb, nodemap, "do_recovery");
We now wait for rerecovery_timeout before we allow
another recovery to take place.
*/
- DEBUG(DEBUG_NOTICE, (__location__ " New recoveries supressed for the rerecovery timeout\n"));
+ DEBUG(DEBUG_NOTICE, ("Just finished a recovery. New recoveries will now be supressed for the rerecovery timeout (%d seconds)\n", ctdb->tunable.rerecovery_timeout));
ctdb_wait_timeout(ctdb, ctdb->tunable.rerecovery_timeout);
- DEBUG(DEBUG_NOTICE, (__location__ " Rerecovery timeout elapsed. Recovery reactivated.\n"));
+ DEBUG(DEBUG_NOTICE, ("The rerecovery timeout has elapsed. We now allow recoveries to trigger again.\n"));
return 0;
}
}
+static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *p)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
+ struct ctdb_context *ctdb = rec->ctdb;
+ int ret;
+
+ DEBUG(DEBUG_NOTICE,("Rebalance all nodes that have had ip assignment changes.\n"));
+
+ ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. ctdb_takeover_run() failed.\n"));
+ rec->need_takeover_run = true;
+ }
+
+ talloc_free(rec->deferred_rebalance_ctx);
+ rec->deferred_rebalance_ctx = NULL;
+}
+
+
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ uint32_t pnn;
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+
+ if (data.dsize != sizeof(uint32_t)) {
+ DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
+ return;
+ }
+
+ if (ctdb->tunable.deferred_rebalance_on_node_add == 0) {
+ return;
+ }
+
+ pnn = *(uint32_t *)&data.dptr[0];
+
+ lcp2_forcerebalance(ctdb, pnn);
+ DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+
+ if (rec->deferred_rebalance_ctx != NULL) {
+ talloc_free(rec->deferred_rebalance_ctx);
+ }
+ rec->deferred_rebalance_ctx = talloc_new(rec);
+ event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+ timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
+ ctdb_rebalance_timeout, rec);
+}
+
+
+
static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid,
TDB_DATA data, void *private_data)
{
if (ret == 0) {
ret = ctdb_takeover_run(ctdb, rec->nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
- culprit));
+ DEBUG(DEBUG_ERR,("Failed to reallocate addresses: ctdb_takeover_run() failed.\n"));
rec->need_takeover_run = true;
}
}
/* called to check that the local allocation of public ip addresses is ok.
*/
-static int verify_local_ip_allocation(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, uint32_t pnn)
+static int verify_local_ip_allocation(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, uint32_t pnn, struct ctdb_node_map *nodemap)
{
TALLOC_CTX *mem_ctx = talloc_new(NULL);
struct ctdb_control_get_ifaces *ifaces = NULL;
/* skip the check if we have started but not finished recovery */
if (timeval_compare(&uptime1->last_recovery_finished,
&uptime1->last_recovery_started) != 1) {
- DEBUG(DEBUG_NOTICE, (__location__ " in the middle of recovery or ip reallocation. skipping public ip address check\n"));
+ DEBUG(DEBUG_INFO, (__location__ " in the middle of recovery or ip reallocation. skipping public ip address check\n"));
talloc_free(mem_ctx);
return 0;
and we dont have ones we shouldnt have.
if we find an inconsistency we set recmode to
active on the local node and wait for the recmaster
- to do a full blown recovery
+ to do a full blown recovery.
+ also if the pnn is -1 and we are healthy and can host the ip
+ we also request a ip reallocation.
*/
- for (j=0; j<ips->num; j++) {
- if (ips->ips[j].pnn == pnn) {
- if (!ctdb_sys_have_ip(&ips->ips[j].addr)) {
- DEBUG(DEBUG_CRIT,("Public address '%s' is missing and we should serve this ip\n",
- ctdb_addr_to_str(&ips->ips[j].addr)));
- need_takeover_run = true;
- }
- } else {
- if (ctdb_sys_have_ip(&ips->ips[j].addr)) {
- DEBUG(DEBUG_CRIT,("We are still serving a public address '%s' that we should not be serving.\n",
- ctdb_addr_to_str(&ips->ips[j].addr)));
+ if (ctdb->tunable.disable_ip_failover == 0) {
+ for (j=0; j<ips->num; j++) {
+ if (ips->ips[j].pnn == -1 && nodemap->nodes[pnn].flags == 0) {
+ DEBUG(DEBUG_CRIT,("Public address '%s' is not assigned and we could serve this ip\n",
+ ctdb_addr_to_str(&ips->ips[j].addr)));
need_takeover_run = true;
+ } else if (ips->ips[j].pnn == pnn) {
+ if (!ctdb_sys_have_ip(&ips->ips[j].addr)) {
+ DEBUG(DEBUG_CRIT,("Public address '%s' is missing and we should serve this ip\n",
+ ctdb_addr_to_str(&ips->ips[j].addr)));
+ need_takeover_run = true;
+ }
+ } else {
+ if (ctdb_sys_have_ip(&ips->ips[j].addr)) {
+ DEBUG(DEBUG_CRIT,("We are still serving a public address '%s' that we should not be serving.\n",
+ ctdb_addr_to_str(&ips->ips[j].addr)));
+ need_takeover_run = true;
+ }
}
}
}
return -1;
}
- state->child = fork();
+ state->child = ctdb_fork(ctdb);
if (state->child == (pid_t)-1) {
DEBUG(DEBUG_CRIT,(__location__ " fork() failed in check_reclock child\n"));
close(state->fd[0]);
}
state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
- EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+ EVENT_FD_READ,
reclock_child_handler,
(void *)state);
talloc_free(state);
return -1;
}
+ tevent_fd_set_auto_close(state->fde);
while (state->status == RECLOCK_CHECKING) {
event_loop_once(ctdb->ev);
/* verify that we have all ip addresses we should have and we dont
* have addresses we shouldnt have.
*/
- if (ctdb->do_checkpublicip) {
+ if (ctdb->tunable.disable_ip_failover == 0) {
if (rec->ip_check_disable_ctx == NULL) {
- if (verify_local_ip_allocation(ctdb, rec, pnn) != 0) {
+ if (verify_local_ip_allocation(ctdb, rec, pnn, nodemap) != 0) {
DEBUG(DEBUG_ERR, (__location__ " Public IPs were inconsistent.\n"));
}
}
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
- ctdb_set_culprit(rec, culprit);
- do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ rec->need_takeover_run = true;
return;
}
ret = ctdb_takeover_run(ctdb, nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses - starting recovery\n"));
- ctdb_set_culprit(rec, ctdb->pnn);
- do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. Try again later\n"));
return;
}
/* register a message port for updating the recovery daemons node assignment for an ip */
ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
+ /* register a message port for forcing a rebalance of a node next
+ reallocation */
+ ctdb_client_set_message_handler(ctdb, CTDB_SRVID_REBALANCE_NODE, recd_node_rebalance_handler, rec);
+
for (;;) {
TALLOC_CTX *mem_ctx = talloc_new(ctdb);
struct timeval start;
struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
if (kill(ctdb->recoverd_pid, 0) != 0) {
- DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer running. Shutting down main daemon\n", (int)ctdb->recoverd_pid));
+ DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer running. Trying to restart recovery daemon.\n", (int)ctdb->recoverd_pid));
- ctdb_stop_recoverd(ctdb);
- ctdb_stop_keepalive(ctdb);
- ctdb_stop_monitoring(ctdb);
- ctdb_release_all_ips(ctdb);
- if (ctdb->methods != NULL) {
- ctdb->methods->shutdown(ctdb);
- }
- ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
+ event_add_timed(ctdb->ev, ctdb, timeval_zero(),
+ ctdb_restart_recd, ctdb);
- exit(10);
+ return;
}
event_add_timed(ctdb->ev, ctdb,
{
int fd[2];
struct signal_event *se;
+ struct tevent_fd *fde;
if (pipe(fd) != 0) {
return -1;
DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
- event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+ fde = event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ,
ctdb_recoverd_parent, &fd[0]);
+ tevent_fd_set_auto_close(fde);
/* set up a handler to pick up sigchld */
se = event_add_signal(ctdb->ev, ctdb,
DEBUG(DEBUG_NOTICE,("Shutting down recovery daemon\n"));
kill(ctdb->recoverd_pid, SIGTERM);
}
+
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+ DEBUG(DEBUG_ERR,("Restarting recovery daemon\n"));
+ ctdb_stop_recoverd(ctdb);
+ ctdb_start_recoverd(ctdb);
+}
git.samba.org / ctdb.git / blobdiff