struct ip_reallocate_list *reallocate_callers;
TALLOC_CTX *ip_check_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
+ TALLOC_CTX *deferred_rebalance_ctx;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
#define MONITOR_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_interval, 0)
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event *te, struct timeval t, void *private_data);
/*
ban a node for a period of time
ZERO_STRUCT(call);
call.call_id = CTDB_NULL_FUNC;
call.flags = CTDB_IMMEDIATE_MIGRATION;
+ call.flags |= CTDB_CALL_FLAG_VACUUM_MIGRATION;
r = v->r;
v->r = (struct ctdb_rec_data *)(r->length + (uint8_t *)r);
hdr = (struct ctdb_ltdb_header *)data.dptr;
if (!params->persistent) {
hdr->dmaster = params->ctdb->pnn;
+ hdr->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
}
/* add the record to the blob ready to send to the nodes */
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
+ rec->need_takeover_run = true;
return -1;
}
rec->need_takeover_run = false;
ret = ctdb_takeover_run(ctdb, nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses\n"));
- return -1;
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. ctdb_takeover_run() failed.\n"));
+ rec->need_takeover_run = true;
}
- DEBUG(DEBUG_NOTICE, (__location__ " Recovery - takeip finished\n"));
/* execute the "recovered" event script on all nodes */
ret = run_recovered_eventscript(ctdb, nodemap, "do_recovery");
}
+static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *p)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
+ struct ctdb_context *ctdb = rec->ctdb;
+ int ret;
+
+ DEBUG(DEBUG_NOTICE,("Rebalance all nodes that have had ip assignment changes.\n"));
+
+ ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. ctdb_takeover_run() failed.\n"));
+ rec->need_takeover_run = true;
+ }
+
+ talloc_free(rec->deferred_rebalance_ctx);
+ rec->deferred_rebalance_ctx = NULL;
+}
+
+
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ uint32_t pnn;
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+
+ if (data.dsize != sizeof(uint32_t)) {
+ DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
+ return;
+ }
+
+ if (ctdb->tunable.deferred_rebalance_on_node_add == 0) {
+ return;
+ }
+
+ pnn = *(uint32_t *)&data.dptr[0];
+
+ lcp2_forcerebalance(ctdb, pnn);
+ DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+
+ if (rec->deferred_rebalance_ctx != NULL) {
+ talloc_free(rec->deferred_rebalance_ctx);
+ }
+ rec->deferred_rebalance_ctx = talloc_new(rec);
+ event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+ timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
+ ctdb_rebalance_timeout, rec);
+}
+
+
+
static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid,
TDB_DATA data, void *private_data)
{
if (ret == 0) {
ret = ctdb_takeover_run(ctdb, rec->nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
- culprit));
+ DEBUG(DEBUG_ERR,("Failed to reallocate addresses: ctdb_takeover_run() failed.\n"));
rec->need_takeover_run = true;
}
}
/* skip the check if we have started but not finished recovery */
if (timeval_compare(&uptime1->last_recovery_finished,
&uptime1->last_recovery_started) != 1) {
- DEBUG(DEBUG_NOTICE, (__location__ " in the middle of recovery or ip reallocation. skipping public ip address check\n"));
+ DEBUG(DEBUG_INFO, (__location__ " in the middle of recovery or ip reallocation. skipping public ip address check\n"));
talloc_free(mem_ctx);
return 0;
return -1;
}
- state->child = fork();
+ state->child = ctdb_fork(ctdb);
if (state->child == (pid_t)-1) {
DEBUG(DEBUG_CRIT,(__location__ " fork() failed in check_reclock child\n"));
close(state->fd[0]);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
- ctdb_set_culprit(rec, culprit);
- do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ rec->need_takeover_run = true;
return;
}
ret = ctdb_takeover_run(ctdb, nodemap);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses - starting recovery\n"));
- ctdb_set_culprit(rec, ctdb->pnn);
- do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. Try again later\n"));
return;
}
/* register a message port for updating the recovery daemons node assignment for an ip */
ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
+ /* register a message port for forcing a rebalance of a node next
+ reallocation */
+ ctdb_client_set_message_handler(ctdb, CTDB_SRVID_REBALANCE_NODE, recd_node_rebalance_handler, rec);
+
for (;;) {
TALLOC_CTX *mem_ctx = talloc_new(ctdb);
struct timeval start;
struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
if (kill(ctdb->recoverd_pid, 0) != 0) {
- DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer running. Shutting down main daemon\n", (int)ctdb->recoverd_pid));
+ DEBUG(DEBUG_ERR,("Recovery daemon (pid:%d) is no longer running. Trying to restart recovery daemon.\n", (int)ctdb->recoverd_pid));
- ctdb_stop_recoverd(ctdb);
- ctdb_stop_keepalive(ctdb);
- ctdb_stop_monitoring(ctdb);
- ctdb_release_all_ips(ctdb);
- if (ctdb->methods != NULL) {
- ctdb->methods->shutdown(ctdb);
- }
- ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
+ event_add_timed(ctdb->ev, ctdb, timeval_zero(),
+ ctdb_restart_recd, ctdb);
- exit(10);
+ return;
}
event_add_timed(ctdb->ev, ctdb,
DEBUG(DEBUG_NOTICE,("Shutting down recovery daemon\n"));
kill(ctdb->recoverd_pid, SIGTERM);
}
+
+static void ctdb_restart_recd(struct event_context *ev, struct timed_event *te,
+ struct timeval t, void *private_data)
+{
+ struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+ DEBUG(DEBUG_ERR,("Restarting recovery daemon\n"));
+ ctdb_stop_recoverd(ctdb);
+ ctdb_start_recoverd(ctdb);
+}
git.samba.org / ctdb.git / blobdiff