#include "dlinklist.h"
-/* most recent reload all ips request we need to perform during the
- next monitoring loop
-*/
-struct reloadips_all_reply *reload_all_ips_request = NULL;
-
/* List of SRVID requests that need to be processed */
struct srvid_list {
struct srvid_list *next, *prev;
struct ctdb_context *ctdb;
uint32_t recmaster;
uint32_t num_active;
+ uint32_t num_lmasters;
uint32_t num_connected;
uint32_t last_culprit_node;
struct ctdb_node_map *nodemap;
bool takeover_run_in_progress;
TALLOC_CTX *takeover_runs_disable_ctx;
struct ctdb_control_get_ifaces *ifaces;
- TALLOC_CTX *deferred_rebalance_ctx;
+ uint32_t *force_rebalance_nodes;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
return 0;
}
-/*
- reload the nodes file
-*/
-static void reload_nodes_file(struct ctdb_context *ctdb)
-{
- ctdb->nodes = NULL;
- ctdb_load_nodes_file(ctdb);
-}
-
static int ctdb_reload_remote_public_ips(struct ctdb_context *ctdb,
struct ctdb_recoverd *rec,
struct ctdb_node_map *nodemap,
struct srvid_request dtr;
TDB_DATA data;
int i;
+ uint32_t *rebalance_nodes = rec->force_rebalance_nodes;
int ret;
bool ok;
+ DEBUG(DEBUG_NOTICE, ("Takeover run starting\n"));
+
if (rec->takeover_run_in_progress) {
DEBUG(DEBUG_ERR, (__location__
" takeover run already in progress \n"));
}
}
- ret = ctdb_takeover_run(rec->ctdb, nodemap, takeover_fail_callback,
+ ret = ctdb_takeover_run(rec->ctdb, nodemap,
+ rec->force_rebalance_nodes,
+ takeover_fail_callback,
banning_credits_on_fail ? rec : NULL);
/* Reenable takeover runs and IP checks on other nodes */
}
if (ret != 0) {
- DEBUG(DEBUG_ERR, ("IP reallocation failed\n"));
+ DEBUG(DEBUG_ERR, ("ctdb_takeover_run() failed\n"));
ok = false;
goto done;
}
ok = true;
+ /* Takeover run was successful so clear force rebalance targets */
+ if (rebalance_nodes == rec->force_rebalance_nodes) {
+ TALLOC_FREE(rec->force_rebalance_nodes);
+ } else {
+ DEBUG(DEBUG_WARNING,
+ ("Rebalance target nodes changed during takeover run - not clearing\n"));
+ }
done:
rec->need_takeover_run = !ok;
talloc_free(nodes);
rec->takeover_run_in_progress = false;
+
+ DEBUG(DEBUG_NOTICE, ("Takeover run %s\n", ok ? "completed successfully" : "unsuccessful"));
return ok;
}
ret = update_flags_on_all_nodes(ctdb, nodemap, i, nodemap->nodes[i].flags);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to update flags on all nodes for node %d\n", i));
- return -1;
+ if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
+ DEBUG(DEBUG_WARNING, (__location__ "Unable to update flags on inactive node %d\n", i));
+ } else {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to update flags on all nodes for node %d\n", i));
+ return -1;
+ }
}
}
DEBUG(DEBUG_ERR, (__location__ " Reload nodes file from recovery daemon\n"));
- reload_nodes_file(rec->ctdb);
+ ctdb_load_nodes_file(rec->ctdb);
}
-static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *p)
+static void ctdb_rebalance_timeout(struct event_context *ev,
+ struct timed_event *te,
+ struct timeval t, void *p)
{
struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
- DEBUG(DEBUG_NOTICE,
- ("Rebalance all nodes that have had ip assignment changes.\n"));
+ if (rec->force_rebalance_nodes == NULL) {
+ DEBUG(DEBUG_ERR,
+ ("Rebalance timeout occurred - no nodes to rebalance\n"));
+ return;
+ }
+ DEBUG(DEBUG_NOTICE,
+ ("Rebalance timeout occurred - do takeover run\n"));
do_takeover_run(rec, rec->nodemap, false);
-
- talloc_free(rec->deferred_rebalance_ctx);
- rec->deferred_rebalance_ctx = NULL;
}
-static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid,
- TDB_DATA data, void *private_data)
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb,
+ uint64_t srvid,
+ TDB_DATA data, void *private_data)
{
uint32_t pnn;
+ uint32_t *t;
+ int len;
struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ if (rec->recmaster != ctdb_get_pnn(ctdb)) {
+ return;
+ }
+
if (data.dsize != sizeof(uint32_t)) {
DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
return;
pnn = *(uint32_t *)&data.dptr[0];
- lcp2_forcerebalance(ctdb, pnn);
- DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+ DEBUG(DEBUG_NOTICE,("Setting up rebalance of IPs to node %u\n", pnn));
- if (rec->deferred_rebalance_ctx != NULL) {
- talloc_free(rec->deferred_rebalance_ctx);
+ /* Copy any existing list of nodes. There's probably some
+ * sort of realloc variant that will do this but we need to
+ * make sure that freeing the old array also cancels the timer
+ * event for the timeout... not sure if realloc will do that.
+ */
+ len = (rec->force_rebalance_nodes != NULL) ?
+ talloc_array_length(rec->force_rebalance_nodes) :
+ 0;
+
+ /* This allows duplicates to be added but they don't cause
+ * harm. A call to add a duplicate PNN arguably means that
+ * the timeout should be reset, so this is the simplest
+ * solution.
+ */
+ t = talloc_zero_array(rec, uint32_t, len+1);
+ CTDB_NO_MEMORY_VOID(ctdb, t);
+ if (len > 0) {
+ memcpy(t, rec->force_rebalance_nodes, sizeof(uint32_t) * len);
}
- rec->deferred_rebalance_ctx = talloc_new(rec);
- event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx,
+ t[len] = pnn;
+
+ talloc_free(rec->force_rebalance_nodes);
+
+ rec->force_rebalance_nodes = t;
+ event_add_timed(ctdb->ev, rec->force_rebalance_nodes,
timeval_current_ofs(ctdb->tunable.deferred_rebalance_on_node_add, 0),
ctdb_rebalance_timeout, rec);
}
data2, rec);
}
-/*
- handler for reload all ips.
-*/
-static void ip_reloadall_handler(struct ctdb_context *ctdb, uint64_t srvid,
- TDB_DATA data, void *private_data)
-{
- struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
-
- if (data.dsize != sizeof(struct reloadips_all_reply)) {
- DEBUG(DEBUG_ERR, (__location__ " Wrong size of return address.\n"));
- return;
- }
-
- reload_all_ips_request = (struct reloadips_all_reply *)talloc_steal(rec, data.dptr);
-
- DEBUG(DEBUG_NOTICE,("RELOAD_ALL_IPS message received from node:%d srvid:%d\n", reload_all_ips_request->pnn, (int)reload_all_ips_request->srvid));
- return;
-}
-
-static void async_reloadips_callback(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
-{
- uint32_t *status = callback_data;
-
- if (res != 0) {
- DEBUG(DEBUG_ERR,("Reload ips all failed on node %d\n", node_pnn));
- *status = 1;
- }
-}
-
-static int
-reload_all_ips(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, struct ctdb_node_map *nodemap, struct reloadips_all_reply *rips)
-{
- TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
- uint32_t *nodes;
- uint32_t status;
- int i;
-
- DEBUG(DEBUG_ERR,("RELOAD ALL IPS on all active nodes\n"));
- for (i = 0; i< nodemap->num; i++) {
- if (nodemap->nodes[i].flags != 0) {
- DEBUG(DEBUG_ERR, ("Can not reload ips on all nodes. Node %d is not up and healthy\n", i));
- talloc_free(tmp_ctx);
- return -1;
- }
- }
-
- /* send the flags update to all connected nodes */
- nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
- status = 0;
- if (ctdb_client_async_control(ctdb, CTDB_CONTROL_RELOAD_PUBLIC_IPS,
- nodes, 0,
- CONTROL_TIMEOUT(),
- false, tdb_null,
- async_reloadips_callback, NULL,
- &status) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Failed to reloadips on all nodes.\n"));
- talloc_free(tmp_ctx);
- return -1;
- }
-
- if (status != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Failed to reloadips on all nodes.\n"));
- talloc_free(tmp_ctx);
- return -1;
- }
-
- ctdb_client_send_message(ctdb, rips->pnn, rips->srvid, tdb_null);
-
- talloc_free(tmp_ctx);
- return 0;
-}
-
-
/*
handler for ip reallocate, just add it to the list of requests and
handle this later in the monitor_cluster loop so we do not recurse
return;
}
- /* if we are not the recmaster we can safely ignore any ip reallocate requests */
+ /* If we are not the recmaster then do some housekeeping */
if (rec->recmaster != pnn) {
+ /* Ignore any IP reallocate requests - only recmaster
+ * processes them
+ */
TALLOC_FREE(rec->reallocate_requests);
+ /* Clear any nodes that should be force rebalanced in
+ * the next takeover run. If the recovery master role
+ * has moved then we don't want to process these some
+ * time in the future.
+ */
+ TALLOC_FREE(rec->force_rebalance_nodes);
}
/* This is a special case. When recovery daemon is started, recmaster
/* count how many active nodes there are */
rec->num_active = 0;
+ rec->num_lmasters = 0;
rec->num_connected = 0;
for (i=0; i<nodemap->num; i++) {
if (!(nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE)) {
rec->num_active++;
+ if (rec->ctdb->nodes[i]->capabilities & CTDB_CAP_LMASTER) {
+ rec->num_lmasters++;
+ }
}
if (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
rec->num_connected++;
if (ctdb->num_nodes != nodemap->num) {
DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) reloading nodes file\n", ctdb->num_nodes, nodemap->num));
- reload_nodes_file(ctdb);
+ ctdb_load_nodes_file(ctdb);
return;
}
}
- /* is there a pending reload all ips ? */
- if (reload_all_ips_request != NULL) {
- reload_all_ips(ctdb, rec, nodemap, reload_all_ips_request);
- talloc_free(reload_all_ips_request);
- reload_all_ips_request = NULL;
- }
-
/* if there are takeovers requested, perform it and notify the waiters */
- if (rec->reallocate_requests) {
+ if (rec->takeover_runs_disable_ctx == NULL &&
+ rec->reallocate_requests) {
process_ipreallocate_requests(ctdb, rec);
}
}
- /* there better be the same number of lmasters in the vnn map
- as there are active nodes or we will have to do a recovery
+ /* There must be the same number of lmasters in the vnn map as
+ * there are active nodes with the lmaster capability... or
+ * do a recovery.
*/
- if (vnnmap->size != rec->num_active) {
- DEBUG(DEBUG_ERR, (__location__ " The vnnmap count is different from the number of active nodes. %u vs %u\n",
- vnnmap->size, rec->num_active));
+ if (vnnmap->size != rec->num_lmasters) {
+ DEBUG(DEBUG_ERR, (__location__ " The vnnmap count is different from the number of active lmaster nodes: %u vs %u\n",
+ vnnmap->size, rec->num_lmasters));
ctdb_set_culprit(rec, ctdb->pnn);
do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
return;
/* register a message port for performing a takeover run */
ctdb_client_set_message_handler(ctdb, CTDB_SRVID_TAKEOVER_RUN, ip_reallocate_handler, rec);
- /* register a message port for performing a reload all ips */
- ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RELOAD_ALL_IPS, ip_reloadall_handler, rec);
-
/* register a message port for disabling the ip check for a short while */
ctdb_client_set_message_handler(ctdb, CTDB_SRVID_DISABLE_IP_CHECK, disable_ip_check_handler, rec);
git.samba.org / ctdb.git / blobdiff