#include "system/wait.h"
#include "popt.h"
#include "cmdline.h"
-#include "../include/ctdb.h"
+#include "../include/ctdb_client.h"
#include "../include/ctdb_private.h"
#include "db_wrap.h"
#include "dlinklist.h"
struct vacuum_info *vacuum_info;
TALLOC_CTX *ip_reallocate_ctx;
struct ip_reallocate_list *reallocate_callers;
+ TALLOC_CTX *ip_check_disable_ctx;
+ struct ctdb_control_get_ifaces *ifaces;
};
#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_END_RECOVERY,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, tdb_null,
NULL, NULL,
NULL) != 0) {
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_START_RECOVERY,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, tdb_null,
NULL,
startrecovery_fail_callback,
static void async_getcap_callback(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
{
if ( (outdata.dsize != sizeof(uint32_t)) || (outdata.dptr == NULL) ) {
- DEBUG(DEBUG_ERR, (__location__ " Invalid lenght/pointer for getcap callback : %u %p\n", (unsigned)outdata.dsize, outdata.dptr));
+ DEBUG(DEBUG_ERR, (__location__ " Invalid length/pointer for getcap callback : %u %p\n", (unsigned)outdata.dsize, outdata.dptr));
return;
}
if (node_pnn < ctdb->num_nodes) {
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_CAPABILITIES,
- nodes, CONTROL_TIMEOUT(),
+ nodes, 0,
+ CONTROL_TIMEOUT(),
false, tdb_null,
async_getcap_callback, NULL,
NULL) != 0) {
return 0;
}
+static void set_recmode_fail_callback(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(callback_data, struct ctdb_recoverd);
+
+ DEBUG(DEBUG_ERR,("Failed to freeze node %u during recovery. Set it as ban culprit for %d credits\n", node_pnn, rec->nodemap->num));
+ ctdb_set_culprit_count(rec, node_pnn, rec->nodemap->num);
+}
+
+static void transaction_start_fail_callback(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(callback_data, struct ctdb_recoverd);
+
+ DEBUG(DEBUG_ERR,("Failed to start recovery transaction on node %u. Set it as ban culprit for %d credits\n", node_pnn, rec->nodemap->num));
+ ctdb_set_culprit_count(rec, node_pnn, rec->nodemap->num);
+}
+
/*
change recovery mode on all nodes
*/
-static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, uint32_t rec_mode)
+static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, struct ctdb_node_map *nodemap, uint32_t rec_mode)
{
TDB_DATA data;
uint32_t *nodes;
/* freeze all nodes */
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (rec_mode == CTDB_RECOVERY_ACTIVE) {
- if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
- nodes, CONTROL_TIMEOUT(),
+ int i;
+
+ for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
+ nodes, i,
+ CONTROL_TIMEOUT(),
false, tdb_null,
- NULL, NULL,
- NULL) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
- talloc_free(tmp_ctx);
- return -1;
+ NULL,
+ set_recmode_fail_callback,
+ rec) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
}
}
data.dptr = (unsigned char *)&rec_mode;
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMODE,
- nodes, CONTROL_TIMEOUT(),
+ nodes, 0,
+ CONTROL_TIMEOUT(),
false, data,
NULL, NULL,
NULL) != 0) {
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMASTER,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, data,
NULL, NULL,
NULL) != 0) {
return 0;
}
+/* update all remote nodes to use the same db priority that we have
+ this can fail if the remove node has not yet been upgraded to
+ support this function, so we always return success and never fail
+ a recovery if this call fails.
+*/
+static int update_db_priority_on_remote_nodes(struct ctdb_context *ctdb,
+ struct ctdb_node_map *nodemap,
+ uint32_t pnn, struct ctdb_dbid_map *dbmap, TALLOC_CTX *mem_ctx)
+{
+ int db;
+ uint32_t *nodes;
+
+ nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
+
+ /* step through all local databases */
+ for (db=0; db<dbmap->num;db++) {
+ TDB_DATA data;
+ struct ctdb_db_priority db_prio;
+ int ret;
+
+ db_prio.db_id = dbmap->dbs[db].dbid;
+ ret = ctdb_ctrl_get_db_priority(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, dbmap->dbs[db].dbid, &db_prio.priority);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to read database priority from local node for db 0x%08x\n", dbmap->dbs[db].dbid));
+ continue;
+ }
+
+ DEBUG(DEBUG_INFO,("Update DB priority for db 0x%08x to %u\n", dbmap->dbs[db].dbid, db_prio.priority));
+
+ data.dptr = (uint8_t *)&db_prio;
+ data.dsize = sizeof(db_prio);
+
+ if (ctdb_client_async_control(ctdb,
+ CTDB_CONTROL_SET_DB_PRIORITY,
+ nodes, 0,
+ CONTROL_TIMEOUT(), false, data,
+ NULL, NULL,
+ NULL) != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to set DB priority for 0x%08x\n", db_prio.db_id));
+ }
+ }
+
+ return 0;
+}
/*
ensure all other nodes have attached to any databases that we have
pull the remote database contents from one node into the recdb
*/
static int pull_one_remote_database(struct ctdb_context *ctdb, uint32_t srcnode,
- struct tdb_wrap *recdb, uint32_t dbid)
+ struct tdb_wrap *recdb, uint32_t dbid,
+ bool persistent)
{
int ret;
TDB_DATA outdata;
static int pull_remote_database(struct ctdb_context *ctdb,
struct ctdb_recoverd *rec,
struct ctdb_node_map *nodemap,
- struct tdb_wrap *recdb, uint32_t dbid)
+ struct tdb_wrap *recdb, uint32_t dbid,
+ bool persistent)
{
int j;
if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
continue;
}
- if (pull_one_remote_database(ctdb, nodemap->nodes[j].pnn, recdb, dbid) != 0) {
+ if (pull_one_remote_database(ctdb, nodemap->nodes[j].pnn, recdb, dbid, persistent) != 0) {
DEBUG(DEBUG_ERR,(__location__ " Failed to pull remote database from node %u\n",
nodemap->nodes[j].pnn));
ctdb_set_culprit_count(rec, nodemap->nodes[j].pnn, nodemap->num);
unsigned tdb_flags;
/* open up the temporary recovery database */
- name = talloc_asprintf(mem_ctx, "%s/recdb.tdb", ctdb->db_directory);
+ name = talloc_asprintf(mem_ctx, "%s/recdb.tdb.%u",
+ ctdb->db_directory_state,
+ ctdb->pnn);
if (name == NULL) {
return NULL;
}
unlink(name);
tdb_flags = TDB_NOLOCK;
- if (!ctdb->do_setsched) {
+ if (ctdb->valgrinding) {
tdb_flags |= TDB_NOMMAP;
}
+ tdb_flags |= TDB_DISALLOW_NESTING;
recdb = tdb_wrap_open(mem_ctx, name, ctdb->tunable.database_hash_size,
tdb_flags, O_RDWR|O_CREAT|O_EXCL, 0600);
struct ctdb_marshall_buffer *recdata;
uint32_t len;
bool failed;
+ bool persistent;
};
static int traverse_recdb(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
/* update the dmaster field to point to us */
hdr = (struct ctdb_ltdb_header *)data.dptr;
- hdr->dmaster = params->ctdb->pnn;
+ if (!params->persistent) {
+ hdr->dmaster = params->ctdb->pnn;
+ }
/* add the record to the blob ready to send to the nodes */
rec = ctdb_marshall_record(params->recdata, 0, key, NULL, data);
push the recdb database out to all nodes
*/
static int push_recdb_database(struct ctdb_context *ctdb, uint32_t dbid,
+ bool persistent,
struct tdb_wrap *recdb, struct ctdb_node_map *nodemap)
{
struct recdb_data params;
params.recdata = recdata;
params.len = offsetof(struct ctdb_marshall_buffer, data);
params.failed = false;
+ params.persistent = persistent;
if (tdb_traverse_read(recdb->tdb, traverse_recdb, ¶ms) == -1) {
DEBUG(DEBUG_ERR,(__location__ " Failed to traverse recdb database\n"));
nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, outdata,
NULL, NULL,
NULL) != 0) {
static int recover_database(struct ctdb_recoverd *rec,
TALLOC_CTX *mem_ctx,
uint32_t dbid,
+ bool persistent,
uint32_t pnn,
struct ctdb_node_map *nodemap,
uint32_t transaction_id)
}
/* pull all remote databases onto the recdb */
- ret = pull_remote_database(ctdb, rec, nodemap, recdb, dbid);
+ ret = pull_remote_database(ctdb, rec, nodemap, recdb, dbid, persistent);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to pull remote database 0x%x\n", dbid));
return -1;
nodes = list_of_active_nodes(ctdb, nodemap, recdb, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, data,
NULL, NULL,
NULL) != 0) {
/* push out the correct database. This sets the dmaster and skips
the empty records */
- ret = push_recdb_database(ctdb, dbid, recdb, nodemap);
+ ret = push_recdb_database(ctdb, dbid, persistent, recdb, nodemap);
if (ret != 0) {
talloc_free(recdb);
return -1;
ctdb_load_nodes_file(ctdb);
}
-
+static int ctdb_reload_remote_public_ips(struct ctdb_context *ctdb,
+ struct ctdb_recoverd *rec,
+ struct ctdb_node_map *nodemap,
+ uint32_t *culprit)
+{
+ int j;
+ int ret;
+
+ if (ctdb->num_nodes != nodemap->num) {
+ DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) invalid param\n",
+ ctdb->num_nodes, nodemap->num));
+ if (culprit) {
+ *culprit = ctdb->pnn;
+ }
+ return -1;
+ }
+
+ for (j=0; j<nodemap->num; j++) {
+ /* release any existing data */
+ if (ctdb->nodes[j]->known_public_ips) {
+ talloc_free(ctdb->nodes[j]->known_public_ips);
+ ctdb->nodes[j]->known_public_ips = NULL;
+ }
+ if (ctdb->nodes[j]->available_public_ips) {
+ talloc_free(ctdb->nodes[j]->available_public_ips);
+ ctdb->nodes[j]->available_public_ips = NULL;
+ }
+
+ if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+ continue;
+ }
+
+ /* grab a new shiny list of public ips from the node */
+ ret = ctdb_ctrl_get_public_ips_flags(ctdb,
+ CONTROL_TIMEOUT(),
+ ctdb->nodes[j]->pnn,
+ ctdb->nodes,
+ 0,
+ &ctdb->nodes[j]->known_public_ips);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read known public ips from node : %u\n",
+ ctdb->nodes[j]->pnn));
+ if (culprit) {
+ *culprit = ctdb->nodes[j]->pnn;
+ }
+ return -1;
+ }
+
+ if (rec->ip_check_disable_ctx == NULL) {
+ if (verify_remote_ip_allocation(ctdb, ctdb->nodes[j]->known_public_ips)) {
+ DEBUG(DEBUG_ERR,("Node %d has inconsistent public ip allocation and needs update.\n", ctdb->nodes[j]->pnn));
+ rec->need_takeover_run = true;
+ }
+ }
+
+ /* grab a new shiny list of public ips from the node */
+ ret = ctdb_ctrl_get_public_ips_flags(ctdb,
+ CONTROL_TIMEOUT(),
+ ctdb->nodes[j]->pnn,
+ ctdb->nodes,
+ CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE,
+ &ctdb->nodes[j]->available_public_ips);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read available public ips from node : %u\n",
+ ctdb->nodes[j]->pnn));
+ if (culprit) {
+ *culprit = ctdb->nodes[j]->pnn;
+ }
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/* when we start a recovery, make sure all nodes use the same reclock file
+ setting
+*/
+static int sync_recovery_lock_file_across_cluster(struct ctdb_recoverd *rec)
+{
+ struct ctdb_context *ctdb = rec->ctdb;
+ TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+ TDB_DATA data;
+ uint32_t *nodes;
+
+ if (ctdb->recovery_lock_file == NULL) {
+ data.dptr = NULL;
+ data.dsize = 0;
+ } else {
+ data.dsize = strlen(ctdb->recovery_lock_file) + 1;
+ data.dptr = (uint8_t *)ctdb->recovery_lock_file;
+ }
+
+ nodes = list_of_active_nodes(ctdb, rec->nodemap, tmp_ctx, true);
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECLOCK_FILE,
+ nodes, 0,
+ CONTROL_TIMEOUT(),
+ false, data,
+ NULL, NULL,
+ rec) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Failed to sync reclock file settings\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ talloc_free(tmp_ctx);
+ return 0;
+}
+
+
/*
we are the recmaster, and recovery is needed - start a recovery run
*/
TDB_DATA data;
uint32_t *nodes;
struct timeval start_time;
+ uint32_t culprit = (uint32_t)-1;
DEBUG(DEBUG_NOTICE, (__location__ " Starting do_recovery\n"));
return -1;
}
ctdb_ctrl_report_recd_lock_latency(ctdb, CONTROL_TIMEOUT(), timeval_elapsed(&start_time));
- DEBUG(DEBUG_ERR,("Recovery lock taken successfully by recovery daemon\n"));
+ DEBUG(DEBUG_NOTICE,("Recovery lock taken successfully by recovery daemon\n"));
}
DEBUG(DEBUG_NOTICE, (__location__ " Recovery initiated due to problem with node %u\n", rec->last_culprit_node));
DEBUG(DEBUG_ERR, (__location__ " Unable to create missing remote databases\n"));
return -1;
}
-
DEBUG(DEBUG_NOTICE, (__location__ " Recovery - created remote databases\n"));
+ /* update the database priority for all remote databases */
+ ret = update_db_priority_on_remote_nodes(ctdb, nodemap, pnn, dbmap, mem_ctx);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to set db priority on remote nodes\n"));
+ }
+ DEBUG(DEBUG_NOTICE, (__location__ " Recovery - updated db priority for all databases\n"));
+
+
+ /* update all other nodes to use the same setting for reclock files
+ as the local recovery master.
+ */
+ sync_recovery_lock_file_across_cluster(rec);
/* set recovery mode to active on all nodes */
- ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
+ ret = set_recovery_mode(ctdb, rec, nodemap, CTDB_RECOVERY_ACTIVE);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to set recovery mode to active on cluster\n"));
return -1;
return -1;
}
+ /*
+ update all nodes to have the same flags that we have
+ */
+ for (i=0;i<nodemap->num;i++) {
+ if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
+ continue;
+ }
+
+ ret = update_flags_on_all_nodes(ctdb, nodemap, i, nodemap->nodes[i].flags);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to update flags on all nodes for node %d\n", i));
+ return -1;
+ }
+ }
+
+ DEBUG(DEBUG_NOTICE, (__location__ " Recovery - updated flags\n"));
+
/* pick a new generation number */
generation = new_generation();
nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, data,
- NULL, NULL,
- NULL) != 0) {
+ NULL,
+ transaction_start_fail_callback,
+ rec) != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to start transactions. Recovery failed.\n"));
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_CANCEL,
+ nodes, 0,
+ CONTROL_TIMEOUT(), false, tdb_null,
+ NULL,
+ NULL,
+ NULL) != 0) {
+ DEBUG(DEBUG_ERR,("Failed to cancel recovery transaction\n"));
+ }
return -1;
}
DEBUG(DEBUG_NOTICE,(__location__ " started transactions on all nodes\n"));
for (i=0;i<dbmap->num;i++) {
- if (recover_database(rec, mem_ctx, dbmap->dbs[i].dbid, pnn, nodemap, generation) != 0) {
+ ret = recover_database(rec, mem_ctx,
+ dbmap->dbs[i].dbid,
+ dbmap->dbs[i].persistent,
+ pnn, nodemap, generation);
+ if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Failed to recover database 0x%x\n", dbmap->dbs[i].dbid));
return -1;
}
/* commit all the changes */
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, data,
NULL, NULL,
NULL) != 0) {
DEBUG(DEBUG_NOTICE, (__location__ " Recovery - updated flags\n"));
/* disable recovery mode */
- ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_NORMAL);
+ ret = set_recovery_mode(ctdb, rec, nodemap, CTDB_RECOVERY_NORMAL);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to set recovery mode to normal on cluster\n"));
return -1;
/*
tell nodes to takeover their public IPs
*/
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, nodemap, &culprit);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
+ culprit));
+ return -1;
+ }
rec->need_takeover_run = false;
ret = ctdb_takeover_run(ctdb, nodemap);
if (ret != 0) {
/* send a message to all clients telling them that the cluster
has been reconfigured */
- ctdb_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECONFIGURE, tdb_null);
+ ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECONFIGURE, tdb_null);
DEBUG(DEBUG_NOTICE, (__location__ " Recovery complete\n"));
rec->need_recovery = false;
+ /* we managed to complete a full recovery, make sure to forgive
+ any past sins by the nodes that could now participate in the
+ recovery.
+ */
+ DEBUG(DEBUG_ERR,("Resetting ban count to 0 for all nodes\n"));
+ for (i=0;i<nodemap->num;i++) {
+ struct ctdb_banning_state *ban_state;
+
+ if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
+ continue;
+ }
+
+ ban_state = (struct ctdb_banning_state *)ctdb->nodes[nodemap->nodes[i].pnn]->ban_state;
+ if (ban_state == NULL) {
+ continue;
+ }
+
+ ban_state->count = 0;
+ }
+
+
/* We just finished a recovery successfully.
We now wait for rerecovery_timeout before we allow
another recovery to take place.
/* send an election message to all active nodes */
DEBUG(DEBUG_INFO,(__location__ " Send election request to all active nodes\n"));
- ctdb_send_message(ctdb, CTDB_BROADCAST_ALL, srvid, election_data);
+ ctdb_client_send_message(ctdb, CTDB_BROADCAST_ALL, srvid, election_data);
/* A new node that is already frozen has entered the cluster.
DEBUG(DEBUG_ERR, ("recovery master memory dump\n"));
- ret = ctdb_send_message(ctdb, rd->pnn, rd->srvid, *dump);
+ ret = ctdb_client_send_message(ctdb, rd->pnn, rd->srvid, *dump);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to send rd memdump reply message\n"));
talloc_free(tmp_ctx);
reload_nodes_file(rec->ctdb);
}
+
+static void reenable_ip_check(struct event_context *ev, struct timed_event *te,
+ struct timeval yt, void *p)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
+
+ talloc_free(rec->ip_check_disable_ctx);
+ rec->ip_check_disable_ctx = NULL;
+}
+
+
+static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ struct ctdb_public_ip *ip;
+
+ if (rec->recmaster != rec->ctdb->pnn) {
+ DEBUG(DEBUG_INFO,("Not recmaster, ignore update ip message\n"));
+ return;
+ }
+
+ if (data.dsize != sizeof(struct ctdb_public_ip)) {
+ DEBUG(DEBUG_ERR,(__location__ " Incorrect size of recd update ip message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(struct ctdb_public_ip)));
+ return;
+ }
+
+ ip = (struct ctdb_public_ip *)data.dptr;
+
+ update_ip_assignment_tree(rec->ctdb, ip);
+}
+
+
+static void disable_ip_check_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ uint32_t timeout;
+
+ if (rec->ip_check_disable_ctx != NULL) {
+ talloc_free(rec->ip_check_disable_ctx);
+ rec->ip_check_disable_ctx = NULL;
+ }
+
+ if (data.dsize != sizeof(uint32_t)) {
+ DEBUG(DEBUG_ERR,(__location__ " Wrong size for data :%lu "
+ "expexting %lu\n", (long unsigned)data.dsize,
+ (long unsigned)sizeof(uint32_t)));
+ return;
+ }
+ if (data.dptr == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " No data recaived\n"));
+ return;
+ }
+
+ timeout = *((uint32_t *)data.dptr);
+ DEBUG(DEBUG_NOTICE,("Disabling ip check for %u seconds\n", timeout));
+
+ rec->ip_check_disable_ctx = talloc_new(rec);
+ CTDB_NO_MEMORY_VOID(ctdb, rec->ip_check_disable_ctx);
+
+ event_add_timed(ctdb->ev, rec->ip_check_disable_ctx, timeval_current_ofs(timeout, 0), reenable_ip_check, rec);
+}
+
+
/*
handler for ip reallocate, just add it to the list of callers and
handle this later in the monitor_cluster loop so we do not recurse
if (rec->ip_reallocate_ctx == NULL) {
rec->ip_reallocate_ctx = talloc_new(rec);
- CTDB_NO_MEMORY_FATAL(ctdb, caller);
+ CTDB_NO_MEMORY_FATAL(ctdb, rec->ip_reallocate_ctx);
}
caller = talloc(rec->ip_reallocate_ctx, struct ip_reallocate_list);
TDB_DATA result;
int32_t ret;
struct ip_reallocate_list *callers;
+ uint32_t culprit;
DEBUG(DEBUG_INFO, ("recovery master forced ip reallocation\n"));
- ret = ctdb_takeover_run(ctdb, rec->nodemap);
+
+ /* update the list of public ips that a node can handle for
+ all connected nodes
+ */
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, rec->nodemap, &culprit);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
+ culprit));
+ rec->need_takeover_run = true;
+ }
+ if (ret == 0) {
+ ret = ctdb_takeover_run(ctdb, rec->nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
+ culprit));
+ rec->need_takeover_run = true;
+ }
+ }
+
result.dsize = sizeof(int32_t);
result.dptr = (uint8_t *)&ret;
for (callers=rec->reallocate_callers; callers; callers=callers->next) {
- DEBUG(DEBUG_INFO,("Sending ip reallocate reply message to %u:%lu\n", callers->rd->pnn, callers->rd->srvid));
- ret = ctdb_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
+
+ /* Someone that sent srvid==0 does not want a reply */
+ if (callers->rd->srvid == 0) {
+ continue;
+ }
+ DEBUG(DEBUG_INFO,("Sending ip reallocate reply message to "
+ "%u:%llu\n", (unsigned)callers->rd->pnn,
+ (unsigned long long)callers->rd->srvid));
+ ret = ctdb_client_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
if (ret != 0) {
- DEBUG(DEBUG_ERR,("Failed to send ip reallocate reply message to %u:%lu\n", callers->rd->pnn, callers->rd->srvid));
+ DEBUG(DEBUG_ERR,("Failed to send ip reallocate reply "
+ "message to %u:%llu\n",
+ (unsigned)callers->rd->pnn,
+ (unsigned long long)callers->rd->srvid));
}
}
DEBUG(DEBUG_INFO,(__location__ " Force an election\n"));
/* set all nodes to recovery mode to stop all internode traffic */
- ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
+ ret = set_recovery_mode(ctdb, rec, nodemap, CTDB_RECOVERY_ACTIVE);
if (ret != 0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to set recovery mode to active on cluster\n"));
return;
uint32_t changed_flags;
int i;
struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ int disabled_flag_changed;
if (data.dsize != sizeof(*c)) {
DEBUG(DEBUG_ERR,(__location__ "Invalid data in ctdb_node_flag_change\n"));
DEBUG(DEBUG_NOTICE,("Node %u has changed flags - now 0x%x was 0x%x\n", c->pnn, c->new_flags, c->old_flags));
}
+ disabled_flag_changed = (nodemap->nodes[i].flags ^ c->new_flags) & NODE_FLAGS_DISABLED;
+
nodemap->nodes[i].flags = c->new_flags;
ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, CONTROL_TIMEOUT(),
lead to an ip address failover but that is handled
during recovery
*/
- if (changed_flags & NODE_FLAGS_DISABLED) {
+ if (disabled_flag_changed) {
rec->need_takeover_run = true;
}
}
{
int ret;
struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)data.dptr;
+ struct ctdb_node_map *nodemap=NULL;
+ TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+ uint32_t recmaster;
+ uint32_t *nodes;
- ret = ctdb_ctrl_modflags(ctdb, CONTROL_TIMEOUT(), c->pnn, c->new_flags, ~c->new_flags);
+ /* find the recovery master */
+ ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
if (ret != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Unable to update nodeflags on remote nodes\n"));
+ DEBUG(DEBUG_ERR, (__location__ " Unable to get recmaster from local node\n"));
+ talloc_free(tmp_ctx);
+ return;
+ }
+
+ /* read the node flags from the recmaster */
+ ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), recmaster, tmp_ctx, &nodemap);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Unable to get nodemap from node %u\n", c->pnn));
+ talloc_free(tmp_ctx);
+ return;
+ }
+ if (c->pnn >= nodemap->num) {
+ DEBUG(DEBUG_ERR,(__location__ " Nodemap from recmaster does not contain node %d\n", c->pnn));
+ talloc_free(tmp_ctx);
+ return;
}
+
+ /* send the flags update to all connected nodes */
+ nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
+
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_MODIFY_FLAGS,
+ nodes, 0, CONTROL_TIMEOUT(),
+ false, data,
+ NULL, NULL,
+ NULL) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " ctdb_control to modify node flags failed\n"));
+
+ talloc_free(tmp_ctx);
+ return;
+ }
+
+ talloc_free(tmp_ctx);
}
}
-/* called to check that the allocation of public ip addresses is ok.
+/* called to check that the local allocation of public ip addresses is ok.
*/
-static int verify_ip_allocation(struct ctdb_context *ctdb, uint32_t pnn)
+static int verify_local_ip_allocation(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, uint32_t pnn)
{
TALLOC_CTX *mem_ctx = talloc_new(NULL);
+ struct ctdb_control_get_ifaces *ifaces = NULL;
struct ctdb_all_public_ips *ips = NULL;
struct ctdb_uptime *uptime1 = NULL;
struct ctdb_uptime *uptime2 = NULL;
int ret, j;
+ bool need_iface_check = false;
+ bool need_takeover_run = false;
ret = ctdb_ctrl_uptime(ctdb, mem_ctx, CONTROL_TIMEOUT(),
CTDB_CURRENT_NODE, &uptime1);
return -1;
}
+
+ /* read the interfaces from the local node */
+ ret = ctdb_ctrl_get_ifaces(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, mem_ctx, &ifaces);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, ("Unable to get interfaces from local node %u\n", pnn));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ if (!rec->ifaces) {
+ need_iface_check = true;
+ } else if (rec->ifaces->num != ifaces->num) {
+ need_iface_check = true;
+ } else if (memcmp(rec->ifaces, ifaces, talloc_get_size(ifaces)) != 0) {
+ need_iface_check = true;
+ }
+
+ if (need_iface_check) {
+ DEBUG(DEBUG_NOTICE, ("The interfaces status has changed on "
+ "local node %u - force takeover run\n",
+ pnn));
+ need_takeover_run = true;
+ }
+
/* read the ip allocation from the local node */
ret = ctdb_ctrl_get_public_ips(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, mem_ctx, &ips);
if (ret != 0) {
/* skip the check if we have started but not finished recovery */
if (timeval_compare(&uptime1->last_recovery_finished,
&uptime1->last_recovery_started) != 1) {
- DEBUG(DEBUG_NOTICE, (__location__ " in the middle of recovery. skipping public ip address check\n"));
+ DEBUG(DEBUG_NOTICE, (__location__ " in the middle of recovery or ip reallocation. skipping public ip address check\n"));
talloc_free(mem_ctx);
return 0;
}
+ talloc_free(rec->ifaces);
+ rec->ifaces = talloc_steal(rec, ifaces);
+
/* verify that we have the ip addresses we should have
and we dont have ones we shouldnt have.
if we find an inconsistency we set recmode to
if (!ctdb_sys_have_ip(&ips->ips[j].addr)) {
DEBUG(DEBUG_CRIT,("Public address '%s' is missing and we should serve this ip\n",
ctdb_addr_to_str(&ips->ips[j].addr)));
- ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
-
- talloc_free(mem_ctx);
- return -1;
- }
- ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
-
- talloc_free(mem_ctx);
- return -1;
- }
+ need_takeover_run = true;
}
} else {
if (ctdb_sys_have_ip(&ips->ips[j].addr)) {
DEBUG(DEBUG_CRIT,("We are still serving a public address '%s' that we should not be serving.\n",
ctdb_addr_to_str(&ips->ips[j].addr)));
+ need_takeover_run = true;
+ }
+ }
+ }
- ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
+ if (need_takeover_run) {
+ struct takeover_run_reply rd;
+ TDB_DATA data;
- talloc_free(mem_ctx);
- return -1;
- }
- ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
+ DEBUG(DEBUG_CRIT,("Trigger takeoverrun\n"));
- talloc_free(mem_ctx);
- return -1;
- }
- }
+ rd.pnn = ctdb->pnn;
+ rd.srvid = 0;
+ data.dptr = (uint8_t *)&rd;
+ data.dsize = sizeof(rd);
+
+ ret = ctdb_client_send_message(ctdb, rec->recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to send ipreallocate to recmaster :%d\n", (int)rec->recmaster));
}
}
-
talloc_free(mem_ctx);
return 0;
}
nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_NODEMAP,
- nodes,
+ nodes, 0,
CONTROL_TIMEOUT(), false, tdb_null,
async_getnodemap_callback,
NULL,
}
close(state->fd[1]);
state->fd[1] = -1;
+ set_close_on_exec(state->fd[0]);
+
+ DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d for check_recovery_lock\n", state->fd[0]));
talloc_set_destructor(state, check_reclock_destructor);
/* register a message port for performing a takeover run */
ctdb_set_message_handler(ctdb, CTDB_SRVID_TAKEOVER_RUN, ip_reallocate_handler, rec);
+ /* register a message port for disabling the ip check for a short while */
+ ctdb_set_message_handler(ctdb, CTDB_SRVID_DISABLE_IP_CHECK, disable_ip_check_handler, rec);
+
+ /* register a message port for updating the recovery daemons node assignment for an ip */
+ ctdb_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
+
again:
if (mem_ctx) {
talloc_free(mem_ctx);
if (ctdb->recovery_mode == CTDB_RECOVERY_NORMAL) {
DEBUG(DEBUG_ERR,("Node is stopped but recovery mode is not active. Activate recovery mode and lock databases\n"));
- ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
+ ret = ctdb_ctrl_freeze_priority(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, 1);
if (ret != 0) {
DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to node being STOPPED\n"));
goto again;
* have addresses we shouldnt have.
*/
if (ctdb->do_checkpublicip) {
- if (verify_ip_allocation(ctdb, pnn) != 0) {
- DEBUG(DEBUG_ERR, (__location__ " Public IPs were inconsistent.\n"));
- goto again;
+ if (rec->ip_check_disable_ctx == NULL) {
+ if (verify_local_ip_allocation(ctdb, rec, pnn) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Public IPs were inconsistent.\n"));
+ }
}
}
goto again;
}
- /* update the list of public ips that a node can handle for
- all connected nodes
- */
if (ctdb->num_nodes != nodemap->num) {
DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) reloading nodes file\n", ctdb->num_nodes, nodemap->num));
reload_nodes_file(ctdb);
goto again;
}
- for (j=0; j<nodemap->num; j++) {
- /* release any existing data */
- if (ctdb->nodes[j]->public_ips) {
- talloc_free(ctdb->nodes[j]->public_ips);
- ctdb->nodes[j]->public_ips = NULL;
- }
-
- if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
- continue;
- }
-
- /* grab a new shiny list of public ips from the node */
- if (ctdb_ctrl_get_public_ips(ctdb, CONTROL_TIMEOUT(),
- ctdb->nodes[j]->pnn,
- ctdb->nodes,
- &ctdb->nodes[j]->public_ips)) {
- DEBUG(DEBUG_ERR,("Failed to read public ips from node : %u\n",
- ctdb->nodes[j]->pnn));
- goto again;
- }
- }
-
/* verify that all active nodes agree that we are the recmaster */
switch (verify_recmaster(rec, nodemap, pnn)) {
*/
switch (verify_recmode(ctdb, nodemap)) {
case MONITOR_RECOVERY_NEEDED:
- ctdb_set_culprit(rec, ctdb->pnn);
do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
goto again;
case MONITOR_FAILED:
/* we might need to change who has what IP assigned */
if (rec->need_takeover_run) {
+ uint32_t culprit = (uint32_t)-1;
+
rec->need_takeover_run = false;
+ /* update the list of public ips that a node can handle for
+ all connected nodes
+ */
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, nodemap, &culprit);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
+ culprit));
+ ctdb_set_culprit(rec, culprit);
+ do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ goto again;
+ }
+
/* execute the "startrecovery" event script on all nodes */
ret = run_startrecovery_eventscript(rec, nodemap);
if (ret!=0) {
DEBUG(DEBUG_ERR, (__location__ " Unable to run the 'startrecovery' event on cluster\n"));
ctdb_set_culprit(rec, ctdb->pnn);
do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ goto again;
}
ret = ctdb_takeover_run(ctdb, nodemap);
DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses - starting recovery\n"));
ctdb_set_culprit(rec, ctdb->pnn);
do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
+ goto again;
}
/* execute the "recovered" event script on all nodes */
if (ctdb->methods != NULL) {
ctdb->methods->shutdown(ctdb);
}
- ctdb_event_script(ctdb, "shutdown");
+ ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
exit(10);
}
exit(1);
}
+ DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
+
event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
ctdb_recoverd_parent, &fd[0]);
git.samba.org / rusty / ctdb.git / blobdiff