static void async_getcap_callback(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data)
{
if ( (outdata.dsize != sizeof(uint32_t)) || (outdata.dptr == NULL) ) {
- DEBUG(DEBUG_ERR, (__location__ " Invalid lenght/pointer for getcap callback : %u %p\n", (unsigned)outdata.dsize, outdata.dptr));
+ DEBUG(DEBUG_ERR, (__location__ " Invalid length/pointer for getcap callback : %u %p\n", (unsigned)outdata.dsize, outdata.dptr));
return;
}
if (node_pnn < ctdb->num_nodes) {
}
static int ctdb_reload_remote_public_ips(struct ctdb_context *ctdb,
+ struct ctdb_recoverd *rec,
struct ctdb_node_map *nodemap,
uint32_t *culprit)
{
return -1;
}
+ if (rec->ip_check_disable_ctx == NULL) {
+ if (verify_remote_ip_allocation(ctdb, ctdb->nodes[j]->known_public_ips)) {
+ DEBUG(DEBUG_ERR,("Node %d has inconsistent public ip allocation and needs update.\n", ctdb->nodes[j]->pnn));
+ rec->need_takeover_run = true;
+ }
+ }
+
/* grab a new shiny list of public ips from the node */
ret = ctdb_ctrl_get_public_ips_flags(ctdb,
CONTROL_TIMEOUT(),
return 0;
}
+/* when we start a recovery, make sure all nodes use the same reclock file
+ setting
+*/
+static int sync_recovery_lock_file_across_cluster(struct ctdb_recoverd *rec)
+{
+ struct ctdb_context *ctdb = rec->ctdb;
+ TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+ TDB_DATA data;
+ uint32_t *nodes;
+
+ if (ctdb->recovery_lock_file == NULL) {
+ data.dptr = NULL;
+ data.dsize = 0;
+ } else {
+ data.dsize = strlen(ctdb->recovery_lock_file) + 1;
+ data.dptr = (uint8_t *)ctdb->recovery_lock_file;
+ }
+
+ nodes = list_of_active_nodes(ctdb, rec->nodemap, tmp_ctx, true);
+ if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECLOCK_FILE,
+ nodes, 0,
+ CONTROL_TIMEOUT(),
+ false, data,
+ NULL, NULL,
+ rec) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Failed to sync reclock file settings\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ talloc_free(tmp_ctx);
+ return 0;
+}
+
+
/*
we are the recmaster, and recovery is needed - start a recovery run
*/
return -1;
}
ctdb_ctrl_report_recd_lock_latency(ctdb, CONTROL_TIMEOUT(), timeval_elapsed(&start_time));
- DEBUG(DEBUG_ERR,("Recovery lock taken successfully by recovery daemon\n"));
+ DEBUG(DEBUG_NOTICE,("Recovery lock taken successfully by recovery daemon\n"));
}
DEBUG(DEBUG_NOTICE, (__location__ " Recovery initiated due to problem with node %u\n", rec->last_culprit_node));
DEBUG(DEBUG_NOTICE, (__location__ " Recovery - updated db priority for all databases\n"));
+ /* update all other nodes to use the same setting for reclock files
+ as the local recovery master.
+ */
+ sync_recovery_lock_file_across_cluster(rec);
+
/* set recovery mode to active on all nodes */
ret = set_recovery_mode(ctdb, rec, nodemap, CTDB_RECOVERY_ACTIVE);
if (ret != 0) {
/*
tell nodes to takeover their public IPs
*/
- ret = ctdb_reload_remote_public_ips(ctdb, nodemap, &culprit);
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, nodemap, &culprit);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
rec->ip_check_disable_ctx = NULL;
}
+
+static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+ struct ctdb_public_ip *ip;
+
+ if (rec->recmaster != rec->ctdb->pnn) {
+ DEBUG(DEBUG_INFO,("Not recmaster, ignore update ip message\n"));
+ return;
+ }
+
+ if (data.dsize != sizeof(struct ctdb_public_ip)) {
+ DEBUG(DEBUG_ERR,(__location__ " Incorrect size of recd update ip message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(struct ctdb_public_ip)));
+ return;
+ }
+
+ ip = (struct ctdb_public_ip *)data.dptr;
+
+ update_ip_assignment_tree(rec->ctdb, ip);
+}
+
+
static void disable_ip_check_handler(struct ctdb_context *ctdb, uint64_t srvid,
TDB_DATA data, void *private_data)
{
/* update the list of public ips that a node can handle for
all connected nodes
*/
- ret = ctdb_reload_remote_public_ips(ctdb, rec->nodemap, &culprit);
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, rec->nodemap, &culprit);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
}
-/* called to check that the allocation of public ip addresses is ok.
+/* called to check that the local allocation of public ip addresses is ok.
*/
-static int verify_ip_allocation(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, uint32_t pnn)
+static int verify_local_ip_allocation(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, uint32_t pnn)
{
TALLOC_CTX *mem_ctx = talloc_new(NULL);
struct ctdb_control_get_ifaces *ifaces = NULL;
/* register a message port for disabling the ip check for a short while */
ctdb_set_message_handler(ctdb, CTDB_SRVID_DISABLE_IP_CHECK, disable_ip_check_handler, rec);
+ /* register a message port for updating the recovery daemons node assignment for an ip */
+ ctdb_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
+
again:
if (mem_ctx) {
talloc_free(mem_ctx);
*/
if (ctdb->do_checkpublicip) {
if (rec->ip_check_disable_ctx == NULL) {
- if (verify_ip_allocation(ctdb, rec, pnn) != 0) {
+ if (verify_local_ip_allocation(ctdb, rec, pnn) != 0) {
DEBUG(DEBUG_ERR, (__location__ " Public IPs were inconsistent.\n"));
}
}
/* update the list of public ips that a node can handle for
all connected nodes
*/
- ret = ctdb_reload_remote_public_ips(ctdb, nodemap, &culprit);
+ ret = ctdb_reload_remote_public_ips(ctdb, rec, nodemap, &culprit);
if (ret != 0) {
DEBUG(DEBUG_ERR,("Failed to read public ips from remote node %d\n",
culprit));
exit(1);
}
- DEBUG(DEBUG_NOTICE, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
+ DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to recovery daemon\n", fd[0]));
event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
ctdb_recoverd_parent, &fd[0]);
git.samba.org / rusty / ctdb.git / blobdiff