Revert "recoverd: Disable takeover runs on other nodes for 5 minutes"

[ctdb.git] / server / ctdb_recoverd.c

diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 2e8ba58960587bc1345b2888e5a1dfe95cfdff89..e5c2887f44f37a78cc7b88b1ba6be357b7d3dd80 100644 (file)
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -129,6 +129,7 @@ struct ctdb_recoverd {
        struct ctdb_context *ctdb;
        uint32_t recmaster;
        uint32_t num_active;
+       uint32_t num_lmasters;
        uint32_t num_connected;
        uint32_t last_culprit_node;
        struct ctdb_node_map *nodemap;
@@ -1480,15 +1481,6 @@ static int recover_database(struct ctdb_recoverd *rec,
        return 0;
 }
 
-/*
-  reload the nodes file 
-*/
-static void reload_nodes_file(struct ctdb_context *ctdb)
-{
-       ctdb->nodes = NULL;
-       ctdb_load_nodes_file(ctdb);
-}
-
 static int ctdb_reload_remote_public_ips(struct ctdb_context *ctdb,
                                         struct ctdb_recoverd *rec,
                                         struct ctdb_node_map *nodemap,
@@ -1699,10 +1691,10 @@ static bool do_takeover_run(struct ctdb_recoverd *rec,
 
        nodes = list_of_connected_nodes(rec->ctdb, nodemap, rec, false);
 
-       /* Disable for 5 minutes.  This can be a tunable later if
+       /* Disable for 60 seconds.  This can be a tunable later if
         * necessary.
         */
-       dtr.data = 300;
+       dtr.data = 60;
        for (i = 0; i < talloc_array_length(nodes); i++) {
                if (ctdb_client_send_message(rec->ctdb, nodes[i],
                                             CTDB_SRVID_DISABLE_TAKEOVER_RUNS,
@@ -1856,8 +1848,12 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
                ret = update_flags_on_all_nodes(ctdb, nodemap, i, nodemap->nodes[i].flags);
                if (ret != 0) {
-                       DEBUG(DEBUG_ERR, (__location__ " Unable to update flags on all nodes for node %d\n", i));
-                       return -1;
+                       if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
+                               DEBUG(DEBUG_WARNING, (__location__ "Unable to update flags on inactive node %d\n", i));
+                       } else {
+                               DEBUG(DEBUG_ERR, (__location__ " Unable to update flags on all nodes for node %d\n", i));
+                               return -1;
+                       }
                }
        }
 
@@ -2359,7 +2355,7 @@ static void reload_nodes_handler(struct ctdb_context *ctdb, uint64_t srvid,
 
        DEBUG(DEBUG_ERR, (__location__ " Reload nodes file from recovery daemon\n"));
 
-       reload_nodes_file(rec->ctdb);
+       ctdb_load_nodes_file(rec->ctdb);
 }
 
 
@@ -3701,10 +3697,14 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
 
        /* count how many active nodes there are */
        rec->num_active    = 0;
+       rec->num_lmasters  = 0;
        rec->num_connected = 0;
        for (i=0; i<nodemap->num; i++) {
                if (!(nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE)) {
                        rec->num_active++;
+                       if (rec->ctdb->nodes[i]->capabilities & CTDB_CAP_LMASTER) {
+                               rec->num_lmasters++;
+                       }
                }
                if (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
                        rec->num_connected++;
@@ -3788,7 +3788,7 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
 
        if (ctdb->num_nodes != nodemap->num) {
                DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) reloading nodes file\n", ctdb->num_nodes, nodemap->num));
-               reload_nodes_file(ctdb);
+               ctdb_load_nodes_file(ctdb);
                return;
        }
 
@@ -3952,12 +3952,13 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec,
        }
 
 
-       /* there better be the same number of lmasters in the vnn map
-          as there are active nodes or we will have to do a recovery
+       /* There must be the same number of lmasters in the vnn map as
+        * there are active nodes with the lmaster capability...  or
+        * do a recovery.
         */
-       if (vnnmap->size != rec->num_active) {
-               DEBUG(DEBUG_ERR, (__location__ " The vnnmap count is different from the number of active nodes. %u vs %u\n", 
-                         vnnmap->size, rec->num_active));
+       if (vnnmap->size != rec->num_lmasters) {
+               DEBUG(DEBUG_ERR, (__location__ " The vnnmap count is different from the number of active lmaster nodes: %u vs %u\n",
+                         vnnmap->size, rec->num_lmasters));
                ctdb_set_culprit(rec, ctdb->pnn);
                do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap);
                return;