When adding ips to nodes, set up a deferred rebalance for the whole node to trigger...

S1035306

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index e3a9a1577ada71504f7893bf23b3daa21371520a..e1237b3f97b09faae29a964e586dc964c8bf8dc5 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -1442,6 +1442,7 @@ bool lcp2_failback(struct ctdb_context *ctdb,
                   struct ctdb_public_ip_list *all_ips,
                   uint32_t *lcp2_imbalances,
                   bool *newly_healthy);
+void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
 void ctdb_takeover_run_core(struct ctdb_context *ctdb,
                            struct ctdb_node_map *nodemap,
                            struct ctdb_public_ip_list **all_ips_p);

diff --git a/include/ctdb_protocol.h b/include/ctdb_protocol.h
index 0422afeb7b73306ef93bf61d145bd86ee30ff822..1a116c149ef89dd3f86063c8c158e69551e652e6 100644 (file)
--- a/include/ctdb_protocol.h
+++ b/include/ctdb_protocol.h
@@ -127,6 +127,11 @@ struct ctdb_call_info {
  */
 #define CTDB_SRVID_TAKEOVER_RUN 0xFB00000000000000LL
 
+/* request recovery daemon to rebalance ips for a node.
+   input is uint32_t for the node id.
+*/
+#define CTDB_SRVID_REBALANCE_NODE 0xFB01000000000000LL
+
 /* A message id to ask the recovery daemon to temporarily disable the
    public ip checks
 */

diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 4963c3f78b7b7f6ae9068d9f874c70f7cae0f434..d3678718f5f6dede4fe511c449156af88113e959 100644 (file)
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -65,6 +65,7 @@ struct ctdb_recoverd {
        struct ip_reallocate_list *reallocate_callers;
        TALLOC_CTX *ip_check_disable_ctx;
        struct ctdb_control_get_ifaces *ifaces;
+       TALLOC_CTX *deferred_rebalance_ctx;
 };
 
 #define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
@@ -1941,6 +1942,53 @@ static void reenable_ip_check(struct event_context *ev, struct timed_event *te,
 }
 
 
+static void ctdb_rebalance_timeout(struct event_context *ev, struct timed_event *te, 
+                                 struct timeval t, void *p)
+{
+       struct ctdb_recoverd *rec = talloc_get_type(p, struct ctdb_recoverd);
+       struct ctdb_context *ctdb = rec->ctdb;
+       int ret;
+
+       DEBUG(DEBUG_NOTICE,("Rebalance all nodes that have had ip assignment changes.\n"));
+
+       ret = ctdb_takeover_run(ctdb, rec->nodemap);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR, (__location__ " Unable to setup public takeover addresses. ctdb_takeover_run() failed.\n"));
+               rec->need_takeover_run = true;
+       }
+
+       talloc_free(rec->deferred_rebalance_ctx);
+       rec->deferred_rebalance_ctx = NULL;
+}
+
+       
+static void recd_node_rebalance_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+                            TDB_DATA data, void *private_data)
+{
+       uint32_t pnn;
+       struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+
+       if (data.dsize != sizeof(uint32_t)) {
+               DEBUG(DEBUG_ERR,(__location__ " Incorrect size of node rebalance message. Was %zd but expected %zd bytes\n", data.dsize, sizeof(uint32_t)));
+               return;
+       }
+
+       pnn = *(uint32_t *)&data.dptr[0];
+
+       lcp2_forcerebalance(ctdb, pnn);
+       DEBUG(DEBUG_NOTICE,("Received message to perform node rebalancing for node %d\n", pnn));
+
+       if (rec->deferred_rebalance_ctx != NULL) {
+               talloc_free(rec->deferred_rebalance_ctx);
+       }
+       rec->deferred_rebalance_ctx = talloc_new(rec);
+       event_add_timed(ctdb->ev, rec->deferred_rebalance_ctx, 
+                       timeval_current_ofs(60, 0),
+                       ctdb_rebalance_timeout, rec);
+}
+
+
+
 static void recd_update_ip_handler(struct ctdb_context *ctdb, uint64_t srvid, 
                             TDB_DATA data, void *private_data)
 {
@@ -3477,6 +3525,10 @@ static void monitor_cluster(struct ctdb_context *ctdb)
        /* register a message port for updating the recovery daemons node assignment for an ip */
        ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECD_UPDATE_IP, recd_update_ip_handler, rec);
 
+       /* register a message port for forcing a rebalance of a node next
+          reallocation */
+       ctdb_client_set_message_handler(ctdb, CTDB_SRVID_REBALANCE_NODE, recd_node_rebalance_handler, rec);
+
        for (;;) {
                TALLOC_CTX *mem_ctx = talloc_new(ctdb);
                struct timeval start;

diff --git a/server/ctdb_takeover.c b/server/ctdb_takeover.c
index 9cbafaf595acd9e878f885cda155ca727802b415..3dc1e46cfe7bc812a1d6b7852c3f712ea3285e96 100644 (file)
--- a/server/ctdb_takeover.c
+++ b/server/ctdb_takeover.c
@@ -1475,6 +1475,32 @@ bool basic_failback(struct ctdb_context *ctdb,
        return false;
 }
 
+struct ctdb_rebalancenodes {
+       struct ctdb_rebalancenodes *next;
+       uint32_t pnn;
+};
+static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
+
+
+/* set this flag to force the node to be rebalanced even if it just didnt
+   become healthy again.
+*/
+void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
+{
+       struct ctdb_rebalancenodes *rebalance;
+
+       for (rebalance = force_rebalance_list; rebalance; rebalance = rebalance->next) {
+               if (rebalance->pnn == pnn) {
+                       return;
+               }
+       }
+
+       rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
+       rebalance->pnn = pnn;
+       rebalance->next = force_rebalance_list;
+       force_rebalance_list = rebalance;
+}
+
 /* Do necessary LCP2 initialisation.  Bury it in a function here so
  * that we can unit test it.
  * Not static, so we can easily link it into a unit test.
@@ -1507,6 +1533,20 @@ void lcp2_init(struct ctdb_context * tmp_ctx,
                        (*newly_healthy)[tmp_ip->pnn] = false;
                }
        }
+
+       /* 3rd step: if a node is forced to re-balance then
+          we allow failback onto the node */
+       while (force_rebalance_list != NULL) {
+               struct ctdb_rebalancenodes *next = force_rebalance_list->next;
+
+               if (force_rebalance_list->pnn <= nodemap->num) {
+                       (*newly_healthy)[force_rebalance_list->pnn] = true;
+               }
+
+               DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of node %d\n", force_rebalance_list->pnn));
+               talloc_free(force_rebalance_list);
+               force_rebalance_list = next;
+       }
 }
 
 /* Allocate any unassigned addresses using the LCP2 algorithm to find

diff --git a/tools/ctdb.c b/tools/ctdb.c
index d49bc8f813346c8362d935db51ae1b647e96a71f..eefda398b1d5f004bd5e512b058503df388f796a 100644 (file)
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -1325,6 +1325,44 @@ static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv
        return 0;
 }
 
+static int rebalance_node(struct ctdb_context *ctdb, uint32_t pnn)
+{
+       uint32_t recmaster;
+       TDB_DATA data;
+
+       if (ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), pnn, &recmaster) != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", pnn));
+               return -1;
+       }
+
+       data.dptr  = (uint8_t *)&pnn;
+       data.dsize = sizeof(uint32_t);
+       if (ctdb_client_send_message(ctdb, recmaster, CTDB_SRVID_REBALANCE_NODE, data) != 0) {
+               DEBUG(DEBUG_ERR,("Failed to send message to force node reallocation\n"));
+               return -1;
+       }
+
+       return 0;
+}
+
+
+/*
+  rebalance a node by setting it to allow failback and triggering a
+  takeover run
+ */
+static int control_rebalancenode(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       switch (options.pnn) {
+       case CTDB_BROADCAST_ALL:
+       case CTDB_CURRENT_NODE:
+               DEBUG(DEBUG_ERR,("You must specify a node number with -n <pnn> for the node to rebalance\n"));
+               return -1;
+       }
+
+       return rebalance_node(ctdb, options.pnn);
+}
+
+
 static int rebalance_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
 {
        struct ctdb_public_ip ip;
@@ -1763,12 +1801,12 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
                return ret;
        }
 
-       if (rebalance_ip(ctdb, &addr) != 0) {
-               DEBUG(DEBUG_ERR,("Error when trying to reassign ip\n"));
+       if (rebalance_node(ctdb, options.pnn) != 0) {
+               DEBUG(DEBUG_ERR,("Error when trying to rebalance node\n"));
                return -1;
        }
 
-       talloc_free(tmp_ctx);
+               talloc_free(tmp_ctx);
        return 0;
 }
 
@@ -5020,6 +5058,7 @@ static const struct {
        { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<tdb-file> <key>" },
        { "writekey",        control_writekey,          true,   false,  "write to a database key", "<tdb-file> <key> <value>" },
        { "checktcpport",    control_chktcpport,        false,  true,  "check if a service is bound to a specific tcp port or not", "<port>" },
+       { "rebalancenode",     control_rebalancenode,   false,  false, "release a node by allowing it to takeover ips", "<pnn>"},
 };
 
 /*