to make it easier/less disruptive to add nodes to a running cluster

add a new control that causes the node to drop the current nodes list
and reread it from the nodes file.
During this operation, the node will also drop the tcp layer and restart it.

When we drop the tcp layer, by talloc_free()ing the ctcp structure
add a destructor to ctcp so that we also can clean up and remove the references in the ctdb structure to the transport layer

add two new commands for the ctdb tool.
one to list all nodes in the nodesfile and the second a command to trigger a node to drop the transport and reinitialize it with the nde nodes file

diff --git a/client/ctdb_client.c b/client/ctdb_client.c
index 1cdfee56e08918c6f012092b952bd9d8eb4aa277..df328c078e1628188b9bcd90ee3fd25fb0261dfd 100644 (file)
--- a/client/ctdb_client.c
+++ b/client/ctdb_client.c
@@ -1241,6 +1241,27 @@ int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb,
        return 0;
 }
 
+/*
+  drop the transport, reload the nodes file and restart the transport
+ */
+int ctdb_ctrl_reload_nodes_file(struct ctdb_context *ctdb, 
+                   struct timeval timeout, uint32_t destnode)
+{
+       int ret;
+       int32_t res;
+
+       ret = ctdb_control(ctdb, destnode, 0, 
+                          CTDB_CONTROL_RELOAD_NODES_FILE, 0, tdb_null, 
+                          NULL, NULL, &res, &timeout, NULL);
+       if (ret != 0 || res != 0) {
+               DEBUG(DEBUG_ERR,(__location__ " ctdb_control for reloadnodesfile failed\n"));
+               return -1;
+       }
+
+       return 0;
+}
+
+
 /*
   set vnn map on a node
  */

diff --git a/include/ctdb.h b/include/ctdb.h
index d9bf992b4a00b5d2d84bb31e12eab990d321839c..d7f754f77afad328d16279918e58f77b57fb9bac 100644 (file)
--- a/include/ctdb.h
+++ b/include/ctdb.h
@@ -295,6 +295,9 @@ int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb,
                    struct timeval timeout, uint32_t destnode, 
                    TALLOC_CTX *mem_ctx, struct ctdb_node_map **nodemap);
 
+int ctdb_ctrl_reload_nodes_file(struct ctdb_context *ctdb, 
+                   struct timeval timeout, uint32_t destnode);
+
 struct ctdb_key_list {
        uint32_t dbid;
        uint32_t num;

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index ab875924fa16614f0c7acd0a88d6bb18d8ed6896..c34a9d994e11eb96629f55a6fea628daf45f07fb 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -490,6 +490,7 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
                    CTDB_CONTROL_UPTIME                  = 69,
                    CTDB_CONTROL_START_RECOVERY          = 70,
                    CTDB_CONTROL_END_RECOVERY            = 71,
+                   CTDB_CONTROL_RELOAD_NODES_FILE       = 72,
 };     
 
 /*
@@ -1244,5 +1245,9 @@ int ctdb_client_async_control(struct ctdb_context *ctdb,
                                bool dont_log_errors,
                                TDB_DATA data);
 
+void ctdb_load_nodes_file(struct ctdb_context *ctdb);
+
+int ctdb_control_reload_nodes_file(struct ctdb_context *ctdb, uint32_t opcode);
+
 
 #endif

diff --git a/server/ctdb_control.c b/server/ctdb_control.c
index 3ea8f161db4ef3db3a310a954f3910965dabd55f..dc890cfb178b519d3d28827e24a6bfebc36a38cb 100644 (file)
--- a/server/ctdb_control.c
+++ b/server/ctdb_control.c
@@ -134,6 +134,10 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
        case CTDB_CONTROL_GET_NODEMAP:
                return ctdb_control_getnodemap(ctdb, opcode, indata, outdata);
 
+       case CTDB_CONTROL_RELOAD_NODES_FILE:
+               CHECK_CONTROL_DATA_SIZE(0);
+               return ctdb_control_reload_nodes_file(ctdb, opcode);
+
        case CTDB_CONTROL_SETVNNMAP:
                return ctdb_control_setvnnmap(ctdb, opcode, indata, outdata);
 

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index c3a0caf0a7a4a73878271c592c75a7d2c0c98139..96882db94485770ae377713c3b92d6c4a9ed14a5 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -171,6 +171,42 @@ ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA ind
        return 0;
 }
 
+static void
+ctdb_reload_nodes_event(struct event_context *ev, struct timed_event *te, 
+                              struct timeval t, void *private_data)
+{
+       int ret;
+       struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+       int ctdb_tcp_init(struct ctdb_context *);
+
+       /* shut down the transport */
+       ctdb->methods->shutdown(ctdb);
+
+       /* start the transport again */
+       ctdb_load_nodes_file(ctdb);
+       ret = ctdb_tcp_init(ctdb);
+       if (ret != 0) {
+               DEBUG(DEBUG_CRIT, (__location__ " Failed to init TCP\n"));
+               exit(1);
+       }
+       ctdb->methods->initialise(ctdb);
+       ctdb->methods->start(ctdb);
+
+       return;
+}
+
+/*
+  reload the nodes file after a short delay (so that we can send the response
+  back first
+*/
+int 
+ctdb_control_reload_nodes_file(struct ctdb_context *ctdb, uint32_t opcode)
+{
+       event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1,0), ctdb_reload_nodes_event, ctdb);
+
+       return 0;
+}
+
 /* 
    a traverse function for pulling all relevent records from pulldb
  */

diff --git a/server/ctdb_server.c b/server/ctdb_server.c
index d97e3cbc4f4e33767c29c507adc5b5954d879554..47a65c93ab45830bfb1836a1b980f71890baef27 100644 (file)
--- a/server/ctdb_server.c
+++ b/server/ctdb_server.c
@@ -134,6 +134,10 @@ int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
        int nlines;
        int i;
 
+       talloc_free(ctdb->nodes);
+       ctdb->nodes     = NULL;
+       ctdb->num_nodes = 0;
+
        talloc_free(ctdb->node_list_file);
        ctdb->node_list_file = talloc_strdup(ctdb, nlist);
 

diff --git a/server/ctdbd.c b/server/ctdbd.c
index a40c4efb752371fef14e691929f777fe0b4fa512..ca366fc2092edeb99ca3e52b7f40477a59b5e9dd 100644 (file)
--- a/server/ctdbd.c
+++ b/server/ctdbd.c
@@ -73,7 +73,16 @@ static void ctdb_recv_pkt(struct ctdb_context *ctdb, uint8_t *data, uint32_t len
        ctdb_input_pkt(ctdb, hdr);
 }
 
+void ctdb_load_nodes_file(struct ctdb_context *ctdb)
+{
+       int ret;
 
+       ret = ctdb_set_nlist(ctdb, options.nlist);
+       if (ret == -1) {
+               DEBUG(DEBUG_ALERT,("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb)));
+               exit(1);
+       }
+}
 
 static const struct ctdb_upcalls ctdb_upcalls = {
        .recv_pkt       = ctdb_recv_pkt,
@@ -188,11 +197,7 @@ int main(int argc, const char *argv[])
        }
 
        /* tell ctdb what nodes are available */
-       ret = ctdb_set_nlist(ctdb, options.nlist);
-       if (ret == -1) {
-               DEBUG(DEBUG_ALERT,("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb)));
-               exit(1);
-       }
+       ctdb_load_nodes_file(ctdb);
 
        /* if a node-ip was specified, verify that it exists in the
           nodes file

diff --git a/tcp/ctdb_tcp.h b/tcp/ctdb_tcp.h
index 7d47cbcfe149f6e3f48ceb6ef1a153bb871fc3ac..9a17bd6b5961a2fb6a7dbdc9a313df0473d6f98d 100644 (file)
--- a/tcp/ctdb_tcp.h
+++ b/tcp/ctdb_tcp.h
@@ -20,6 +20,7 @@
 
 /* ctdb_tcp main state */
 struct ctdb_tcp {
+       struct ctdb_context *ctdb;
        int listen_fd;
 };
 

diff --git a/tcp/tcp_init.c b/tcp/tcp_init.c
index 624f6507efca16edff743db3c74f908f66263e26..527373cb08b12e9e65a55156ecb07b456519201a 100644 (file)
--- a/tcp/tcp_init.c
+++ b/tcp/tcp_init.c
@@ -54,7 +54,10 @@ static int ctdb_tcp_initialise(struct ctdb_context *ctdb)
        int i;
 
        /* listen on our own address */
-       if (ctdb_tcp_listen(ctdb) != 0) return -1;
+       if (ctdb_tcp_listen(ctdb) != 0) {
+               DEBUG(DEBUG_CRIT, (__location__ " Failed to start listening on the CTDB socket\n"));
+               exit(1);
+       }
 
        for (i=0; i<ctdb->num_nodes; i++) {
                if (ctdb_tcp_add_node(ctdb->nodes[i]) != 0) {
@@ -142,6 +145,18 @@ static const struct ctdb_methods ctdb_tcp_methods = {
        .restart      = ctdb_tcp_restart,
 };
 
+static int tcp_ctcp_destructor(struct ctdb_tcp *ctcp)
+{
+       if (ctcp->listen_fd) {
+               close(ctcp->listen_fd);
+       }
+       ctcp->ctdb->private_data = NULL;
+       ctcp->ctdb->methods = NULL;
+       
+       return 0;
+}
+
+               
 /*
   initialise tcp portion of ctdb 
 */
@@ -152,8 +167,11 @@ int ctdb_tcp_init(struct ctdb_context *ctdb)
        CTDB_NO_MEMORY(ctdb, ctcp);
 
        ctcp->listen_fd = -1;
+       ctcp->ctdb      = ctdb;
        ctdb->private_data = ctcp;
        ctdb->methods = &ctdb_tcp_methods;
+
+       talloc_set_destructor(ctcp, tcp_ctcp_destructor);
        return 0;
 }
 

diff --git a/tools/ctdb.c b/tools/ctdb.c
index a82d625ad8ef4f852f4772872ae8b81ac2072079..72e4ceb0435e5733c409ae6bec9f6ed76ecbdf6a 100644 (file)
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -1130,6 +1130,46 @@ static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **
        return 0;
 }
 
+/*
+  list all nodes in the cluster
+ */
+static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       int i, ret;
+       struct ctdb_node_map *nodemap=NULL;
+
+       ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
+               return ret;
+       }
+
+       for(i=0;i<nodemap->num;i++){
+               printf("%s\n", inet_ntoa(nodemap->nodes[i].sin.sin_addr));
+       }
+
+       return 0;
+}
+
+/*
+  reload the nodes file on the local node
+ */
+static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       int ret;
+
+
+       ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(),
+               CTDB_CURRENT_NODE);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on local node You MUST fix that node manually!\n"));
+       }
+       
+                               
+       return 0;
+}
+
+
 static const struct {
        const char *name;
        int (*fn)(struct ctdb_context *, int, const char **);
@@ -1175,6 +1215,8 @@ static const struct {
        { "getsrvids",       getsrvids,                 false, "get a list of all server ids"},
        { "vacuum",          ctdb_vacuum,               false, "vacuum the databases of empty records", "[max_records]"},
        { "repack",          ctdb_repack,               false, "repack all databases", "[max_freelist]"},
+       { "listnodes",       control_listnodes,         false, "list all nodes in the cluster"},
+       { "reloadnodes",         control_reload_nodes_file,             false, "update the nodes file and restart the transport on the local node"},
 };
 
 /*