add a new node state : DELETED.

This is used to mark nodes as being DELETED internally in ctdb
so that nodes are not renumbered if / when they are removed from the nodes file.

This is used to be able to do "ctdb reloadnodes" at runtime without
causing nodes to be renumbered.
To do this, instead of deleting a node from the nodes file, just comment it out like

   1.0.0.1
   #1.0.0.2
   1.0.0.3

After removing 1.0.0.2 from the cluster,  the remaining nodes retain their
pnn's from prior to the deletion, namely 0 and 2

Any line in the nodes file that is commented out represents a DELETED pnn

diff --git a/include/ctdb.h b/include/ctdb.h
index 866ba76e2a8edd4a450bdd11ce712308bbf40787..ea4bcae109e4df071a29262aae1f434653c60763 100644 (file)
--- a/include/ctdb.h
+++ b/include/ctdb.h
@@ -101,6 +101,11 @@ struct ctdb_call_info {
  */
 #define CTDB_SRVID_PUSH_NODE_FLAGS 0xF900000000000000LL
 
+/* 
+   a message ID to get the recovery daemon to reload the nodes file
+ */
+#define CTDB_SRVID_RELOAD_NODES 0xFA00000000000000LL
+
 
 
 /* used on the domain socket, send a pdu to the local daemon */

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index ff007a8ac6121803fc2e236dbc4eb96bada0a010..98dab0785dbab185b2b74f24d0d0e17f07e5918f 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -198,7 +198,8 @@ struct ctdb_node {
 #define NODE_FLAGS_PERMANENTLY_DISABLED        0x00000004 /* administrator has disabled node */
 #define NODE_FLAGS_BANNED              0x00000008 /* recovery daemon has banned the node */
 #define NODE_FLAGS_DISABLED            (NODE_FLAGS_UNHEALTHY|NODE_FLAGS_PERMANENTLY_DISABLED)
-#define NODE_FLAGS_INACTIVE            (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_BANNED)
+#define NODE_FLAGS_DELETED             0x00000010 /* this node has been deleted */
+#define NODE_FLAGS_INACTIVE            (NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED|NODE_FLAGS_BANNED)
        uint32_t flags;
 
        /* used by the dead node monitoring */

diff --git a/server/ctdb_keepalive.c b/server/ctdb_keepalive.c
index 524feb1696617ff6aba77ddcec1bb6b1de3ec169..dfe7cfc622147e069feec96df782b57c476452fa 100644 (file)
--- a/server/ctdb_keepalive.c
+++ b/server/ctdb_keepalive.c
@@ -37,6 +37,11 @@ static void ctdb_check_for_dead_nodes(struct event_context *ev, struct timed_eve
        /* send a keepalive to all other nodes, unless */
        for (i=0;i<ctdb->num_nodes;i++) {
                struct ctdb_node *node = ctdb->nodes[i];
+
+               if (node->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
+
                if (node->pnn == ctdb->pnn) {
                        continue;
                }

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index b4428fa4946e9edaf6974a3081fe5ef973742fdb..7953c6b44735c7a457fd2f6f171332a379d92722 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -242,6 +242,10 @@ ctdb_reload_nodes_event(struct event_context *ev, struct timed_event *te,
                        continue;
                }
 
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
+
                /* any new or different nodes must be added */
                if (ctdb->methods->add_node(ctdb->nodes[i]) != 0) {
                        DEBUG(DEBUG_CRIT, (__location__ " methods->add_node failed at %d\n", i));
@@ -253,6 +257,9 @@ ctdb_reload_nodes_event(struct event_context *ev, struct timed_event *te,
                }
        }
 
+       /* tell the recovery daemon to reaload the nodes file too */
+       ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELOAD_NODES, tdb_null);
+
        talloc_free(tmp_ctx);
        return;
 }

diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 995284fbb22bdb753ad5e6dba80350be75041937..6b2fb5e5556b05f53e5de2aff1593716f66c78b9 100644 (file)
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -1332,12 +1332,6 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
        DEBUG(DEBUG_NOTICE, (__location__ " Starting do_recovery\n"));
 
-       if (ctdb->num_nodes != nodemap->num) {
-               DEBUG(DEBUG_ERR, (__location__ " ctdb->num_nodes (%d) != nodemap->num (%d) reloading nodes file\n", ctdb->num_nodes, nodemap->num));
-               reload_nodes_file(ctdb);
-               return -1;
-       }
-
        /* if recovery fails, force it again */
        rec->need_recovery = true;
 
@@ -1803,6 +1797,21 @@ DEBUG(DEBUG_ERR, ("recovery master memory dump\n"));
        talloc_free(tmp_ctx);
 }
 
+/*
+  handler for reload_nodes
+*/
+static void reload_nodes_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+                            TDB_DATA data, void *private_data)
+{
+       struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+
+       DEBUG(DEBUG_ERR, (__location__ " Reload nodes file from recovery daemon\n"));
+
+       reload_nodes_file(rec->ctdb);
+}
+
+
+
 /*
   handler for recovery master elections
 */
@@ -2371,6 +2380,9 @@ static void monitor_cluster(struct ctdb_context *ctdb)
        /* register a message port for vacuum fetch */
        ctdb_set_message_handler(ctdb, CTDB_SRVID_VACUUM_FETCH, vacuum_fetch_handler, rec);
 
+       /* register a message port for reloadnodes  */
+       ctdb_set_message_handler(ctdb, CTDB_SRVID_RELOAD_NODES, reload_nodes_handler, rec);
+
 again:
        if (mem_ctx) {
                talloc_free(mem_ctx);
@@ -2591,14 +2603,16 @@ again:
                goto again;
        }
        for (j=0; j<nodemap->num; j++) {
-               if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
-                       continue;
-               }
                /* release any existing data */
                if (ctdb->nodes[j]->public_ips) {
                        talloc_free(ctdb->nodes[j]->public_ips);
                        ctdb->nodes[j]->public_ips = NULL;
                }
+
+               if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+                       continue;
+               }
+
                /* grab a new shiny list of public ips from the node */
                if (ctdb_ctrl_get_public_ips(ctdb, CONTROL_TIMEOUT(),
                        ctdb->nodes[j]->pnn, 

diff --git a/server/ctdb_server.c b/server/ctdb_server.c
index bd1d7ed6215cd6e37e99994bdd3b9fb72236be67..1eaeae2367d05c92494a9a9e5f5bd824f2ca30e0 100644 (file)
--- a/server/ctdb_server.c
+++ b/server/ctdb_server.c
@@ -45,6 +45,9 @@ int ctdb_ip_to_nodeid(struct ctdb_context *ctdb, const char *nodeip)
        int nodeid;
 
        for (nodeid=0;nodeid<ctdb->num_nodes;nodeid++) {
+               if (ctdb->nodes[nodeid]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
                if (!strcmp(ctdb->nodes[nodeid]->address.address, nodeip)) {
                        return nodeid;
                }
@@ -89,7 +92,7 @@ int ctdb_set_tdb_dir_persistent(struct ctdb_context *ctdb, const char *dir)
 }
 
 /*
-  add a node to the list of active nodes
+  add a node to the list of nodes
 */
 static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
 {
@@ -136,6 +139,46 @@ static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
        return 0;
 }
 
+/*
+  add an entry for a "deleted" node to the list of nodes.
+  a "deleted" node is a node that is commented out from the nodes file.
+  this is used to prevent that subsequent nodes in the nodes list
+  change their pnn value if a node is "delete" by commenting it out and then
+  using "ctdb reloadnodes" at runtime.
+*/
+static int ctdb_add_deleted_node(struct ctdb_context *ctdb)
+{
+       struct ctdb_node *node, **nodep;
+
+       nodep = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes+1);
+       CTDB_NO_MEMORY(ctdb, nodep);
+
+       ctdb->nodes = nodep;
+       nodep = &ctdb->nodes[ctdb->num_nodes];
+       (*nodep) = talloc_zero(ctdb->nodes, struct ctdb_node);
+       CTDB_NO_MEMORY(ctdb, *nodep);
+       node = *nodep;
+       
+       if (ctdb_parse_address(ctdb, node, "0.0.0.0", &node->address) != 0) {
+               DEBUG(DEBUG_ERR,("Failed to setup deleted node %d\n", ctdb->num_nodes));
+               return -1;
+       }
+       node->ctdb = ctdb;
+       node->name = talloc_strdup(node, "0.0.0.0:0");
+
+       /* this assumes that the nodes are kept in sorted order, and no gaps */
+       node->pnn = ctdb->num_nodes;
+
+       /* this node is permanently deleted/disconnected */
+       node->flags = NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED;
+
+       ctdb->num_nodes++;
+       node->dead_count = 0;
+
+       return 0;
+}
+
+
 /*
   setup the node list from a file
 */
@@ -167,6 +210,10 @@ int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
                        node++;
                }
                if (*node == '#') {
+                       if (ctdb_add_deleted_node(ctdb) != 0) {
+                               talloc_free(lines);
+                               return -1;
+                       }
                        continue;
                }
                if (strcmp(node, "") == 0) {
@@ -188,7 +235,11 @@ int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
        CTDB_NO_MEMORY(ctdb, ctdb->vnn_map->map);
 
        for(i=0;i<ctdb->vnn_map->size;i++) {
-               ctdb->vnn_map->map[i] = i;
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       ctdb->vnn_map->map[i] = -1;
+               } else {
+                       ctdb->vnn_map->map[i] = i;
+               }
        }
        
        talloc_free(lines);
@@ -437,7 +488,10 @@ static void ctdb_broadcast_packet_all(struct ctdb_context *ctdb,
                                      struct ctdb_req_header *hdr)
 {
        int i;
-       for (i=0;i<ctdb->num_nodes;i++) {
+       for (i=0; i < ctdb->num_nodes; i++) {
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
                hdr->destnode = ctdb->nodes[i]->pnn;
                ctdb_queue_packet(ctdb, hdr);
        }
@@ -463,7 +517,10 @@ static void ctdb_broadcast_packet_connected(struct ctdb_context *ctdb,
                                            struct ctdb_req_header *hdr)
 {
        int i;
-       for (i=0;i<ctdb->num_nodes;i++) {
+       for (i=0; i < ctdb->num_nodes; i++) {
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
                if (!(ctdb->nodes[i]->flags & NODE_FLAGS_DISCONNECTED)) {
                        hdr->destnode = ctdb->nodes[i]->pnn;
                        ctdb_queue_packet(ctdb, hdr);

diff --git a/server/ctdb_takeover.c b/server/ctdb_takeover.c
index 9eac660e8d9f57dce1e2af43c8200c6e0f2aa727..21f7dc84899bce32ee3a8aea93f8b9593c1823f5 100644 (file)
--- a/server/ctdb_takeover.c
+++ b/server/ctdb_takeover.c
@@ -673,6 +673,10 @@ create_merged_ip_list(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx)
        for (i=0;i<ctdb->num_nodes;i++) {
                public_ips = ctdb->nodes[i]->public_ips;
 
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
+
                /* there were no public ips for this node */
                if (public_ips == NULL) {
                        continue;

diff --git a/tcp/tcp_connect.c b/tcp/tcp_connect.c
index 9d28d48a1f0d5a2c155bfd9b964b2d94965746c2..fc169e70b708eb96eb67835d5b5fc70a27d7bf0b 100644 (file)
--- a/tcp/tcp_connect.c
+++ b/tcp/tcp_connect.c
@@ -296,7 +296,11 @@ static int ctdb_tcp_listen_automatic(struct ctdb_context *ctdb)
                return -1;
        }
 
-       for (i=0;i<ctdb->num_nodes;i++) {
+       for (i=0; i < ctdb->num_nodes; i++) {
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
+
                /* if node_ip is specified we will only try to bind to that
                   ip.
                */

diff --git a/tcp/tcp_init.c b/tcp/tcp_init.c
index c0606f0ec5464f388de9b3f8b57d3da8cdbfc7cd..58ed6c8a28b7305ff4eab0a96aa8f9bf82df2f37 100644 (file)
--- a/tcp/tcp_init.c
+++ b/tcp/tcp_init.c
@@ -69,7 +69,10 @@ static int ctdb_tcp_initialise(struct ctdb_context *ctdb)
                exit(1);
        }
 
-       for (i=0; i<ctdb->num_nodes; i++) {
+       for (i=0; i < ctdb->num_nodes; i++) {
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
                if (ctdb_tcp_add_node(ctdb->nodes[i]) != 0) {
                        DEBUG(DEBUG_CRIT, ("methods->add_node failed at %d\n", i));
                        return -1;
@@ -135,7 +138,10 @@ static int ctdb_tcp_start(struct ctdb_context *ctdb)
 {
        int i;
 
-       for (i=0; i<ctdb->num_nodes; i++) {
+       for (i=0; i < ctdb->num_nodes; i++) {
+               if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
+                       continue;
+               }
                ctdb_tcp_connect_node(ctdb->nodes[i]);
        }
 

diff --git a/tools/ctdb.c b/tools/ctdb.c
index 57e1a8dffe351e40694578649c50a7ce342ed00b..fa4c5f9c2756bdebe61147a8122671d9a8bb22e5 100644 (file)
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -492,6 +492,7 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
                        { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
                        { NODE_FLAGS_BANNED,                "BANNED" },
                        { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
+                       { NODE_FLAGS_DELETED,               "DELETED" },
                };
                char *flags_str = NULL;
                int j;