CTDB_FATAL() try to drop all ip addresses when shutting down through ctdb_fatal()

If we shutdown through a ctdb_fatal(), unless there is an external mechanism to trap this and restart ctdb, we will get a duplicate address problem since addresses held on the node remain which collides with the addresses when failed over to other nodes.

During ctdb_fatal(), spawn a child process to try to drop all held public addresses.

diff --git a/common/ctdb_util.c b/common/ctdb_util.c
index 835bbfd5e2388a248de6d2b34c975ba72d838796..50a8a0dcd7a0619136210e1dbbf17cb6e449da7a 100644 (file)
--- a/common/ctdb_util.c
+++ b/common/ctdb_util.c
@@ -57,6 +57,9 @@ void ctdb_set_error(struct ctdb_context *ctdb, const char *fmt, ...)
 void ctdb_fatal(struct ctdb_context *ctdb, const char *msg)
 {
        DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n", msg));
+       if (ctdb->emergency_shutdown != NULL) {
+               ctdb->emergency_shutdown(ctdb);
+       }
        abort();
 }
 

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 41a761a5355fbea779675c63fc58d6ca8fbe7216..38aab198abc5027f8f939e30a4d601ec83a99c78 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -474,6 +474,9 @@ struct ctdb_context {
 
        /* used in the recovery daemon to remember the ip allocation */
        struct trbt_tree *ip_tree;
+
+       /* emergency shutdown cleanup */
+       void (*emergency_shutdown)(struct ctdb_context *);
 };
 
 struct ctdb_db_context {
@@ -1576,4 +1579,6 @@ int verify_remote_ip_allocation(struct ctdb_context *ctdb,
 int update_ip_assignment_tree(struct ctdb_context *ctdb,
                                struct ctdb_public_ip *ip);
 
+void ctdb_emergency_shutdown(struct ctdb_context *ctdb);
+
 #endif

diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 4c35b1c7b7928fdc1e67402c5d81e983c57d8740..8e5fcbf12c13e771e400aa42e8be8bcc6d414a6d 100644 (file)
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -63,6 +63,11 @@ static void ctdb_start_transport(struct ctdb_context *ctdb)
                exit(11);
        }
 
+       /* set up a function so we try to clean up during emergency shutdown such as ctdb_fatal()
+          at the very least we should drop all ips at this stage
+       */
+       ctdb->emergency_shutdown = ctdb_emergency_shutdown;
+
        /* Make sure we log something when the daemon terminates */
        atexit(print_exit_message);
 

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index 4435ab315d65bfb220bdabd337b5d5016fcadff5..c3c514de171b05ee3f1f83b306ba4fed6f9852f6 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -1262,3 +1262,14 @@ int32_t ctdb_control_continue_node(struct ctdb_context *ctdb)
 
        return 0;
 }
+
+void ctdb_emergency_shutdown(struct ctdb_context *ctdb)
+{
+       DEBUG(DEBUG_ERR,("CTDB_FATAL create child process to release all held ips\n"));
+       if (fork() == 0) {
+               DEBUG(DEBUG_ERR,("CTDB_FATAL drop all ip\n"));
+               ctdb_release_all_ips(ctdb);
+               DEBUG(DEBUG_ERR,("CTDB_FATAL all ips dropped\n"));
+               _exit(0);
+       }
+}