Interface monitoring: add a event to trigger every 30 seconds to check that all inter...

This will make it much easier to root-cause problems such as
S1029023
when an external application deleted the interface while it is still is in use by ctdbd.

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 52c0be28c5b0b0876bec49bdce90180ed166e5ea..b74cbcf0a29b6dfcd98994d646bbbecb5ad428dd 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -414,6 +414,7 @@ struct ctdb_context {
        uint32_t recovery_mode;
        TALLOC_CTX *tickle_update_context;
        TALLOC_CTX *keepalive_ctx;
+       TALLOC_CTX *check_public_ifaces_ctx;
        struct ctdb_tunable tunable;
        enum ctdb_freeze_mode freeze_mode[NUM_DB_PRIORITIES+1];
        struct ctdb_freeze_handle *freeze_handles[NUM_DB_PRIORITIES+1];

diff --git a/server/ctdb_takeover.c b/server/ctdb_takeover.c
index f3644427878dd9cc115850e1db923b2396073636..4114b40f83e0ccb76da575a39d3e1a1ed8d70761 100644 (file)
--- a/server/ctdb_takeover.c
+++ b/server/ctdb_takeover.c
@@ -951,6 +951,51 @@ int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir)
        return 0;
 }
 
+static void ctdb_check_interfaces_event(struct event_context *ev, struct timed_event *te, 
+                                 struct timeval t, void *private_data)
+{
+       struct ctdb_context *ctdb = talloc_get_type(private_data, 
+                                                       struct ctdb_context);
+       struct ctdb_vnn *vnn;
+
+       for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
+               int i;
+
+               for (i=0; vnn->ifaces[i] != NULL; i++) {
+                       if (!ctdb_sys_check_iface_exists(vnn->ifaces[i])) {
+                               DEBUG(DEBUG_CRIT,("Interface %s does not exist but is used by public ip %s\n",
+                                       vnn->ifaces[i],
+                                       ctdb_addr_to_str(&vnn->public_address)));
+                       }
+               }
+       }
+
+       event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx, 
+               timeval_current_ofs(30, 0), 
+               ctdb_check_interfaces_event, ctdb);
+}
+
+
+static int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb)
+{
+       if (ctdb->check_public_ifaces_ctx != NULL) {
+               talloc_free(ctdb->check_public_ifaces_ctx);
+               ctdb->check_public_ifaces_ctx = NULL;
+       }
+
+       ctdb->check_public_ifaces_ctx = talloc_new(ctdb);
+       if (ctdb->check_public_ifaces_ctx == NULL) {
+               ctdb_fatal(ctdb, "failed to allocate context for checking interfaces");
+       }
+
+       event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx, 
+               timeval_current_ofs(30, 0), 
+               ctdb_check_interfaces_event, ctdb);
+
+       return 0;
+}
+
+
 /*
   setup the public address lists from a file
 */
@@ -1013,6 +1058,9 @@ int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist)
                }
        }
 
+
+       ctdb_start_monitoring_interfaces(ctdb);
+
        talloc_free(lines);
        return 0;
 }