Add ctdb_fork(0 which will fork a child process and drop the real-time

[sahlberg/ctdb.git] / server / ctdb_monitor.c

diff --git a/server/ctdb_monitor.c b/server/ctdb_monitor.c
index f4223772b6c8cef06f8d8aff0d7111fe31b2f18a..fa642fb3fd7269a98a9267a5c58cbbf61976a86f 100644 (file)
--- a/server/ctdb_monitor.c
+++ b/server/ctdb_monitor.c
@@ -19,7 +19,7 @@
 */
 
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "system/filesys.h"
 #include "system/wait.h"
 #include "../include/ctdb_private.h"
@@ -75,7 +75,7 @@ static int ctdb_run_notification_script_child(struct ctdb_context *ctdb, const c
        return ret;
 }
 
-static void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
+void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
 {
        pid_t child;
 
@@ -83,7 +83,7 @@ static void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *
                return;
        }
 
-       child = fork();
+       child = ctdb_fork(ctdb);
        if (child == (pid_t)-1) {
                DEBUG(DEBUG_ERR,("Failed to fork() a notification child process\n"));
                return;
@@ -91,6 +91,7 @@ static void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *
        if (child == 0) {
                int ret;
 
+               debug_extra = talloc_asprintf(NULL, "notification-%s:", event);
                ret = ctdb_run_notification_script_child(ctdb, event);
                if (ret != 0) {
                        DEBUG(DEBUG_ERR,(__location__ " Notification script failed\n"));
@@ -123,14 +124,21 @@ static void ctdb_health_callback(struct ctdb_context *ctdb, int status, void *p)
        rddata.dptr = (uint8_t *)&rd;
        rddata.dsize = sizeof(rd);
 
+       if (status == -ETIME) {
+               ctdb->event_script_timeouts++;
+
+               if (ctdb->event_script_timeouts >= ctdb->tunable.script_timeout_count) {
+                       DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_timeout_count));
+               } else {
+                       /* We pretend this is OK. */
+                       goto after_change_status;
+               }
+       }
+
        if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
                DEBUG(DEBUG_NOTICE,("monitor event failed - disabling node\n"));
                node->flags |= NODE_FLAGS_UNHEALTHY;
                ctdb->monitor->next_interval = 5;
-               if (ctdb->tunable.disable_when_unhealthy != 0) {
-                       DEBUG(DEBUG_INFO, ("DISABLING node since it became unhealthy\n"));
-                       node->flags |= NODE_FLAGS_DISABLED;
-               }
 
                ctdb_run_notification_script(ctdb, "unhealthy");
 
@@ -157,6 +165,7 @@ static void ctdb_health_callback(struct ctdb_context *ctdb, int status, void *p)
 
        }
 
+after_change_status:
        next_interval = ctdb->monitor->next_interval;
 
        ctdb->monitor->next_interval *= 2;
@@ -194,7 +203,7 @@ static void ctdb_startup_callback(struct ctdb_context *ctdb, int status, void *p
        } else if (status == 0) {
                DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
                ctdb->done_startup = true;
-               ctdb->monitor->next_interval = 5;
+               ctdb->monitor->next_interval = 2;
                ctdb_run_notification_script(ctdb, "startup");
        }
 
@@ -212,10 +221,13 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve
                              struct timeval t, void *private_data)
 {
        struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+       int ret;
 
        DEBUG(DEBUG_NOTICE,("CTDB_WAIT_UNTIL_RECOVERED\n"));
 
        if (ctdb->vnn_map->generation == INVALID_GENERATION) {
+               ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
                DEBUG(DEBUG_NOTICE,(__location__ " generation is INVALID. Wait one more second\n"));
                event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
                                     timeval_current_ofs(1, 0), 
@@ -224,6 +236,8 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve
        }
 
        if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
+               ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
                DEBUG(DEBUG_NOTICE,(__location__ " in recovery. Wait one more second\n"));
                event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
                                     timeval_current_ofs(1, 0), 
@@ -232,7 +246,9 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve
        }
 
 
-       if (timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
+       if (!fast_start && timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
+               ctdb->db_persistent_startup_generation = INVALID_GENERATION;
+
                DEBUG(DEBUG_NOTICE,(__location__ " wait for pending recoveries to end. Wait one more second.\n"));
 
                event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
@@ -241,10 +257,52 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve
                return;
        }
 
+       if (ctdb->vnn_map->generation == ctdb->db_persistent_startup_generation) {
+               DEBUG(DEBUG_INFO,(__location__ " skip ctdb_recheck_persistent_health() "
+                                 "until the next recovery\n"));
+               event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+                                    timeval_current_ofs(1, 0),
+                                    ctdb_wait_until_recovered, ctdb);
+               return;
+       }
+
+       ctdb->db_persistent_startup_generation = ctdb->vnn_map->generation;
+       ret = ctdb_recheck_persistent_health(ctdb);
+       if (ret != 0) {
+               ctdb->db_persistent_check_errors++;
+               if (ctdb->db_persistent_check_errors < ctdb->max_persistent_check_errors) {
+                       DEBUG(ctdb->db_persistent_check_errors==1?DEBUG_ERR:DEBUG_WARNING,
+                             (__location__ "ctdb_recheck_persistent_health() "
+                             "failed (%llu of %llu times) - retry later\n",
+                             (unsigned long long)ctdb->db_persistent_check_errors,
+                             (unsigned long long)ctdb->max_persistent_check_errors));
+                       event_add_timed(ctdb->ev,
+                                       ctdb->monitor->monitor_context,
+                                       timeval_current_ofs(1, 0),
+                                       ctdb_wait_until_recovered, ctdb);
+                       return;
+               }
+               DEBUG(DEBUG_ALERT,(__location__
+                                 "ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown\n",
+                                 (unsigned long long)ctdb->db_persistent_check_errors));
+               ctdb_stop_recoverd(ctdb);
+               ctdb_stop_keepalive(ctdb);
+               ctdb_stop_monitoring(ctdb);
+               ctdb_release_all_ips(ctdb);
+               if (ctdb->methods != NULL) {
+                       ctdb->methods->shutdown(ctdb);
+               }
+               ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
+               DEBUG(DEBUG_ALERT,("ctdb_recheck_persistent_health() failed - Stopping CTDB daemon\n"));
+               exit(11);
+       }
+       ctdb->db_persistent_check_errors = 0;
+       DEBUG(DEBUG_NOTICE,(__location__
+                          "ctdb_start_monitoring: ctdb_recheck_persistent_health() OK\n"));
 
        DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n"));
        event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
-                            timeval_current_ofs(1, 0), 
+                            timeval_current(),
                             ctdb_check_health, ctdb);
 }
 
@@ -413,6 +471,11 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 
        DEBUG(DEBUG_INFO, ("Control modflags on node %u - flags now 0x%x\n", c->pnn, node->flags));
 
+       if (node->flags == 0 && !ctdb->done_startup) {
+               DEBUG(DEBUG_ERR, (__location__ " Node %u became healthy - force recovery for startup\n",
+                                 c->pnn));
+               ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
+       }
 
        /* tell the recovery daemon something has changed */
        ctdb_daemon_send_message(ctdb, ctdb->pnn,