Restart recovery dameon if it looks like it hung.

[sahlberg/ctdb.git] / server / ctdb_recover.c

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index 4a1e04c2f1f8e0400bb323a29f95fef1f6cfa3cb..eb3bf0a50c7dca2f9cbb31304d64b0e6ed9a4c95 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -18,7 +18,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "lib/tdb/include/tdb.h"
 #include "system/time.h"
 #include "system/network.h"
@@ -57,7 +57,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") != NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -68,7 +72,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") == NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -95,6 +103,7 @@ static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb, uint32_t pr
                if (ctdb_db->priority != priority) {
                        continue;
                }
+               tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
                }
@@ -340,10 +349,8 @@ static int traverse_pulldb(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
        }
        params->pulldata = talloc_realloc_size(NULL, params->pulldata, rec->length + params->len);
        if (params->pulldata == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed to expand pulldb_data to %u (%u records)\n", 
-                        rec->length + params->len, params->pulldata->count));
-               params->failed = true;
-               return -1;
+               DEBUG(DEBUG_CRIT,(__location__ " Failed to expand pulldb_data to %u\n", rec->length + params->len));
+               ctdb_fatal(params->ctdb, "failed to allocate memory for recovery. shutting down\n");
        }
        params->pulldata->count++;
        memcpy(params->len+(uint8_t *)params->pulldata, rec, rec->length);
@@ -577,7 +584,7 @@ static int set_recmode_destructor(struct ctdb_set_recmode_state *state)
 {
        double l = timeval_elapsed(&state->start_time);
 
-       ctdb_reclock_latency(state->ctdb, "daemon reclock", &state->ctdb->statistics.reclock.ctdbd, l);
+       CTDB_UPDATE_RECLOCK_LATENCY(state->ctdb, "daemon reclock", reclock.ctdbd, l);
 
        if (state->fd[0] != -1) {
                state->fd[0] = -1;
@@ -623,6 +630,11 @@ static void set_recmode_handler(struct event_context *ev, struct fd_event *fde,
 
        state->ctdb->recovery_mode = state->recmode;
 
+       /* release any deferred attach calls from clients */
+       if (state->recmode == CTDB_RECOVERY_NORMAL) {
+               ctdb_process_deferred_attach(state->ctdb);
+       }
+
        ctdb_request_control_reply(state->ctdb, state->c, NULL, 0, NULL);
        talloc_free(state);
        return;
@@ -641,6 +653,22 @@ ctdb_drop_all_ips_event(struct event_context *ev, struct timed_event *te,
        ctdb_release_all_ips(ctdb);
 }
 
+/*
+ * Set up an event to drop all public ips if we remain in recovery for too
+ * long
+ */
+int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb)
+{
+       if (ctdb->release_ips_ctx != NULL) {
+               talloc_free(ctdb->release_ips_ctx);
+       }
+       ctdb->release_ips_ctx = talloc_new(ctdb);
+       CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
+
+       event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+       return 0;
+}
+
 /*
   set the recovery mode
  */
@@ -661,11 +689,9 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                talloc_free(ctdb->release_ips_ctx);
                ctdb->release_ips_ctx = NULL;
        } else {
-               talloc_free(ctdb->release_ips_ctx);
-               ctdb->release_ips_ctx = talloc_new(ctdb);
-               CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
-
-               event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+               if (ctdb_deferred_drop_all_ips(ctdb) != 0) {
+                       DEBUG(DEBUG_ERR,("Failed to set up deferred drop all ips\n"));
+               }
        }
 
        if (recmode != ctdb->recovery_mode) {
@@ -695,6 +721,11 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
        state->fd[0] = -1;
        state->fd[1] = -1;
 
+       /* release any deferred attach calls from clients */
+       if (recmode == CTDB_RECOVERY_NORMAL) {
+               ctdb_process_deferred_attach(ctdb);
+       }
+
        if (ctdb->tunable.verify_recovery_lock == 0) {
                /* dont need to verify the reclock file */
                ctdb->recovery_mode = recmode;
@@ -725,6 +756,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                char cc = 0;
                close(state->fd[0]);
 
+               debug_extra = talloc_asprintf(NULL, "set_recmode:");
                /* we should not be able to get the lock on the reclock file, 
                  as it should  be held by the recovery master 
                */
@@ -754,7 +786,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                                    ctdb_set_recmode_timeout, state);
 
        state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
-                               EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+                               EVENT_FD_READ,
                                set_recmode_handler,
                                (void *)state);
 
@@ -762,6 +794,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                talloc_free(state);
                return -1;
        }
+       tevent_fd_set_auto_close(state->fde);
 
        state->ctdb    = ctdb;
        state->recmode = recmode;
@@ -938,6 +971,7 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo
        struct recovery_callback_state *state = talloc_get_type(p, struct recovery_callback_state);
 
        ctdb_enable_monitoring(ctdb);
+       CTDB_INCREMENT_STAT(ctdb, num_recoveries);
 
        if (status != 0) {
                DEBUG(DEBUG_ERR,(__location__ " recovered event script failed (status %d)\n", status));
@@ -964,6 +998,8 @@ int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb,
 
        DEBUG(DEBUG_NOTICE,("Recovery has finished\n"));
 
+       ctdb_persistent_finish_trans3_commits(ctdb);
+
        state = talloc(ctdb, struct recovery_callback_state);
        CTDB_NO_MEMORY(ctdb, state);
 
@@ -1161,18 +1197,10 @@ static void ctdb_recd_ping_timeout(struct event_context *ev, struct timed_event
                return;
        }
 
-       DEBUG(DEBUG_ERR, ("Final timeout for recovery daemon ping. Shutting down ctdb daemon. (This can be caused if the cluster filesystem has hung)\n"));
+       DEBUG(DEBUG_ERR, ("Final timeout for recovery daemon ping. Restarting recovery daemon. (This can be caused if the cluster filesystem has hung)\n"));
 
        ctdb_stop_recoverd(ctdb);
-       ctdb_stop_keepalive(ctdb);
-       ctdb_stop_monitoring(ctdb);
-       ctdb_release_all_ips(ctdb);
-       if (ctdb->methods != NULL) {
-               ctdb->methods->shutdown(ctdb);
-       }
-       ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
-       DEBUG(DEBUG_ERR, ("Recovery daemon ping timeout. Daemon has been shut down.\n"));
-       exit(0);
+       ctdb_start_recoverd(ctdb);
 }
 
 /* The recovery daemon will ping us at regular intervals.