Restart recovery dameon if it looks like it hung.

[sahlberg/ctdb.git] / server / ctdb_recover.c

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index 81e2d4b2eb4d7b1a1e97d8b29d4139459613b1c4..eb3bf0a50c7dca2f9cbb31304d64b0e6ed9a4c95 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -57,7 +57,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") != NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -68,7 +72,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") == NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -95,6 +103,7 @@ static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb, uint32_t pr
                if (ctdb_db->priority != priority) {
                        continue;
                }
+               tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
                }
@@ -575,7 +584,7 @@ static int set_recmode_destructor(struct ctdb_set_recmode_state *state)
 {
        double l = timeval_elapsed(&state->start_time);
 
-       ctdb_reclock_latency(state->ctdb, "daemon reclock", &state->ctdb->statistics.reclock.ctdbd, l);
+       CTDB_UPDATE_RECLOCK_LATENCY(state->ctdb, "daemon reclock", reclock.ctdbd, l);
 
        if (state->fd[0] != -1) {
                state->fd[0] = -1;
@@ -621,6 +630,11 @@ static void set_recmode_handler(struct event_context *ev, struct fd_event *fde,
 
        state->ctdb->recovery_mode = state->recmode;
 
+       /* release any deferred attach calls from clients */
+       if (state->recmode == CTDB_RECOVERY_NORMAL) {
+               ctdb_process_deferred_attach(state->ctdb);
+       }
+
        ctdb_request_control_reply(state->ctdb, state->c, NULL, 0, NULL);
        talloc_free(state);
        return;
@@ -639,6 +653,22 @@ ctdb_drop_all_ips_event(struct event_context *ev, struct timed_event *te,
        ctdb_release_all_ips(ctdb);
 }
 
+/*
+ * Set up an event to drop all public ips if we remain in recovery for too
+ * long
+ */
+int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb)
+{
+       if (ctdb->release_ips_ctx != NULL) {
+               talloc_free(ctdb->release_ips_ctx);
+       }
+       ctdb->release_ips_ctx = talloc_new(ctdb);
+       CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
+
+       event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+       return 0;
+}
+
 /*
   set the recovery mode
  */
@@ -659,11 +689,9 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                talloc_free(ctdb->release_ips_ctx);
                ctdb->release_ips_ctx = NULL;
        } else {
-               talloc_free(ctdb->release_ips_ctx);
-               ctdb->release_ips_ctx = talloc_new(ctdb);
-               CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
-
-               event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+               if (ctdb_deferred_drop_all_ips(ctdb) != 0) {
+                       DEBUG(DEBUG_ERR,("Failed to set up deferred drop all ips\n"));
+               }
        }
 
        if (recmode != ctdb->recovery_mode) {
@@ -693,6 +721,11 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
        state->fd[0] = -1;
        state->fd[1] = -1;
 
+       /* release any deferred attach calls from clients */
+       if (recmode == CTDB_RECOVERY_NORMAL) {
+               ctdb_process_deferred_attach(ctdb);
+       }
+
        if (ctdb->tunable.verify_recovery_lock == 0) {
                /* dont need to verify the reclock file */
                ctdb->recovery_mode = recmode;
@@ -938,7 +971,7 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo
        struct recovery_callback_state *state = talloc_get_type(p, struct recovery_callback_state);
 
        ctdb_enable_monitoring(ctdb);
-       ctdb->statistics.num_recoveries++;
+       CTDB_INCREMENT_STAT(ctdb, num_recoveries);
 
        if (status != 0) {
                DEBUG(DEBUG_ERR,(__location__ " recovered event script failed (status %d)\n", status));
@@ -965,6 +998,8 @@ int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb,
 
        DEBUG(DEBUG_NOTICE,("Recovery has finished\n"));
 
+       ctdb_persistent_finish_trans3_commits(ctdb);
+
        state = talloc(ctdb, struct recovery_callback_state);
        CTDB_NO_MEMORY(ctdb, state);
 
@@ -1162,18 +1197,10 @@ static void ctdb_recd_ping_timeout(struct event_context *ev, struct timed_event
                return;
        }
 
-       DEBUG(DEBUG_ERR, ("Final timeout for recovery daemon ping. Shutting down ctdb daemon. (This can be caused if the cluster filesystem has hung)\n"));
+       DEBUG(DEBUG_ERR, ("Final timeout for recovery daemon ping. Restarting recovery daemon. (This can be caused if the cluster filesystem has hung)\n"));
 
        ctdb_stop_recoverd(ctdb);
-       ctdb_stop_keepalive(ctdb);
-       ctdb_stop_monitoring(ctdb);
-       ctdb_release_all_ips(ctdb);
-       if (ctdb->methods != NULL) {
-               ctdb->methods->shutdown(ctdb);
-       }
-       ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
-       DEBUG(DEBUG_ERR, ("Recovery daemon ping timeout. Daemon has been shut down.\n"));
-       exit(0);
+       ctdb_start_recoverd(ctdb);
 }
 
 /* The recovery daemon will ping us at regular intervals.