ctdb: hold transaction locks during freeze, mark during recover.

[sahlberg/ctdb.git] / server / ctdb_recover.c

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index 9020363016e11b785fc4060d308ac6ee6e9b10b1..4db4d97f0bbc61c5053a1a15b15b3101b25f2968 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -18,7 +18,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "lib/tdb/include/tdb.h"
 #include "system/time.h"
 #include "system/network.h"
@@ -57,7 +57,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") != NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -68,7 +72,11 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t prio
                if (strstr(ctdb_db->db_name, "notify") == NULL) {
                        continue;
                }
+               if (tdb_transaction_write_lock_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                        return -1;
                }
        }
@@ -95,6 +103,7 @@ static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb, uint32_t pr
                if (ctdb_db->priority != priority) {
                        continue;
                }
+               tdb_transaction_write_lock_unmark(ctdb_db->ltdb->tdb);
                if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
                }
@@ -340,10 +349,8 @@ static int traverse_pulldb(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
        }
        params->pulldata = talloc_realloc_size(NULL, params->pulldata, rec->length + params->len);
        if (params->pulldata == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed to expand pulldb_data to %u (%u records)\n", 
-                        rec->length + params->len, params->pulldata->count));
-               params->failed = true;
-               return -1;
+               DEBUG(DEBUG_CRIT,(__location__ " Failed to expand pulldb_data to %u\n", rec->length + params->len));
+               ctdb_fatal(params->ctdb, "failed to allocate memory for recovery. shutting down\n");
        }
        params->pulldata->count++;
        memcpy(params->len+(uint8_t *)params->pulldata, rec, rec->length);
@@ -577,7 +584,7 @@ static int set_recmode_destructor(struct ctdb_set_recmode_state *state)
 {
        double l = timeval_elapsed(&state->start_time);
 
-       ctdb_reclock_latency(state->ctdb, "daemon reclock", &state->ctdb->statistics.reclock.ctdbd, l);
+       CTDB_UPDATE_RECLOCK_LATENCY(state->ctdb, "daemon reclock", reclock.ctdbd, l);
 
        if (state->fd[0] != -1) {
                state->fd[0] = -1;
@@ -641,6 +648,22 @@ ctdb_drop_all_ips_event(struct event_context *ev, struct timed_event *te,
        ctdb_release_all_ips(ctdb);
 }
 
+/*
+ * Set up an event to drop all public ips if we remain in recovery for too
+ * long
+ */
+int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb)
+{
+       if (ctdb->release_ips_ctx != NULL) {
+               talloc_free(ctdb->release_ips_ctx);
+       }
+       ctdb->release_ips_ctx = talloc_new(ctdb);
+       CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
+
+       event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+       return 0;
+}
+
 /*
   set the recovery mode
  */
@@ -661,11 +684,9 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                talloc_free(ctdb->release_ips_ctx);
                ctdb->release_ips_ctx = NULL;
        } else {
-               talloc_free(ctdb->release_ips_ctx);
-               ctdb->release_ips_ctx = talloc_new(ctdb);
-               CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
-
-               event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+               if (ctdb_deferred_drop_all_ips(ctdb) != 0) {
+                       DEBUG(DEBUG_ERR,("Failed to set up deferred drop all ips\n"));
+               }
        }
 
        if (recmode != ctdb->recovery_mode) {
@@ -725,6 +746,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                char cc = 0;
                close(state->fd[0]);
 
+               debug_extra = talloc_asprintf(NULL, "set_recmode:");
                /* we should not be able to get the lock on the reclock file, 
                  as it should  be held by the recovery master 
                */
@@ -754,7 +776,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                                    ctdb_set_recmode_timeout, state);
 
        state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
-                               EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+                               EVENT_FD_READ,
                                set_recmode_handler,
                                (void *)state);
 
@@ -762,6 +784,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                talloc_free(state);
                return -1;
        }
+       tevent_fd_set_auto_close(state->fde);
 
        state->ctdb    = ctdb;
        state->recmode = recmode;
@@ -819,7 +842,7 @@ bool ctdb_recovery_lock(struct ctdb_context *ctdb, bool keep)
        }
 
        if (keep) {
-               DEBUG(DEBUG_ERR, ("Recovery lock taken successfully\n"));
+               DEBUG(DEBUG_NOTICE, ("Recovery lock taken successfully\n"));
        }
 
        DEBUG(DEBUG_NOTICE,("ctdb_recovery_lock: Got recovery lock on '%s'\n", ctdb->recovery_lock_file));
@@ -938,6 +961,7 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo
        struct recovery_callback_state *state = talloc_get_type(p, struct recovery_callback_state);
 
        ctdb_enable_monitoring(ctdb);
+       CTDB_INCREMENT_STAT(ctdb, num_recoveries);
 
        if (status != 0) {
                DEBUG(DEBUG_ERR,(__location__ " recovered event script failed (status %d)\n", status));
@@ -967,7 +991,7 @@ int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb,
        state = talloc(ctdb, struct recovery_callback_state);
        CTDB_NO_MEMORY(ctdb, state);
 
-       state->c    = talloc_steal(state, c);
+       state->c    = c;
 
        ctdb_disable_monitoring(ctdb);
 
@@ -986,6 +1010,7 @@ int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb,
        }
 
        /* tell the control that we will be reply asynchronously */
+       state->c    = talloc_steal(state, c);
        *async_reply = true;
        return 0;
 }