initial attempt at freezing databases in priority order

diff --git a/client/ctdb_client.c b/client/ctdb_client.c
index b27086b579e463d168beed2aeb6c8cf9eb651b0d..e60a72ebb52f71e566d246b7ceb44537c6ddae94 100644 (file)
--- a/client/ctdb_client.c
+++ b/client/ctdb_client.c
@@ -1986,7 +1986,14 @@ int ctdb_ctrl_freeze_priority(struct ctdb_context *ctdb, struct timeval timeout,
 /* Freeze all databases */
 int ctdb_ctrl_freeze(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
 {
-       return ctdb_ctrl_freeze_priority(ctdb, timeout, destnode, 0);
+       int i;
+
+       for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb_ctrl_freeze_priority(ctdb, timeout, destnode, i) != 0) {
+                       return -1;
+               }
+       }
+       return 0;
 }
 
 /*
@@ -2407,7 +2414,7 @@ int ctdb_ctrl_modflags(struct ctdb_context *ctdb, struct timeval timeout, uint32
        nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
 
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_MODIFY_FLAGS,
-                                       nodes,
+                                       nodes, 0,
                                        timeout, false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -2900,6 +2907,7 @@ int ctdb_client_async_wait(struct ctdb_context *ctdb, struct client_async_data *
 int ctdb_client_async_control(struct ctdb_context *ctdb,
                                enum ctdb_controls opcode,
                                uint32_t *nodes,
+                               uint64_t srvid,
                                struct timeval timeout,
                                bool dont_log_errors,
                                TDB_DATA data,
@@ -2925,7 +2933,7 @@ int ctdb_client_async_control(struct ctdb_context *ctdb,
        for (j=0; j<num_nodes; j++) {
                uint32_t pnn = nodes[j];
 
-               state = ctdb_control_send(ctdb, pnn, 0, opcode, 
+               state = ctdb_control_send(ctdb, pnn, srvid, opcode, 
                                          0, data, async_data, &timeout, NULL);
                if (state == NULL) {
                        DEBUG(DEBUG_ERR,(__location__ " Failed to call async control %u\n", (unsigned)opcode));

diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 774a36603061e088b3534f11d4fb524bba5063e9..b033949e4d73baf4a43c6d74275752d0d919b767 100644 (file)
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -364,6 +364,7 @@ enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN
 /* This capability is set if CTDB_LVS_PUBLIC_IP is set */
 #define CTDB_CAP_LVS                   0x00000004
 
+#define NUM_DB_PRIORITIES 3
 /* main state of the ctdb daemon */
 struct ctdb_context {
        struct event_context *ev;
@@ -374,8 +375,10 @@ struct ctdb_context {
        TALLOC_CTX *tickle_update_context;
        TALLOC_CTX *keepalive_ctx;
        struct ctdb_tunable tunable;
-       enum ctdb_freeze_mode freeze_mode;
-       struct ctdb_freeze_handle *freeze_handle;
+       enum ctdb_freeze_mode freeze_mode[NUM_DB_PRIORITIES+1];
+       struct ctdb_freeze_handle *freeze_handles[NUM_DB_PRIORITIES+1];
+       bool freeze_transaction_started;
+       uint32_t freeze_transaction_id;
        struct ctdb_address address;
        const char *name;
        const char *db_directory;
@@ -1172,7 +1175,7 @@ void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_contr
                                TDB_DATA *outdata, int32_t status, const char *errormsg);
 
 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
-int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c);
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority);
 
 int ctdb_start_recoverd(struct ctdb_context *ctdb);
 void ctdb_stop_recoverd(struct ctdb_context *ctdb);
@@ -1311,7 +1314,7 @@ int ctdb_ctrl_get_all_tunables(struct ctdb_context *ctdb,
                               uint32_t destnode,
                               struct ctdb_tunable *tunables);
 
-void ctdb_start_freeze(struct ctdb_context *ctdb);
+int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority);
 
 bool parse_ip_mask(const char *s, const char *iface, ctdb_sock_addr *addr, unsigned *mask);
 bool parse_ip_port(const char *s, ctdb_sock_addr *addr);
@@ -1406,6 +1409,7 @@ int ctdb_client_async_wait(struct ctdb_context *ctdb, struct client_async_data *
 int ctdb_client_async_control(struct ctdb_context *ctdb,
                                enum ctdb_controls opcode,
                                uint32_t *nodes,
+                               uint64_t srvid,
                                struct timeval timeout,
                                bool dont_log_errors,
                                TDB_DATA data,

diff --git a/server/ctdb_control.c b/server/ctdb_control.c
index 2f0b94072a04376cbeb38ea0ef34a8079fa3ce82..1574039d543d50987ef5b6f69a93bebafb972157 100644 (file)
--- a/server/ctdb_control.c
+++ b/server/ctdb_control.c
@@ -98,9 +98,15 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
        }
 
        case CTDB_CONTROL_STATISTICS: {
+               int i;
                CHECK_CONTROL_DATA_SIZE(0);
                ctdb->statistics.memory_used = talloc_total_size(NULL);
-               ctdb->statistics.frozen = (ctdb->freeze_mode == CTDB_FREEZE_FROZEN);
+               ctdb->statistics.frozen = 0;
+               for (i=1; i<= NUM_DB_PRIORITIES; i++) {
+                       if (ctdb->freeze_mode[i] == CTDB_FREEZE_FROZEN) {
+                               ctdb->statistics.frozen = 1;
+                       }
+               }
                ctdb->statistics.recovering = (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE);
                outdata->dptr = (uint8_t *)&ctdb->statistics;
                outdata->dsize = sizeof(ctdb->statistics);
@@ -249,7 +255,7 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 
        case CTDB_CONTROL_THAW:
                CHECK_CONTROL_DATA_SIZE(0);
-               return ctdb_control_thaw(ctdb, c);
+               return ctdb_control_thaw(ctdb, (uint32_t)c->srvid);
 
        case CTDB_CONTROL_SET_RECMODE:
                CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));              

diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 1a94cfc661ba60f037e5df8a65d0a146ed0177fc..5c4655d26e67fe850ec02b177afd13131bcfab1a 100644 (file)
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -724,7 +724,7 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork)
        }
 
        /* start frozen, then let the first election sort things out */
-       if (!ctdb_blocking_freeze(ctdb)) {
+       if (ctdb_blocking_freeze(ctdb)) {
                ctdb_fatal(ctdb, "Failed to get initial freeze\n");
        }
 

diff --git a/server/ctdb_freeze.c b/server/ctdb_freeze.c
index ed0bbe8d6cb64e60b1c2668ba8039d7712598264..53a76f3b78b10bf26e3f3bd015e388d2428161ad 100644 (file)
--- a/server/ctdb_freeze.c
+++ b/server/ctdb_freeze.c
@@ -30,10 +30,13 @@
 /*
   lock all databases
  */
-static int ctdb_lock_all_databases(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases(struct ctdb_context *ctdb, uint32_t priority)
 {
        struct ctdb_db_context *ctdb_db;
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (ctdb_db->priority != priority) {
+                       continue;
+               }
                DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
                if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
@@ -50,17 +53,17 @@ struct ctdb_freeze_waiter {
        struct ctdb_freeze_waiter *next, *prev;
        struct ctdb_context *ctdb;
        struct ctdb_req_control *c;
+       uint32_t priority;
        int32_t status;
 };
 
 /* a handle to a freeze lock child process */
 struct ctdb_freeze_handle {
        struct ctdb_context *ctdb;
+       uint32_t priority;
        pid_t child;
        int fd;
        struct ctdb_freeze_waiter *waiters;
-       bool transaction_started;
-       uint32_t transaction_id;
 };
 
 /*
@@ -71,9 +74,14 @@ static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
        struct ctdb_context *ctdb = h->ctdb;
        struct ctdb_db_context *ctdb_db;
 
+       DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
+
        /* cancel any pending transactions */
-       if (ctdb->freeze_handle && ctdb->freeze_handle->transaction_started) {
+       if (ctdb->freeze_transaction_started) {
                for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+                       if (ctdb_db->priority != h->priority) {
+                               continue;
+                       }
                        tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
                        if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
                                DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
@@ -81,11 +89,11 @@ static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
                        }
                        tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
                }
-               ctdb->freeze_handle->transaction_started = false;
+               ctdb->freeze_transaction_started = false;
        }
 
-       ctdb->freeze_mode = CTDB_FREEZE_NONE;
-       ctdb->freeze_handle = NULL;
+       ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
+       ctdb->freeze_handles[h->priority] = NULL;
 
        kill(h->child, SIGKILL);
        return 0;
@@ -101,11 +109,8 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
        int32_t status;
        struct ctdb_freeze_waiter *w;
 
-       if (h->ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+       if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
                DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
-               if (h->ctdb->freeze_handle == h) {
-                       h->ctdb->freeze_handle = NULL;
-               }
                talloc_free(h);
                return;
        }
@@ -122,12 +127,12 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
                return;
        }
 
-       h->ctdb->freeze_mode = CTDB_FREEZE_FROZEN;
+       h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 
        /* notify the waiters */
-       while ((w = h->ctdb->freeze_handle->waiters)) {
+       while ((w = h->ctdb->freeze_handles[h->priority]->waiters)) {
                w->status = status;
-               DLIST_REMOVE(h->ctdb->freeze_handle->waiters, w);
+               DLIST_REMOVE(h->ctdb->freeze_handles[h->priority]->waiters, w);
                talloc_free(w);
        }
 }
@@ -136,7 +141,7 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
   create a child which gets locks on all the open databases, then calls the callback telling the parent
   that it is done
  */
-static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
+static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, uint32_t priority)
 {
        struct ctdb_freeze_handle *h;
        int fd[2];
@@ -145,7 +150,8 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
        h = talloc_zero(ctdb, struct ctdb_freeze_handle);
        CTDB_NO_MEMORY_NULL(ctdb, h);
 
-       h->ctdb = ctdb;
+       h->ctdb     = ctdb;
+       h->priority = priority;
 
        /* use socketpair() instead of pipe() so we have bi-directional fds */
        if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
@@ -166,7 +172,7 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
                int count = 0;
                /* in the child */
                close(fd[0]);
-               ret = ctdb_lock_all_databases(ctdb);
+               ret = ctdb_lock_all_databases(ctdb, priority);
                if (ret != 0) {
                        _exit(0);
                }
@@ -214,27 +220,34 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
  */
 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 {
-       DLIST_REMOVE(w->ctdb->freeze_handle->waiters, w);
+       DLIST_REMOVE(w->ctdb->freeze_handles[w->priority]->waiters, w);
        ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
        return 0;
 }
 
 /*
-  start the freeze process
+  start the freeze process for a certain priority
  */
-void ctdb_start_freeze(struct ctdb_context *ctdb)
+int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 {
-       if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+       if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+               DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+               return -1;
+       }
+
+       if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
                /* we're already frozen */
-               return;
+               return 0;
        }
 
        /* if there isn't a freeze lock child then create one */
-       if (!ctdb->freeze_handle) {
-               ctdb->freeze_handle = ctdb_freeze_lock(ctdb);
-               CTDB_NO_MEMORY_VOID(ctdb, ctdb->freeze_handle);
-               ctdb->freeze_mode = CTDB_FREEZE_PENDING;
+       if (!ctdb->freeze_handles[priority]) {
+               ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
+               CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
+               ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
        }
+
+       return 0;
 }
 
 /*
@@ -247,21 +260,32 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
 
        priority = (uint32_t)c->srvid;
 
-       if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+       DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
+
+       if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+               DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+               return -1;
+       }
+
+       if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
                /* we're already frozen */
                return 0;
        }
 
-       ctdb_start_freeze(ctdb);
+       if (ctdb_start_freeze(ctdb, priority) != 0) {
+               DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
+               return -1;
+       }
 
        /* add ourselves to list of waiters */
-       w = talloc(ctdb->freeze_handle, struct ctdb_freeze_waiter);
+       w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
        CTDB_NO_MEMORY(ctdb, w);
-       w->ctdb   = ctdb;
-       w->c      = talloc_steal(w, c);
-       w->status = -1;
+       w->ctdb     = ctdb;
+       w->c        = talloc_steal(w, c);
+       w->priority = priority;
+       w->status   = -1;
        talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
-       DLIST_ADD(ctdb->freeze_handle->waiters, w);
+       DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 
        /* we won't reply till later */
        *async_reply = True;
@@ -274,29 +298,30 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
  */
 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 {
-       ctdb_start_freeze(ctdb);
+       int i;
+
+       for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb_start_freeze(ctdb, i)) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
+                       continue;
+               }
 
-       /* block until frozen */
-       while (ctdb->freeze_mode == CTDB_FREEZE_PENDING) {
-               event_loop_once(ctdb->ev);
+               /* block until frozen */
+               while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
+                       event_loop_once(ctdb->ev);
+               }
        }
 
-       return ctdb->freeze_mode == CTDB_FREEZE_FROZEN;
+       return 0;
 }
 
 
-
-/*
-  thaw the databases
- */
-int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
+static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 {
-       uint32_t priority;
-
-       priority = (uint32_t)c->srvid;
+       DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 
        /* cancel any pending transactions */
-       if (ctdb->freeze_handle && ctdb->freeze_handle->transaction_started) {
+       if (ctdb->freeze_transaction_started) {
                struct ctdb_db_context *ctdb_db;
 
                for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
@@ -308,6 +333,7 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
                        tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
                }
        }
+       ctdb->freeze_transaction_started = false;
 
 #if 0
        /* this hack can be used to get a copy of the databases at the end of a recovery */
@@ -319,9 +345,30 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
        system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 #endif
 
+       talloc_free(ctdb->freeze_handles[priority]);
+       ctdb->freeze_handles[priority] = NULL;
+}
+
+/*
+  thaw the databases
+ */
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
+{
+
+       if (priority > NUM_DB_PRIORITIES) {
+               DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+               return -1;
+       }
+
+       if (priority == 0) {
+               int i;
+               for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+                       thaw_priority(ctdb, i);
+               }
+       } else {
+               thaw_priority(ctdb, priority);
+       }
 
-       talloc_free(ctdb->freeze_handle);
-       ctdb->freeze_handle = NULL;
        ctdb_call_resend_all(ctdb);
        return 0;
 }
@@ -333,19 +380,21 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 {
        struct ctdb_db_context *ctdb_db;
+       int i;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
-               return -1;
+       for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+                       return -1;
+               }
        }
 
-
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
                int ret;
 
                tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 
-               if (ctdb->freeze_handle->transaction_started) {
+               if (ctdb->freeze_transaction_started) {
                        if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
                                DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
                                         ctdb_db->db_name));
@@ -366,8 +415,8 @@ int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
                }
        }
 
-       ctdb->freeze_handle->transaction_started = true;
-       ctdb->freeze_handle->transaction_id = id;
+       ctdb->freeze_transaction_started = true;
+       ctdb->freeze_transaction_id = id;
 
        return 0;
 }
@@ -378,18 +427,21 @@ int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 {
        struct ctdb_db_context *ctdb_db;
-
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
-               return -1;
+       int i;
+       
+       for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+                       DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+                       return -1;
+               }
        }
 
-       if (!ctdb->freeze_handle->transaction_started) {
+       if (!ctdb->freeze_transaction_started) {
                DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
                return -1;
        }
 
-       if (id != ctdb->freeze_handle->transaction_id) {
+       if (id != ctdb->freeze_transaction_id) {
                DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
                return -1;
        }
@@ -409,15 +461,15 @@ int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
                                }
                                tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
                        }
-                       ctdb->freeze_handle->transaction_started = false;
+                       ctdb->freeze_transaction_started = false;
 
                        return -1;
                }
                tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
        }
 
-       ctdb->freeze_handle->transaction_started = false;
-       ctdb->freeze_handle->transaction_id = 0;
+       ctdb->freeze_transaction_started = false;
+       ctdb->freeze_transaction_id = 0;
 
        return 0;
 }
@@ -430,24 +482,24 @@ int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
        struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
        struct ctdb_db_context *ctdb_db;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+       ctdb_db = find_ctdb_db(ctdb, w.db_id);
+       if (!ctdb_db) {
+               DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
                return -1;
        }
 
-       if (!ctdb->freeze_handle->transaction_started) {
-               DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
+       if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+               DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
                return -1;
        }
 
-       if (w.transaction_id != ctdb->freeze_handle->transaction_id) {
-               DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
+       if (!ctdb->freeze_transaction_started) {
+               DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
                return -1;
        }
 
-       ctdb_db = find_ctdb_db(ctdb, w.db_id);
-       if (!ctdb_db) {
-               DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
+       if (w.transaction_id != ctdb->freeze_transaction_id) {
+               DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
                return -1;
        }
 

diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index 8044c4ed2a1e064746170611c378794f9aa48e2b..49686eb51ae978eaf4e3d9d2554715450a33b7e9 100644 (file)
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -503,6 +503,11 @@ int32_t ctdb_control_set_db_priority(struct ctdb_context *ctdb, TDB_DATA indata)
                return -1;
        }
 
+       if ((db_prio->priority<1) || (db_prio->priority>NUM_DB_PRIORITIES)) {
+               DEBUG(DEBUG_ERR,("Trying to set invalid priority : %u\n", db_prio->priority));
+               return -1;
+       }
+
        ctdb_db->priority = db_prio->priority;
        DEBUG(DEBUG_INFO,("Setting DB priority to %u for db 0x%08x\n", db_prio->priority, db_prio->db_id));
 

diff --git a/server/ctdb_monitor.c b/server/ctdb_monitor.c
index 14da3b77e4bc2ecce4eedd3956076eb332721f32..486042346dea01908d2a46c9e6d75273c69b6a12 100644 (file)
--- a/server/ctdb_monitor.c
+++ b/server/ctdb_monitor.c
@@ -285,7 +285,8 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 {
        struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)indata.dptr;
        struct ctdb_node *node;
-       
+       int i;
+
        if (c->pnn >= ctdb->num_nodes) {
                DEBUG(DEBUG_ERR,(__location__ " Node %d is invalid, num_nodes :%d\n", c->pnn, ctdb->num_nodes));
                return -1;
@@ -319,7 +320,11 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
                */
                ctdb->vnn_map->generation = INVALID_GENERATION;
 
-               ctdb_start_freeze(ctdb);
+               for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+                       if (ctdb_start_freeze(ctdb, i) != 0) {
+                               DEBUG(DEBUG_ERR,(__location__ " Failed to freeze db priority %u\n", i));
+                       }
+               }
                ctdb_release_all_ips(ctdb);
                ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
        }

diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index b4428fa4946e9edaf6974a3081fe5ef973742fdb..f4e30921e7177a0bcab9a73a6b5c0a832a0257c8 100644 (file)
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -31,14 +31,23 @@
 /*
   lock all databases - mark only
  */
-static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t priority)
 {
        struct ctdb_db_context *ctdb_db;
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+
+       if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+               DEBUG(DEBUG_ERR,(__location__ " Illegal priority when trying to mark all databases Prio:%u\n", priority));
+               return -1;
+       }
+
+       if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
                DEBUG(DEBUG_ERR,("Attempt to mark all databases locked when not frozen\n"));
                return -1;
        }
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (ctdb_db->priority != priority) {
+                       continue;
+               }
                if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
                }
@@ -49,14 +58,23 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
 /*
   lock all databases - unmark only
  */
-static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb, uint32_t priority)
 {
        struct ctdb_db_context *ctdb_db;
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+
+       if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+               DEBUG(DEBUG_ERR,(__location__ " Illegal priority when trying to mark all databases Prio:%u\n", priority));
+               return -1;
+       }
+
+       if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
                DEBUG(DEBUG_ERR,("Attempt to unmark all databases locked when not frozen\n"));
                return -1;
        }
        for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (ctdb_db->priority != priority) {
+                       continue;
+               }
                if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
                        return -1;
                }
@@ -90,10 +108,13 @@ int
 ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
 {
        struct ctdb_vnn_map_wire *map = (struct ctdb_vnn_map_wire *)indata.dptr;
+       int i;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_ERR,("Attempt to set vnnmap when not frozen\n"));
-               return -1;
+       for(i=1; i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+                       DEBUG(DEBUG_ERR,("Attempt to set vnnmap when not frozen\n"));
+                       return -1;
+               }
        }
 
        talloc_free(ctdb->vnn_map);
@@ -315,11 +336,6 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
        struct pulldb_data params;
        struct ctdb_marshall_buffer *reply;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_pull_db when not frozen\n"));
-               return -1;
-       }
-
        pull = (struct ctdb_control_pulldb *)indata.dptr;
        
        ctdb_db = find_ctdb_db(ctdb, pull->db_id);
@@ -328,6 +344,11 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
                return -1;
        }
 
+       if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_pull_db when not frozen\n"));
+               return -1;
+       }
+
        reply = talloc_zero(outdata, struct ctdb_marshall_buffer);
        CTDB_NO_MEMORY(ctdb, reply);
 
@@ -338,19 +359,19 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
        params.len = offsetof(struct ctdb_marshall_buffer, data);
        params.failed = false;
 
-       if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+       if (ctdb_lock_all_databases_mark(ctdb, ctdb_db->priority) != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
                return -1;
        }
 
        if (tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_pulldb, &params) == -1) {
                DEBUG(DEBUG_ERR,(__location__ " Failed to get traverse db '%s'\n", ctdb_db->db_name));
-               ctdb_lock_all_databases_unmark(ctdb);
+               ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
                talloc_free(params.pulldata);
                return -1;
        }
 
-       ctdb_lock_all_databases_unmark(ctdb);
+       ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 
        outdata->dptr = (uint8_t *)params.pulldata;
        outdata->dsize = params.len;
@@ -368,11 +389,6 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
        int i, ret;
        struct ctdb_rec_data *rec;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_push_db when not frozen\n"));
-               return -1;
-       }
-
        if (indata.dsize < offsetof(struct ctdb_marshall_buffer, data)) {
                DEBUG(DEBUG_ERR,(__location__ " invalid data in pulldb reply\n"));
                return -1;
@@ -384,7 +400,12 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
                return -1;
        }
 
-       if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+       if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_push_db when not frozen\n"));
+               return -1;
+       }
+
+       if (ctdb_lock_all_databases_mark(ctdb, ctdb_db->priority) != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
                return -1;
        }
@@ -423,11 +444,11 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
        DEBUG(DEBUG_DEBUG,("finished push of %u records for dbid 0x%x\n",
                 reply->count, reply->db_id));
 
-       ctdb_lock_all_databases_unmark(ctdb);
+       ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
        return 0;
 
 failed:
-       ctdb_lock_all_databases_unmark(ctdb);
+       ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
        return -1;
 }
 
@@ -461,25 +482,25 @@ int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata)
        struct ctdb_control_set_dmaster *p = (struct ctdb_control_set_dmaster *)indata.dptr;
        struct ctdb_db_context *ctdb_db;
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_set_dmaster when not frozen\n"));
-               return -1;
-       }
-
        ctdb_db = find_ctdb_db(ctdb, p->db_id);
        if (!ctdb_db) {
                DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", p->db_id));
                return -1;
        }
 
-       if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+       if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+               DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_set_dmaster when not frozen\n"));
+               return -1;
+       }
+
+       if (ctdb_lock_all_databases_mark(ctdb,  ctdb_db->priority) != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
                return -1;
        }
 
        tdb_traverse(ctdb_db->ltdb->tdb, traverse_setdmaster, &p->dmaster);
 
-       ctdb_lock_all_databases_unmark(ctdb);
+       ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
        
        return 0;
 }
@@ -590,7 +611,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
                                 const char **errormsg)
 {
        uint32_t recmode = *(uint32_t *)indata.dptr;
-       int ret;
+       int i, ret;
        struct ctdb_set_recmode_state *state;
        pid_t parent = getpid();
 
@@ -609,11 +630,13 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
        }
 
 
-       if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-               DEBUG(DEBUG_ERR,("Attempt to change recovery mode to %u when not frozen\n", 
-                        recmode));
-               (*errormsg) = "Cannot change recovery mode while not frozen";
-               return -1;
+       for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+                       DEBUG(DEBUG_ERR,("Attempt to change recovery mode to %u when not frozen\n", 
+                                recmode));
+                       (*errormsg) = "Cannot change recovery mode while not frozen";
+                       return -1;
+               }
        }
 
        if (recmode != ctdb->recovery_mode) {
@@ -629,9 +652,11 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
 
        /* some special handling when ending recovery mode */
 
-       /* force the databased to thaw */
-       if (ctdb->freeze_handle) {
-               ctdb_control_thaw(ctdb);
+       /* force the databases to thaw */
+       for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+               if (ctdb->freeze_handles[i] != NULL) {
+                       ctdb_control_thaw(ctdb, i);
+               }
        }
 
        state = talloc(ctdb, struct ctdb_set_recmode_state);

diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index e78e1741f0df4e2257fc54c63b05a1ebe6c40bd0..512a37a55bcaf276e021d0dec1d90a0f8c455a0f 100644 (file)
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -212,7 +212,7 @@ static int run_recovered_eventscript(struct ctdb_context *ctdb, struct ctdb_node
 
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_END_RECOVERY,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, tdb_null,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -288,7 +288,7 @@ static int run_startrecovery_eventscript(struct ctdb_recoverd *rec, struct ctdb_
 
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_START_RECOVERY,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, tdb_null,
                                        NULL,
                                        startrecovery_fail_callback,
@@ -326,7 +326,8 @@ static int update_capabilities(struct ctdb_context *ctdb, struct ctdb_node_map *
 
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_CAPABILITIES,
-                                       nodes, CONTROL_TIMEOUT(),
+                                       nodes, 0,
+                                       CONTROL_TIMEOUT(),
                                        false, tdb_null,
                                        async_getcap_callback, NULL,
                                        NULL) != 0) {
@@ -362,15 +363,20 @@ static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_recoverd *re
        /* freeze all nodes */
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (rec_mode == CTDB_RECOVERY_ACTIVE) {
-               if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
-                                               nodes, CONTROL_TIMEOUT(),
+               int i;
+
+               for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+                       if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
+                                               nodes, i,
+                                               CONTROL_TIMEOUT(),
                                                false, tdb_null,
                                                NULL,
                                                set_recmode_fail_callback,
                                                rec) != 0) {
-                       DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
-                       talloc_free(tmp_ctx);
-                       return -1;
+                               DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
+                               talloc_free(tmp_ctx);
+                               return -1;
+                       }
                }
        }
 
@@ -379,7 +385,8 @@ static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_recoverd *re
        data.dptr = (unsigned char *)&rec_mode;
 
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMODE,
-                                       nodes, CONTROL_TIMEOUT(),
+                                       nodes, 0,
+                                       CONTROL_TIMEOUT(),
                                        false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -409,7 +416,7 @@ static int set_recovery_master(struct ctdb_context *ctdb, struct ctdb_node_map *
 
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMASTER,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -456,7 +463,7 @@ static int update_db_priority_on_remote_nodes(struct ctdb_context *ctdb,
 
                if (ctdb_client_async_control(ctdb,
                                        CTDB_CONTROL_SET_DB_PRIORITY,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -1274,7 +1281,7 @@ static int push_recdb_database(struct ctdb_context *ctdb, uint32_t dbid,
 
        nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, outdata,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -1334,7 +1341,7 @@ static int recover_database(struct ctdb_recoverd *rec,
 
        nodes = list_of_active_nodes(ctdb, nodemap, recdb, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -1489,7 +1496,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
        nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -1510,7 +1517,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
        /* commit all the changes */
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, data,
                                        NULL, NULL,
                                        NULL) != 0) {
@@ -2368,7 +2375,7 @@ static int get_remote_nodemaps(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
 
        nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
        if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_NODEMAP,
-                                       nodes,
+                                       nodes, 0,
                                        CONTROL_TIMEOUT(), false, tdb_null,
                                        async_getnodemap_callback,
                                        NULL,