4 Copyright (C) Andrew Tridgell 2007
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>.
20 #include "lib/tdb/include/tdb.h"
21 #include "system/network.h"
22 #include "system/filesys.h"
23 #include "system/wait.h"
24 #include "../include/ctdb_private.h"
25 #include "lib/util/dlinklist.h"
27 #include "../common/rb_tree.h"
30 a list of control requests waiting for a freeze lock child to get
33 struct ctdb_freeze_waiter {
34 struct ctdb_freeze_waiter *next, *prev;
35 struct ctdb_context *ctdb;
36 struct ctdb_req_control *c;
41 /* a handle to a freeze lock child process */
42 struct ctdb_freeze_handle {
43 struct ctdb_context *ctdb;
45 struct lock_request *lreq;
46 struct ctdb_freeze_waiter *waiters;
50 destroy a freeze handle
52 static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
54 struct ctdb_context *ctdb = h->ctdb;
55 struct ctdb_db_context *ctdb_db;
57 DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
59 /* cancel any pending transactions */
60 if (ctdb->freeze_transaction_started) {
61 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
62 if (ctdb_db->priority != h->priority) {
65 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
66 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
67 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
70 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
72 ctdb->freeze_transaction_started = false;
75 ctdb->freeze_mode[h->priority] = CTDB_FREEZE_NONE;
76 ctdb->freeze_handles[h->priority] = NULL;
78 ctdb_lock_free_request_context(h->lreq);
83 called when the child writes its status to us
85 static void ctdb_freeze_lock_handler(void *private_data, bool locked)
87 struct ctdb_freeze_handle *h = talloc_get_type_abort(private_data,
88 struct ctdb_freeze_handle);
89 struct ctdb_freeze_waiter *w;
91 if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
92 DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
98 DEBUG(DEBUG_ERR,("Failed to get locks in ctdb_freeze_child\n"));
99 /* we didn't get the locks - destroy the handle */
104 h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
106 /* notify the waiters */
107 if (h != h->ctdb->freeze_handles[h->priority]) {
108 DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
110 while ((w = h->waiters)) {
112 DLIST_REMOVE(h->waiters, w);
118 destroy a waiter for a freeze mode change
120 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
122 ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
127 start the freeze process for a certain priority
129 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
131 struct ctdb_freeze_handle *h;
134 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
138 if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
139 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
143 if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
144 /* we're already frozen */
148 /* Stop any vacuuming going on: we don't want to wait. */
149 ctdb_stop_vacuuming(ctdb);
151 /* if there isn't a freeze lock child then create one */
152 if (ctdb->freeze_handles[priority] == NULL) {
153 h = talloc_zero(ctdb, struct ctdb_freeze_handle);
154 CTDB_NO_MEMORY(ctdb, h);
156 h->priority = priority;
157 talloc_set_destructor(h, ctdb_freeze_handle_destructor);
159 h->lreq = ctdb_lock_alldb_prio(ctdb, priority, false, ctdb_freeze_lock_handler, h);
160 CTDB_NO_MEMORY(ctdb, h->lreq);
161 ctdb->freeze_handles[priority] = h;
162 ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
171 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
173 struct ctdb_freeze_waiter *w;
176 priority = (uint32_t)c->srvid;
178 DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
181 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
185 if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
186 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
190 if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
191 /* we're already frozen */
195 if (ctdb_start_freeze(ctdb, priority) != 0) {
196 DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
200 /* add ourselves to list of waiters */
201 if (ctdb->freeze_handles[priority] == NULL) {
202 DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
206 w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
207 CTDB_NO_MEMORY(ctdb, w);
209 w->c = talloc_steal(w, c);
210 w->priority = priority;
212 talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
213 DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
215 /* we won't reply till later */
222 block until we are frozen, used during daemon startup
224 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
228 for (i=1; i<=NUM_DB_PRIORITIES; i++) {
229 if (ctdb_start_freeze(ctdb, i)) {
230 DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
234 /* block until frozen */
235 while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
236 event_loop_once(ctdb->ev);
244 static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
246 DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
248 /* cancel any pending transactions */
249 if (ctdb->freeze_transaction_started) {
250 struct ctdb_db_context *ctdb_db;
252 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
253 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
254 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
255 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
258 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
261 ctdb->freeze_transaction_started = false;
264 /* this hack can be used to get a copy of the databases at the end of a recovery */
265 system("mkdir -p /var/ctdb.saved; /usr/bin/rsync --delete -a /var/ctdb/ /var/ctdb.saved/$$ 2>&1 > /dev/null");
269 /* and this one for local testing */
270 system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
273 if (ctdb->freeze_handles[priority] != NULL) {
274 talloc_free(ctdb->freeze_handles[priority]);
275 ctdb->freeze_handles[priority] = NULL;
282 int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
285 if (priority > NUM_DB_PRIORITIES) {
286 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
292 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
293 thaw_priority(ctdb, i);
296 thaw_priority(ctdb, priority);
299 ctdb_call_resend_all(ctdb);
305 start a transaction on all databases - used for recovery
307 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
309 struct ctdb_db_context *ctdb_db;
312 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
313 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
314 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
319 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
322 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
324 if (ctdb->freeze_transaction_started) {
325 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
326 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
328 /* not a fatal error */
332 ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
334 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
337 DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction for db '%s'\n",
343 ctdb->freeze_transaction_started = true;
344 ctdb->freeze_transaction_id = id;
350 cancel a transaction for all databases - used for recovery
352 int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
354 struct ctdb_db_context *ctdb_db;
356 DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
358 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
359 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
361 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
362 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n", ctdb_db->db_name));
363 /* not a fatal error */
366 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
369 ctdb->freeze_transaction_started = false;
375 commit transactions on all databases
377 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
379 struct ctdb_db_context *ctdb_db;
381 int healthy_nodes = 0;
383 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
384 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
385 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
390 if (!ctdb->freeze_transaction_started) {
391 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
395 if (id != ctdb->freeze_transaction_id) {
396 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
400 DEBUG(DEBUG_DEBUG,(__location__ " num_nodes[%d]\n", ctdb->num_nodes));
401 for (i=0; i < ctdb->num_nodes; i++) {
402 DEBUG(DEBUG_DEBUG,(__location__ " node[%d].flags[0x%X]\n",
403 i, ctdb->nodes[i]->flags));
404 if (ctdb->nodes[i]->flags == 0) {
408 DEBUG(DEBUG_INFO,(__location__ " healthy_nodes[%d]\n", healthy_nodes));
410 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
413 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
414 ret = tdb_transaction_commit(ctdb_db->ltdb->tdb);
416 DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
420 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
422 ret = ctdb_update_persistent_health(ctdb, ctdb_db, NULL, healthy_nodes);
424 DEBUG(DEBUG_CRIT,(__location__ " Failed to update persistent health for db '%s'. "
425 "Cancel all remaining transactions and resetting transaction_started to false.\n",
431 ctdb->freeze_transaction_started = false;
432 ctdb->freeze_transaction_id = 0;
437 /* cancel any pending transactions */
438 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
439 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
440 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
441 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
444 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
446 ctdb->freeze_transaction_started = false;
452 wipe a database - only possible when in a frozen transaction
454 int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
456 struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
457 struct ctdb_db_context *ctdb_db;
459 ctdb_db = find_ctdb_db(ctdb, w.db_id);
461 DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
465 if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
466 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
470 if (!ctdb->freeze_transaction_started) {
471 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
475 if (w.transaction_id != ctdb->freeze_transaction_id) {
476 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
480 if (tdb_wipe_all(ctdb_db->ltdb->tdb) != 0) {
481 DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database for db '%s'\n",
486 if (!ctdb_db->persistent) {
487 talloc_free(ctdb_db->delete_queue);
488 ctdb_db->delete_queue = trbt_create(ctdb_db, 0);
489 if (ctdb_db->delete_queue == NULL) {
490 DEBUG(DEBUG_ERR, (__location__ " Failed to re-create "
491 "the vacuum tree.\n"));