server/ctdb_freeze.c

   1 /*
   2    ctdb freeze handling
   3
   4    Copyright (C) Andrew Tridgell  2007
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19 #include "includes.h"
  20 #include "lib/tdb/include/tdb.h"
  21 #include "system/network.h"
  22 #include "system/filesys.h"
  23 #include "system/wait.h"
  24 #include "../include/ctdb_private.h"
  25 #include "lib/util/dlinklist.h"
  26 #include "db_wrap.h"
  27 #include "../common/rb_tree.h"
  28
  29 /*
  30   a list of control requests waiting for a freeze lock child to get
  31   the database locks
  32  */
  33 struct ctdb_freeze_waiter {
  34         struct ctdb_freeze_waiter *next, *prev;
  35         struct ctdb_context *ctdb;
  36         struct ctdb_req_control *c;
  37         uint32_t priority;
  38         int32_t status;
  39 };
  40
  41 /* a handle to a freeze lock child process */
  42 struct ctdb_freeze_handle {
  43         struct ctdb_context *ctdb;
  44         uint32_t priority;
  45         struct lock_request *lreq;
  46         struct ctdb_freeze_waiter *waiters;
  47 };
  48
  49 /*
  50   destroy a freeze handle
  51  */
  52 static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
  53 {
  54         struct ctdb_context *ctdb = h->ctdb;
  55         struct ctdb_db_context *ctdb_db;
  56
  57         DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
  58
  59         /* cancel any pending transactions */
  60         if (ctdb->freeze_transaction_started) {
  61                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  62                         if (ctdb_db->priority != h->priority) {
  63                                 continue;
  64                         }
  65                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
  66                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
  67                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
  68                                          ctdb_db->db_name));
  69                         }
  70                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
  71                 }
  72                 ctdb->freeze_transaction_started = false;
  73         }
  74
  75         ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
  76         ctdb->freeze_handles[h->priority] = NULL;
  77
  78         ctdb_lock_free_request_context(h->lreq);
  79         return 0;
  80 }
  81
  82 /*
  83   called when the child writes its status to us
  84  */
  85 static void ctdb_freeze_lock_handler(void *private_data, bool locked)
  86 {
  87         struct ctdb_freeze_handle *h = talloc_get_type_abort(private_data,
  88                                                              struct ctdb_freeze_handle);
  89         struct ctdb_freeze_waiter *w;
  90
  91         if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
  92                 DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
  93                 talloc_free(h);
  94                 return;
  95         }
  96
  97         if (!locked) {
  98                 DEBUG(DEBUG_ERR,("Failed to get locks in ctdb_freeze_child\n"));
  99                 /* we didn't get the locks - destroy the handle */
 100                 talloc_free(h);
 101                 return;
 102         }
 103
 104         h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 105
 106         /* notify the waiters */
 107         if (h != h->ctdb->freeze_handles[h->priority]) {
 108                 DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
 109         }
 110         while ((w = h->waiters)) {
 111                 w->status = 0;
 112                 DLIST_REMOVE(h->waiters, w);
 113                 talloc_free(w);
 114         }
 115 }
 116
 117 /*
 118   destroy a waiter for a freeze mode change
 119  */
 120 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 121 {
 122         ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
 123         return 0;
 124 }
 125
 126 /*
 127   start the freeze process for a certain priority
 128  */
 129 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 130 {
 131         struct ctdb_freeze_handle *h;
 132
 133         if (priority == 0) {
 134                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 135                 priority = 1;
 136         }
 137
 138         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 139                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 140                 return -1;
 141         }
 142
 143         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 144                 /* we're already frozen */
 145                 return 0;
 146         }
 147
 148         /* Stop any vacuuming going on: we don't want to wait. */
 149         ctdb_stop_vacuuming(ctdb);
 150
 151         /* if there isn't a freeze lock child then create one */
 152         if (ctdb->freeze_handles[priority] == NULL) {
 153                 h = talloc_zero(ctdb, struct ctdb_freeze_handle);
 154                 CTDB_NO_MEMORY(ctdb, h);
 155                 h->ctdb = ctdb;
 156                 h->priority = priority;
 157                 talloc_set_destructor(h, ctdb_freeze_handle_destructor);
 158
 159                 h->lreq = ctdb_lock_alldb_prio(ctdb, priority, false, ctdb_freeze_lock_handler, h);
 160                 CTDB_NO_MEMORY(ctdb, h->lreq);
 161                 ctdb->freeze_handles[priority] = h;
 162                 ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
 163         }
 164
 165         return 0;
 166 }
 167
 168 /*
 169   freeze the databases
 170  */
 171 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
 172 {
 173         struct ctdb_freeze_waiter *w;
 174         uint32_t priority;
 175
 176         priority = (uint32_t)c->srvid;
 177
 178         DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 179
 180         if (priority == 0) {
 181                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 182                 priority = 1;
 183         }
 184
 185         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 186                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 187                 return -1;
 188         }
 189
 190         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 191                 /* we're already frozen */
 192                 return 0;
 193         }
 194
 195         if (ctdb_start_freeze(ctdb, priority) != 0) {
 196                 DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
 197                 return -1;
 198         }
 199
 200         /* add ourselves to list of waiters */
 201         if (ctdb->freeze_handles[priority] == NULL) {
 202                 DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
 203                 return -1;
 204         }
 205
 206         w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
 207         CTDB_NO_MEMORY(ctdb, w);
 208         w->ctdb     = ctdb;
 209         w->c        = talloc_steal(w, c);
 210         w->priority = priority;
 211         w->status   = -1;
 212         talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
 213         DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 214
 215         /* we won't reply till later */
 216         *async_reply = true;
 217         return 0;
 218 }
 219
 220
 221 /*
 222   block until we are frozen, used during daemon startup
 223  */
 224 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 225 {
 226         int i;
 227
 228         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
 229                 if (ctdb_start_freeze(ctdb, i)) {
 230                         DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
 231                         continue;
 232                 }
 233
 234                 /* block until frozen */
 235                 while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
 236                         event_loop_once(ctdb->ev);
 237                 }
 238         }
 239
 240         return true;
 241 }
 242
 243
 244 static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 245 {
 246         DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 247
 248         /* cancel any pending transactions */
 249         if (ctdb->freeze_transaction_started) {
 250                 struct ctdb_db_context *ctdb_db;
 251
 252                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 253                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 254                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 255                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 256                                          ctdb_db->db_name));
 257                         }
 258                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 259                 }
 260         }
 261         ctdb->freeze_transaction_started = false;
 262
 263 #if 0
 264         /* this hack can be used to get a copy of the databases at the end of a recovery */
 265         system("mkdir -p /var/ctdb.saved; /usr/bin/rsync --delete -a /var/ctdb/ /var/ctdb.saved/$$ 2>&1 > /dev/null");
 266 #endif
 267
 268 #if 0
 269         /* and this one for local testing */
 270         system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 271 #endif
 272
 273         if (ctdb->freeze_handles[priority] != NULL) {
 274                 talloc_free(ctdb->freeze_handles[priority]);
 275                 ctdb->freeze_handles[priority] = NULL;
 276         }
 277 }
 278
 279 /*
 280   thaw the databases
 281  */
 282 int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
 283 {
 284
 285         if (priority > NUM_DB_PRIORITIES) {
 286                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 287                 return -1;
 288         }
 289
 290         if (priority == 0) {
 291                 int i;
 292                 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 293                         thaw_priority(ctdb, i);
 294                 }
 295         } else {
 296                 thaw_priority(ctdb, priority);
 297         }
 298
 299         ctdb_call_resend_all(ctdb);
 300         return 0;
 301 }
 302
 303
 304 /*
 305   start a transaction on all databases - used for recovery
 306  */
 307 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 308 {
 309         struct ctdb_db_context *ctdb_db;
 310         int i;
 311
 312         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 313                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 314                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 315                         return -1;
 316                 }
 317         }
 318
 319         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 320                 int ret;
 321
 322                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 323
 324                 if (ctdb->freeze_transaction_started) {
 325                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 326                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 327                                          ctdb_db->db_name));
 328                                 /* not a fatal error */
 329                         }
 330                 }
 331
 332                 ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
 333
 334                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 335
 336                 if (ret != 0) {
 337                         DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction for db '%s'\n",
 338                                  ctdb_db->db_name));
 339                         return -1;
 340                 }
 341         }
 342
 343         ctdb->freeze_transaction_started = true;
 344         ctdb->freeze_transaction_id = id;
 345
 346         return 0;
 347 }
 348
 349 /*
 350   cancel a transaction for all databases - used for recovery
 351  */
 352 int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
 353 {
 354         struct ctdb_db_context *ctdb_db;
 355
 356         DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
 357
 358         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 359                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 360
 361                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 362                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",  ctdb_db->db_name));
 363                         /* not a fatal error */
 364                 }
 365
 366                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 367         }
 368
 369         ctdb->freeze_transaction_started = false;
 370
 371         return 0;
 372 }
 373
 374 /*
 375   commit transactions on all databases
 376  */
 377 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 378 {
 379         struct ctdb_db_context *ctdb_db;
 380         int i;
 381         int healthy_nodes = 0;
 382
 383         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 384                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 385                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 386                         return -1;
 387                 }
 388         }
 389
 390         if (!ctdb->freeze_transaction_started) {
 391                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 392                 return -1;
 393         }
 394
 395         if (id != ctdb->freeze_transaction_id) {
 396                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
 397                 return -1;
 398         }
 399
 400         DEBUG(DEBUG_DEBUG,(__location__ " num_nodes[%d]\n", ctdb->num_nodes));
 401         for (i=0; i < ctdb->num_nodes; i++) {
 402                 DEBUG(DEBUG_DEBUG,(__location__ " node[%d].flags[0x%X]\n",
 403                                    i, ctdb->nodes[i]->flags));
 404                 if (ctdb->nodes[i]->flags == 0) {
 405                         healthy_nodes++;
 406                 }
 407         }
 408         DEBUG(DEBUG_INFO,(__location__ " healthy_nodes[%d]\n", healthy_nodes));
 409
 410         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 411                 int ret;
 412
 413                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 414                 ret = tdb_transaction_commit(ctdb_db->ltdb->tdb);
 415                 if (ret != 0) {
 416                         DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
 417                                  ctdb_db->db_name));
 418                         goto fail;
 419                 }
 420                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 421
 422                 ret = ctdb_update_persistent_health(ctdb, ctdb_db, NULL, healthy_nodes);
 423                 if (ret != 0) {
 424                         DEBUG(DEBUG_CRIT,(__location__ " Failed to update persistent health for db '%s'. "
 425                                          "Cancel all remaining transactions and resetting transaction_started to false.\n",
 426                                          ctdb_db->db_name));
 427                         goto fail;
 428                 }
 429         }
 430
 431         ctdb->freeze_transaction_started = false;
 432         ctdb->freeze_transaction_id = 0;
 433
 434         return 0;
 435
 436 fail:
 437         /* cancel any pending transactions */
 438         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 439                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 440                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 441                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 442                                  ctdb_db->db_name));
 443                 }
 444                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 445         }
 446         ctdb->freeze_transaction_started = false;
 447
 448         return -1;
 449 }
 450
 451 /*
 452   wipe a database - only possible when in a frozen transaction
 453  */
 454 int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
 455 {
 456         struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
 457         struct ctdb_db_context *ctdb_db;
 458
 459         ctdb_db = find_ctdb_db(ctdb, w.db_id);
 460         if (!ctdb_db) {
 461                 DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
 462                 return -1;
 463         }
 464
 465         if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
 466                 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 467                 return -1;
 468         }
 469
 470         if (!ctdb->freeze_transaction_started) {
 471                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 472                 return -1;
 473         }
 474
 475         if (w.transaction_id != ctdb->freeze_transaction_id) {
 476                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
 477                 return -1;
 478         }
 479
 480         if (tdb_wipe_all(ctdb_db->ltdb->tdb) != 0) {
 481                 DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database for db '%s'\n",
 482                          ctdb_db->db_name));
 483                 return -1;
 484         }
 485
 486         if (!ctdb_db->persistent) {
 487                 talloc_free(ctdb_db->delete_queue);
 488                 ctdb_db->delete_queue = trbt_create(ctdb_db, 0);
 489                 if (ctdb_db->delete_queue == NULL) {
 490                         DEBUG(DEBUG_ERR, (__location__ " Failed to re-create "
 491                                           "the vacuum tree.\n"));
 492                         return -1;
 493                 }
 494         }
 495
 496         return 0;
 497 }