server/ctdb_freeze.c

   1 /*
   2    ctdb freeze handling
   3
   4    Copyright (C) Andrew Tridgell  2007
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19 #include "includes.h"
  20 #include "lib/tevent/tevent.h"
  21 #include "lib/tdb/include/tdb.h"
  22 #include "system/network.h"
  23 #include "system/filesys.h"
  24 #include "system/wait.h"
  25 #include "../include/ctdb_private.h"
  26 #include "lib/util/dlinklist.h"
  27 #include "db_wrap.h"
  28
  29 static bool later_db(const char *name)
  30 {
  31         return (strstr(name, "notify") || strstr(name, "serverid"));
  32 }
  33
  34 /*
  35   lock all databases
  36  */
  37 static int ctdb_lock_all_databases(struct ctdb_context *ctdb, uint32_t priority)
  38 {
  39         struct ctdb_db_context *ctdb_db;
  40         /* REMOVE later */
  41         /* This double loop is for backward compatibility and deadlock
  42            avoidance for old samba versions that not yet support
  43            the set prio call.
  44            This code shall be removed later
  45         */
  46         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  47                 if (ctdb_db->priority != priority) {
  48                         continue;
  49                 }
  50                 if (later_db(ctdb_db->db_name)) {
  51                         continue;
  52                 }
  53                 DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
  54                 if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
  55                         DEBUG(DEBUG_ERR,(__location__ " Failed to lock database %s\n", ctdb_db->db_name));
  56                         return -1;
  57                 }
  58         }
  59         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
  60                 if (ctdb_db->priority != priority) {
  61                         continue;
  62                 }
  63                 if (!later_db(ctdb_db->db_name)) {
  64                         continue;
  65                 }
  66                 DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
  67                 if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
  68                         DEBUG(DEBUG_ERR,(__location__ " Failed to lock database %s\n", ctdb_db->db_name));
  69                         return -1;
  70                 }
  71         }
  72         return 0;
  73 }
  74
  75 /*
  76   a list of control requests waiting for a freeze lock child to get
  77   the database locks
  78  */
  79 struct ctdb_freeze_waiter {
  80         struct ctdb_freeze_waiter *next, *prev;
  81         struct ctdb_context *ctdb;
  82         struct ctdb_req_control *c;
  83         uint32_t priority;
  84         int32_t status;
  85 };
  86
  87 /* a handle to a freeze lock child process */
  88 struct ctdb_freeze_handle {
  89         struct ctdb_context *ctdb;
  90         uint32_t priority;
  91         pid_t child;
  92         int fd;
  93         struct ctdb_freeze_waiter *waiters;
  94 };
  95
  96 /*
  97   destroy a freeze handle
  98  */
  99 static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
 100 {
 101         struct ctdb_context *ctdb = h->ctdb;
 102         struct ctdb_db_context *ctdb_db;
 103
 104         DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
 105
 106         /* cancel any pending transactions */
 107         if (ctdb->freeze_transaction_started) {
 108                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 109                         if (ctdb_db->priority != h->priority) {
 110                                 continue;
 111                         }
 112                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 113                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 114                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 115                                          ctdb_db->db_name));
 116                         }
 117                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 118                 }
 119                 ctdb->freeze_transaction_started = false;
 120         }
 121
 122         ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
 123         ctdb->freeze_handles[h->priority] = NULL;
 124
 125         kill(h->child, SIGKILL);
 126         return 0;
 127 }
 128
 129 /*
 130   called when the child writes its status to us
 131  */
 132 static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *fde,
 133                                        uint16_t flags, void *private_data)
 134 {
 135         struct ctdb_freeze_handle *h = talloc_get_type(private_data, struct ctdb_freeze_handle);
 136         int32_t status;
 137         struct ctdb_freeze_waiter *w;
 138
 139         if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
 140                 DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
 141                 talloc_free(h);
 142                 return;
 143         }
 144
 145         if (read(h->fd, &status, sizeof(status)) != sizeof(status)) {
 146                 DEBUG(DEBUG_ERR,("read error from freeze lock child\n"));
 147                 status = -1;
 148         }
 149
 150         if (status == -1) {
 151                 DEBUG(DEBUG_ERR,("Failed to get locks in ctdb_freeze_child\n"));
 152                 /* we didn't get the locks - destroy the handle */
 153                 talloc_free(h);
 154                 return;
 155         }
 156
 157         h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 158
 159         /* notify the waiters */
 160         if (h != h->ctdb->freeze_handles[h->priority]) {
 161                 DEBUG(DEBUG_ERR,("lockwait finished but h is not linked\n"));
 162         }
 163         while ((w = h->waiters)) {
 164                 w->status = status;
 165                 DLIST_REMOVE(h->waiters, w);
 166                 talloc_free(w);
 167         }
 168 }
 169
 170 /*
 171   create a child which gets locks on all the open databases, then calls the callback telling the parent
 172   that it is done
 173  */
 174 static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, uint32_t priority)
 175 {
 176         struct ctdb_freeze_handle *h;
 177         int fd[2];
 178         struct fd_event *fde;
 179
 180         h = talloc_zero(ctdb, struct ctdb_freeze_handle);
 181         CTDB_NO_MEMORY_NULL(ctdb, h);
 182
 183         h->ctdb     = ctdb;
 184         h->priority = priority;
 185
 186         if (pipe(fd) == -1) {
 187                 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
 188                 talloc_free(h);
 189                 return NULL;
 190         }
 191
 192         h->child = fork();
 193         if (h->child == -1) {
 194                 DEBUG(DEBUG_ERR,("Failed to fork child for ctdb_freeze_lock\n"));
 195                 talloc_free(h);
 196                 return NULL;
 197         }
 198
 199         if (h->child == 0) {
 200                 int ret;
 201
 202                 /* in the child */
 203                 close(fd[0]);
 204
 205                 ret = ctdb_lock_all_databases(ctdb, priority);
 206                 if (ret != 0) {
 207                         _exit(0);
 208                 }
 209
 210                 ret = write(fd[1], &ret, sizeof(ret));
 211                 if (ret != sizeof(ret)) {
 212                         DEBUG(DEBUG_ERR, (__location__ " Failed to write to socket from freeze child. ret:%d errno:%u\n", ret, errno));
 213                         _exit(1);
 214                 }
 215
 216                 while (1) {
 217                         sleep(1);
 218                         if (kill(ctdb->ctdbd_pid, 0) != 0) {
 219                                 DEBUG(DEBUG_ERR,("Parent died. Exiting lock wait child\n"));
 220
 221                                 _exit(0);
 222                         }
 223                 }
 224         }
 225
 226         talloc_set_destructor(h, ctdb_freeze_handle_destructor);
 227
 228         close(fd[1]);
 229         set_close_on_exec(fd[0]);
 230
 231         h->fd = fd[0];
 232
 233
 234         fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ,
 235                            ctdb_freeze_lock_handler, h);
 236         if (fde == NULL) {
 237                 DEBUG(DEBUG_ERR,("Failed to setup fd event for ctdb_freeze_lock\n"));
 238                 close(fd[0]);
 239                 talloc_free(h);
 240                 return NULL;
 241         }
 242         tevent_fd_set_auto_close(fde);
 243
 244         return h;
 245 }
 246
 247 /*
 248   destroy a waiter for a freeze mode change
 249  */
 250 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 251 {
 252         ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
 253         return 0;
 254 }
 255
 256 /*
 257   start the freeze process for a certain priority
 258  */
 259 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 260 {
 261         if (priority == 0) {
 262                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 263                 priority = 1;
 264         }
 265
 266         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 267                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 268                 return -1;
 269         }
 270
 271         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 272                 /* we're already frozen */
 273                 return 0;
 274         }
 275
 276         /* if there isn't a freeze lock child then create one */
 277         if (ctdb->freeze_handles[priority] == NULL) {
 278                 ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
 279                 CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
 280                 ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
 281         }
 282
 283         return 0;
 284 }
 285
 286 /*
 287   freeze the databases
 288  */
 289 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
 290 {
 291         struct ctdb_freeze_waiter *w;
 292         uint32_t priority;
 293
 294         priority = (uint32_t)c->srvid;
 295
 296         DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
 297
 298         if (priority == 0) {
 299                 DEBUG(DEBUG_ERR,("Freeze priority 0 requested, remapping to priority 1\n"));
 300                 priority = 1;
 301         }
 302
 303         if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
 304                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 305                 return -1;
 306         }
 307
 308         if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 309                 /* we're already frozen */
 310                 return 0;
 311         }
 312
 313         if (ctdb_start_freeze(ctdb, priority) != 0) {
 314                 DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
 315                 return -1;
 316         }
 317
 318         /* add ourselves to list of waiters */
 319         if (ctdb->freeze_handles[priority] == NULL) {
 320                 DEBUG(DEBUG_ERR,("No freeze lock handle when adding a waiter\n"));
 321                 return -1;
 322         }
 323
 324         w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
 325         CTDB_NO_MEMORY(ctdb, w);
 326         w->ctdb     = ctdb;
 327         w->c        = talloc_steal(w, c);
 328         w->priority = priority;
 329         w->status   = -1;
 330         talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
 331         DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 332
 333         /* we won't reply till later */
 334         *async_reply = True;
 335         return 0;
 336 }
 337
 338
 339 /*
 340   block until we are frozen, used during daemon startup
 341  */
 342 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 343 {
 344         int i;
 345
 346         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
 347                 if (ctdb_start_freeze(ctdb, i)) {
 348                         DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
 349                         continue;
 350                 }
 351
 352                 /* block until frozen */
 353                 while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
 354                         event_loop_once(ctdb->ev);
 355                 }
 356         }
 357
 358         return 0;
 359 }
 360
 361
 362 static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 363 {
 364         DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 365
 366         /* cancel any pending transactions */
 367         if (ctdb->freeze_transaction_started) {
 368                 struct ctdb_db_context *ctdb_db;
 369
 370                 for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 371                         tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 372                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 373                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 374                                          ctdb_db->db_name));
 375                         }
 376                         tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 377                 }
 378         }
 379         ctdb->freeze_transaction_started = false;
 380
 381 #if 0
 382         /* this hack can be used to get a copy of the databases at the end of a recovery */
 383         system("mkdir -p /var/ctdb.saved; /usr/bin/rsync --delete -a /var/ctdb/ /var/ctdb.saved/$$ 2>&1 > /dev/null");
 384 #endif
 385
 386 #if 0
 387         /* and this one for local testing */
 388         system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 389 #endif
 390
 391         if (ctdb->freeze_handles[priority] != NULL) {
 392                 talloc_free(ctdb->freeze_handles[priority]);
 393                 ctdb->freeze_handles[priority] = NULL;
 394         }
 395 }
 396
 397 /*
 398   thaw the databases
 399  */
 400 int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
 401 {
 402
 403         if (priority > NUM_DB_PRIORITIES) {
 404                 DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
 405                 return -1;
 406         }
 407
 408         if (priority == 0) {
 409                 int i;
 410                 for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 411                         thaw_priority(ctdb, i);
 412                 }
 413         } else {
 414                 thaw_priority(ctdb, priority);
 415         }
 416
 417         ctdb_call_resend_all(ctdb);
 418         return 0;
 419 }
 420
 421
 422 /*
 423   start a transaction on all databases - used for recovery
 424  */
 425 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 426 {
 427         struct ctdb_db_context *ctdb_db;
 428         int i;
 429
 430         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 431                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 432                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 433                         return -1;
 434                 }
 435         }
 436
 437         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 438                 int ret;
 439
 440                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 441
 442                 if (ctdb->freeze_transaction_started) {
 443                         if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 444                                 DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 445                                          ctdb_db->db_name));
 446                                 /* not a fatal error */
 447                         }
 448                 }
 449
 450                 ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
 451
 452                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 453
 454                 if (ret != 0) {
 455                         DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction for db '%s'\n",
 456                                  ctdb_db->db_name));
 457                         return -1;
 458                 }
 459         }
 460
 461         ctdb->freeze_transaction_started = true;
 462         ctdb->freeze_transaction_id = id;
 463
 464         return 0;
 465 }
 466
 467 /*
 468   cancel a transaction for all databases - used for recovery
 469  */
 470 int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb)
 471 {
 472         struct ctdb_db_context *ctdb_db;
 473
 474         DEBUG(DEBUG_ERR,(__location__ " recovery transaction cancelled called\n"));
 475
 476         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 477                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 478
 479                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 480                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",  ctdb_db->db_name));
 481                         /* not a fatal error */
 482                 }
 483
 484                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 485         }
 486
 487         ctdb->freeze_transaction_started = false;
 488
 489         return 0;
 490 }
 491
 492 /*
 493   commit transactions on all databases
 494  */
 495 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 496 {
 497         struct ctdb_db_context *ctdb_db;
 498         int i;
 499         int healthy_nodes = 0;
 500
 501         for (i=1;i<=NUM_DB_PRIORITIES; i++) {
 502                 if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
 503                         DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 504                         return -1;
 505                 }
 506         }
 507
 508         if (!ctdb->freeze_transaction_started) {
 509                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 510                 return -1;
 511         }
 512
 513         if (id != ctdb->freeze_transaction_id) {
 514                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
 515                 return -1;
 516         }
 517
 518         DEBUG(DEBUG_DEBUG,(__location__ " num_nodes[%d]\n", ctdb->num_nodes));
 519         for (i=0; i < ctdb->num_nodes; i++) {
 520                 DEBUG(DEBUG_DEBUG,(__location__ " node[%d].flags[0x%X]\n",
 521                                    i, ctdb->nodes[i]->flags));
 522                 if (ctdb->nodes[i]->flags == 0) {
 523                         healthy_nodes++;
 524                 }
 525         }
 526         DEBUG(DEBUG_INFO,(__location__ " healthy_nodes[%d]\n", healthy_nodes));
 527
 528         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 529                 int ret;
 530
 531                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 532                 ret = tdb_transaction_commit(ctdb_db->ltdb->tdb);
 533                 if (ret != 0) {
 534                         DEBUG(DEBUG_ERR,(__location__ " Failed to commit transaction for db '%s'. Cancel all transactions and resetting transaction_started to false.\n",
 535                                  ctdb_db->db_name));
 536                         goto fail;
 537                 }
 538                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 539
 540                 ret = ctdb_update_persistent_health(ctdb, ctdb_db, NULL, healthy_nodes);
 541                 if (ret != 0) {
 542                         DEBUG(DEBUG_CRIT,(__location__ " Failed to update persistent health for db '%s'. "
 543                                          "Cancel all remaining transactions and resetting transaction_started to false.\n",
 544                                          ctdb_db->db_name));
 545                         goto fail;
 546                 }
 547         }
 548
 549         ctdb->freeze_transaction_started = false;
 550         ctdb->freeze_transaction_id = 0;
 551
 552         return 0;
 553
 554 fail:
 555         /* cancel any pending transactions */
 556         for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 557                 tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 558                 if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 559                         DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 560                                  ctdb_db->db_name));
 561                 }
 562                 tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 563         }
 564         ctdb->freeze_transaction_started = false;
 565
 566         return -1;
 567 }
 568
 569 /*
 570   wipe a database - only possible when in a frozen transaction
 571  */
 572 int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
 573 {
 574         struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
 575         struct ctdb_db_context *ctdb_db;
 576
 577         ctdb_db = find_ctdb_db(ctdb, w.db_id);
 578         if (!ctdb_db) {
 579                 DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
 580                 return -1;
 581         }
 582
 583         if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
 584                 DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 585                 return -1;
 586         }
 587
 588         if (!ctdb->freeze_transaction_started) {
 589                 DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 590                 return -1;
 591         }
 592
 593         if (w.transaction_id != ctdb->freeze_transaction_id) {
 594                 DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
 595                 return -1;
 596         }
 597
 598         if (tdb_wipe_all(ctdb_db->ltdb->tdb) != 0) {
 599                 DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database for db '%s'\n",
 600                          ctdb_db->db_name));
 601                 return -1;
 602         }
 603
 604         return 0;
 605 }