server/ctdb_monitor.c

   1 /*
   2    monitoring links to all other nodes to detect dead nodes
   3
   4
   5    Copyright (C) Ronnie Sahlberg 2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "system/filesys.h"
  23 #include "system/wait.h"
  24 #include "../include/ctdb_private.h"
  25
  26 struct ctdb_monitor_state {
  27         uint32_t monitoring_mode;
  28         TALLOC_CTX *monitor_context;
  29         uint32_t next_interval;
  30 };
  31
  32 static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
  33                               struct timeval t, void *private_data);
  34
  35 /*
  36   setup the notification script
  37 */
  38 int ctdb_set_notification_script(struct ctdb_context *ctdb, const char *script)
  39 {
  40         ctdb->notification_script = talloc_strdup(ctdb, script);
  41         CTDB_NO_MEMORY(ctdb, ctdb->notification_script);
  42         return 0;
  43 }
  44
  45 static int ctdb_run_notification_script_child(struct ctdb_context *ctdb, const char *event)
  46 {
  47         struct stat st;
  48         int ret;
  49         char *cmd;
  50
  51         if (stat(ctdb->notification_script, &st) != 0) {
  52                 DEBUG(DEBUG_ERR,("Could not stat notification script %s. Can not send notifications.\n", ctdb->notification_script));
  53                 return -1;
  54         }
  55         if (!(st.st_mode & S_IXUSR)) {
  56                 DEBUG(DEBUG_ERR,("Notification script %s is not executable.\n", ctdb->notification_script));
  57                 return -1;
  58         }
  59
  60         cmd = talloc_asprintf(ctdb, "%s %s\n", ctdb->notification_script, event);
  61         CTDB_NO_MEMORY(ctdb, cmd);
  62
  63         ret = system(cmd);
  64         /* if the system() call was successful, translate ret into the
  65            return code from the command
  66         */
  67         if (ret != -1) {
  68                 ret = WEXITSTATUS(ret);
  69         }
  70         if (ret != 0) {
  71                 DEBUG(DEBUG_ERR,("Notification script \"%s\" failed with error %d\n", cmd, ret));
  72         }
  73
  74         return ret;
  75 }
  76
  77 void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event)
  78 {
  79         pid_t child;
  80
  81         if (ctdb->notification_script == NULL) {
  82                 return;
  83         }
  84
  85         child = ctdb_fork(ctdb);
  86         if (child == (pid_t)-1) {
  87                 DEBUG(DEBUG_ERR,("Failed to fork() a notification child process\n"));
  88                 return;
  89         }
  90         if (child == 0) {
  91                 int ret;
  92
  93                 debug_extra = talloc_asprintf(NULL, "notification-%s:", event);
  94                 ret = ctdb_run_notification_script_child(ctdb, event);
  95                 if (ret != 0) {
  96                         DEBUG(DEBUG_ERR,(__location__ " Notification script failed\n"));
  97                 }
  98                 _exit(0);
  99         }
 100
 101         return;
 102 }
 103
 104 /*
 105   called when a health monitoring event script finishes
 106  */
 107 static void ctdb_health_callback(struct ctdb_context *ctdb, int status, void *p)
 108 {
 109         struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
 110         TDB_DATA data;
 111         struct ctdb_node_flag_change c;
 112         uint32_t next_interval;
 113         int ret;
 114         TDB_DATA rddata;
 115         struct takeover_run_reply rd;
 116         const char *state_str = NULL;
 117
 118         c.pnn = ctdb->pnn;
 119         c.old_flags = node->flags;
 120
 121         rd.pnn   = ctdb->pnn;
 122         rd.srvid = CTDB_SRVID_TAKEOVER_RUN_RESPONSE;
 123
 124         rddata.dptr = (uint8_t *)&rd;
 125         rddata.dsize = sizeof(rd);
 126
 127         if (status == -ECANCELED) {
 128                 DEBUG(DEBUG_ERR,("Monitoring event was cancelled\n"));
 129                 goto after_change_status;
 130         }
 131
 132         if (status == -ETIME) {
 133                 ctdb->event_script_timeouts++;
 134
 135                 if (ctdb->event_script_timeouts >= ctdb->tunable.script_timeout_count) {
 136                         DEBUG(DEBUG_ERR, ("Maximum timeout count %u reached for eventscript. Making node unhealthy\n", ctdb->tunable.script_timeout_count));
 137                 } else {
 138                         /* We pretend this is OK. */
 139                         goto after_change_status;
 140                 }
 141         }
 142
 143         if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
 144                 DEBUG(DEBUG_NOTICE,("monitor event failed - disabling node\n"));
 145                 node->flags |= NODE_FLAGS_UNHEALTHY;
 146                 ctdb->monitor->next_interval = 5;
 147
 148                 ctdb_run_notification_script(ctdb, "unhealthy");
 149         } else if (status == 0 && (node->flags & NODE_FLAGS_UNHEALTHY)) {
 150                 DEBUG(DEBUG_NOTICE,("monitor event OK - node re-enabled\n"));
 151                 node->flags &= ~NODE_FLAGS_UNHEALTHY;
 152                 ctdb->monitor->next_interval = 5;
 153
 154                 ctdb_run_notification_script(ctdb, "healthy");
 155         }
 156
 157 after_change_status:
 158         next_interval = ctdb->monitor->next_interval;
 159
 160         ctdb->monitor->next_interval *= 2;
 161         if (ctdb->monitor->next_interval > ctdb->tunable.monitor_interval) {
 162                 ctdb->monitor->next_interval = ctdb->tunable.monitor_interval;
 163         }
 164
 165         event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 166                                 timeval_current_ofs(next_interval, 0),
 167                                 ctdb_check_health, ctdb);
 168
 169         if (c.old_flags == node->flags) {
 170                 return;
 171         }
 172
 173         c.new_flags = node->flags;
 174
 175         data.dptr = (uint8_t *)&c;
 176         data.dsize = sizeof(c);
 177
 178         /* ask the recovery daemon to push these changes out to all nodes */
 179         ctdb_daemon_send_message(ctdb, ctdb->pnn,
 180                                  CTDB_SRVID_PUSH_NODE_FLAGS, data);
 181
 182         if (c.new_flags & NODE_FLAGS_UNHEALTHY) {
 183                 state_str = "UNHEALTHY";
 184         } else {
 185                 state_str = "HEALTHY";
 186         }
 187
 188         /* ask the recmaster to reallocate all addresses */
 189         DEBUG(DEBUG_ERR,("Node became %s. Ask recovery master %u to perform ip reallocation\n",
 190                          state_str, ctdb->recovery_master));
 191         ret = ctdb_daemon_send_message(ctdb, ctdb->recovery_master, CTDB_SRVID_TAKEOVER_RUN, rddata);
 192         if (ret != 0) {
 193                 DEBUG(DEBUG_ERR,(__location__ " Failed to send ip takeover run request message to %u\n", ctdb->recovery_master));
 194         }
 195 }
 196
 197
 198 /*
 199   called when the startup event script finishes
 200  */
 201 static void ctdb_startup_callback(struct ctdb_context *ctdb, int status, void *p)
 202 {
 203         if (status != 0) {
 204                 DEBUG(DEBUG_ERR,("startup event failed\n"));
 205         } else if (status == 0) {
 206                 DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
 207                 ctdb->done_startup = true;
 208                 ctdb->monitor->next_interval = 2;
 209                 ctdb_run_notification_script(ctdb, "startup");
 210         }
 211
 212         event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 213                         timeval_current_ofs(ctdb->monitor->next_interval, 0),
 214                         ctdb_check_health, ctdb);
 215 }
 216
 217
 218 /*
 219   wait until we have finished initial recoveries before we start the
 220   monitoring events
 221  */
 222 static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_event *te,
 223                               struct timeval t, void *private_data)
 224 {
 225         struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
 226         int ret;
 227         static int count = 0;
 228
 229         count++;
 230
 231         if (count < 60 || count%600 == 0) {
 232                 DEBUG(DEBUG_NOTICE,("CTDB_WAIT_UNTIL_RECOVERED\n"));
 233                 if (ctdb->nodes[ctdb->pnn]->flags & NODE_FLAGS_STOPPED) {
 234                         DEBUG(DEBUG_NOTICE,("Node is STOPPED. Node will NOT recover.\n"));
 235                 }
 236         }
 237
 238         if (ctdb->vnn_map->generation == INVALID_GENERATION) {
 239                 ctdb->db_persistent_startup_generation = INVALID_GENERATION;
 240
 241                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 242                                      timeval_current_ofs(1, 0),
 243                                      ctdb_wait_until_recovered, ctdb);
 244                 return;
 245         }
 246
 247         if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
 248                 ctdb->db_persistent_startup_generation = INVALID_GENERATION;
 249
 250                 DEBUG(DEBUG_NOTICE,(__location__ " in recovery. Wait one more second\n"));
 251                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 252                                      timeval_current_ofs(1, 0),
 253                                      ctdb_wait_until_recovered, ctdb);
 254                 return;
 255         }
 256
 257
 258         if (!fast_start && timeval_elapsed(&ctdb->last_recovery_finished) < (ctdb->tunable.rerecovery_timeout + 3)) {
 259                 ctdb->db_persistent_startup_generation = INVALID_GENERATION;
 260
 261                 DEBUG(DEBUG_NOTICE,(__location__ " wait for pending recoveries to end. Wait one more second.\n"));
 262
 263                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 264                                      timeval_current_ofs(1, 0),
 265                                      ctdb_wait_until_recovered, ctdb);
 266                 return;
 267         }
 268
 269         if (ctdb->vnn_map->generation == ctdb->db_persistent_startup_generation) {
 270                 DEBUG(DEBUG_INFO,(__location__ " skip ctdb_recheck_persistent_health() "
 271                                   "until the next recovery\n"));
 272                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 273                                      timeval_current_ofs(1, 0),
 274                                      ctdb_wait_until_recovered, ctdb);
 275                 return;
 276         }
 277
 278         ctdb->db_persistent_startup_generation = ctdb->vnn_map->generation;
 279         ret = ctdb_recheck_persistent_health(ctdb);
 280         if (ret != 0) {
 281                 ctdb->db_persistent_check_errors++;
 282                 if (ctdb->db_persistent_check_errors < ctdb->max_persistent_check_errors) {
 283                         DEBUG(ctdb->db_persistent_check_errors==1?DEBUG_ERR:DEBUG_WARNING,
 284                               (__location__ "ctdb_recheck_persistent_health() "
 285                               "failed (%llu of %llu times) - retry later\n",
 286                               (unsigned long long)ctdb->db_persistent_check_errors,
 287                               (unsigned long long)ctdb->max_persistent_check_errors));
 288                         event_add_timed(ctdb->ev,
 289                                         ctdb->monitor->monitor_context,
 290                                         timeval_current_ofs(1, 0),
 291                                         ctdb_wait_until_recovered, ctdb);
 292                         return;
 293                 }
 294                 DEBUG(DEBUG_ALERT,(__location__
 295                                   "ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown\n",
 296                                   (unsigned long long)ctdb->db_persistent_check_errors));
 297                 ctdb_stop_recoverd(ctdb);
 298                 ctdb_stop_keepalive(ctdb);
 299                 ctdb_stop_monitoring(ctdb);
 300                 ctdb_release_all_ips(ctdb);
 301                 if (ctdb->methods != NULL) {
 302                         ctdb->methods->shutdown(ctdb);
 303                 }
 304                 ctdb_event_script(ctdb, CTDB_EVENT_SHUTDOWN);
 305                 DEBUG(DEBUG_ALERT,("ctdb_recheck_persistent_health() failed - Stopping CTDB daemon\n"));
 306                 exit(11);
 307         }
 308         ctdb->db_persistent_check_errors = 0;
 309
 310         DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n"));
 311         event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 312                              timeval_current(),
 313                              ctdb_check_health, ctdb);
 314 }
 315
 316
 317 /*
 318   see if the event scripts think we are healthy
 319  */
 320 static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
 321                               struct timeval t, void *private_data)
 322 {
 323         struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
 324         int ret = 0;
 325
 326         if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
 327             (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->done_startup)) {
 328                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 329                                 timeval_current_ofs(ctdb->monitor->next_interval, 0),
 330                                 ctdb_check_health, ctdb);
 331                 return;
 332         }
 333
 334         if (!ctdb->done_startup) {
 335                 ret = ctdb_event_script_callback(ctdb,
 336                                                  ctdb->monitor->monitor_context, ctdb_startup_callback,
 337                                                  ctdb, false,
 338                                                  CTDB_EVENT_STARTUP, "%s", "");
 339         } else {
 340                 int i;
 341                 int skip_monitoring = 0;
 342
 343                 if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL) {
 344                         skip_monitoring = 1;
 345                         DEBUG(DEBUG_ERR,("Skip monitoring during recovery\n"));
 346                 }
 347                 for (i=1; i<=NUM_DB_PRIORITIES; i++) {
 348                         if (ctdb->freeze_handles[i] != NULL) {
 349                                 DEBUG(DEBUG_ERR,("Skip monitoring since databases are frozen\n"));
 350                                 skip_monitoring = 1;
 351                                 break;
 352                         }
 353                 }
 354                 if (skip_monitoring != 0) {
 355                         event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 356                                         timeval_current_ofs(ctdb->monitor->next_interval, 0),
 357                                         ctdb_check_health, ctdb);
 358                         return;
 359                 } else {
 360                         ret = ctdb_event_script_callback(ctdb,
 361                                         ctdb->monitor->monitor_context, ctdb_health_callback,
 362                                         ctdb, false,
 363                                         CTDB_EVENT_MONITOR, "%s", "");
 364                 }
 365         }
 366
 367         if (ret != 0) {
 368                 DEBUG(DEBUG_ERR,("Unable to launch monitor event script\n"));
 369                 ctdb->monitor->next_interval = 5;
 370                 event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 371                         timeval_current_ofs(5, 0),
 372                         ctdb_check_health, ctdb);
 373         }
 374 }
 375
 376 /*
 377   (Temporaily) Disabling monitoring will stop the monitor event scripts
 378   from running   but node health checks will still occur
 379 */
 380 void ctdb_disable_monitoring(struct ctdb_context *ctdb)
 381 {
 382         ctdb->monitor->monitoring_mode = CTDB_MONITORING_DISABLED;
 383         DEBUG(DEBUG_INFO,("Monitoring has been disabled\n"));
 384 }
 385
 386 /*
 387    Re-enable running monitor events after they have been disabled
 388  */
 389 void ctdb_enable_monitoring(struct ctdb_context *ctdb)
 390 {
 391         ctdb->monitor->monitoring_mode  = CTDB_MONITORING_ACTIVE;
 392         ctdb->monitor->next_interval = 5;
 393         DEBUG(DEBUG_INFO,("Monitoring has been enabled\n"));
 394 }
 395
 396 /* stop any monitoring
 397    this should only be done when shutting down the daemon
 398 */
 399 void ctdb_stop_monitoring(struct ctdb_context *ctdb)
 400 {
 401         talloc_free(ctdb->monitor->monitor_context);
 402         ctdb->monitor->monitor_context = NULL;
 403
 404         ctdb->monitor->monitoring_mode  = CTDB_MONITORING_DISABLED;
 405         ctdb->monitor->next_interval = 5;
 406         DEBUG(DEBUG_NOTICE,("Monitoring has been stopped\n"));
 407 }
 408
 409 /*
 410   start watching for nodes that might be dead
 411  */
 412 void ctdb_start_monitoring(struct ctdb_context *ctdb)
 413 {
 414         if (ctdb->monitor != NULL) {
 415                 return;
 416         }
 417
 418         ctdb->monitor = talloc(ctdb, struct ctdb_monitor_state);
 419         CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor);
 420
 421         ctdb->monitor->next_interval = 5;
 422
 423         ctdb->monitor->monitor_context = talloc_new(ctdb->monitor);
 424         CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor->monitor_context);
 425
 426         event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 427                              timeval_current_ofs(1, 0),
 428                              ctdb_wait_until_recovered, ctdb);
 429
 430         ctdb->monitor->monitoring_mode  = CTDB_MONITORING_ACTIVE;
 431         DEBUG(DEBUG_NOTICE,("Monitoring has been started\n"));
 432 }
 433
 434
 435 /*
 436   modify flags on a node
 437  */
 438 int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 439 {
 440         struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)indata.dptr;
 441         struct ctdb_node *node;
 442         uint32_t old_flags;
 443
 444         if (c->pnn >= ctdb->num_nodes) {
 445                 DEBUG(DEBUG_ERR,(__location__ " Node %d is invalid, num_nodes :%d\n", c->pnn, ctdb->num_nodes));
 446                 return -1;
 447         }
 448
 449         node         = ctdb->nodes[c->pnn];
 450         old_flags    = node->flags;
 451         if (c->pnn != ctdb->pnn) {
 452                 c->old_flags  = node->flags;
 453         }
 454         node->flags   = c->new_flags & ~NODE_FLAGS_DISCONNECTED;
 455         node->flags  |= (c->old_flags & NODE_FLAGS_DISCONNECTED);
 456
 457         /* we dont let other nodes modify our STOPPED status */
 458         if (c->pnn == ctdb->pnn) {
 459                 node->flags &= ~NODE_FLAGS_STOPPED;
 460                 if (old_flags & NODE_FLAGS_STOPPED) {
 461                         node->flags |= NODE_FLAGS_STOPPED;
 462                 }
 463         }
 464
 465         /* we dont let other nodes modify our BANNED status */
 466         if (c->pnn == ctdb->pnn) {
 467                 node->flags &= ~NODE_FLAGS_BANNED;
 468                 if (old_flags & NODE_FLAGS_BANNED) {
 469                         node->flags |= NODE_FLAGS_BANNED;
 470                 }
 471         }
 472
 473         if (node->flags == c->old_flags) {
 474                 DEBUG(DEBUG_INFO, ("Control modflags on node %u - Unchanged - flags 0x%x\n", c->pnn, node->flags));
 475                 return 0;
 476         }
 477
 478         DEBUG(DEBUG_INFO, ("Control modflags on node %u - flags now 0x%x\n", c->pnn, node->flags));
 479
 480         if (node->flags == 0 && !ctdb->done_startup) {
 481                 DEBUG(DEBUG_ERR, (__location__ " Node %u became healthy - force recovery for startup\n",
 482                                   c->pnn));
 483                 ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
 484         }
 485
 486         /* tell the recovery daemon something has changed */
 487         ctdb_daemon_send_message(ctdb, ctdb->pnn,
 488                                  CTDB_SRVID_SET_NODE_FLAGS, indata);
 489
 490         /* if we have become banned, we should go into recovery mode */
 491         if ((node->flags & NODE_FLAGS_BANNED) && !(c->old_flags & NODE_FLAGS_BANNED) && (node->pnn == ctdb->pnn)) {
 492                 return ctdb_local_node_got_banned(ctdb);
 493         }
 494
 495         return 0;
 496 }
 497
 498 /*
 499   return the monitoring mode
 500  */
 501 int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb)
 502 {
 503         if (ctdb->monitor == NULL) {
 504                 return CTDB_MONITORING_DISABLED;
 505         }
 506         return ctdb->monitor->monitoring_mode;
 507 }
 508
 509 /*
 510  * Check if monitoring has been stopped
 511  */
 512 bool ctdb_stopped_monitoring(struct ctdb_context *ctdb)
 513 {
 514         return (ctdb->monitor->monitor_context == NULL ? true : false);
 515 }