From ed6814ff0a59ddbb1c1b3128b505380f60d7aeb7 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 18 Apr 2013 20:30:14 +1000 Subject: [PATCH] ctdbd: Add new runstate CTDB_RUNSTATE_FIRST_RECOVERY This adds more serialisation to the startup, ensuring that the "startup" event runs after everything to do with the first recovery (including the "recovered" event). Given that it now takes longer to get to the "startup" state, the initscript needs to wait until ctdbd gets to "first_recovery". Signed-off-by: Martin Schwenke Pair-programmed-with: Amitay Isaacs --- common/ctdb_util.c | 1 + config/ctdb.init | 2 +- doc/ctdb.1.xml | 2 +- include/ctdb_private.h | 1 + server/ctdb_daemon.c | 2 +- server/ctdb_monitor.c | 10 +++++++++- server/ctdb_recover.c | 4 ++++ tools/ctdb.c | 2 +- 8 files changed, 19 insertions(+), 5 deletions(-) diff --git a/common/ctdb_util.c b/common/ctdb_util.c index 270ad625..a910a0cd 100644 --- a/common/ctdb_util.c +++ b/common/ctdb_util.c @@ -711,6 +711,7 @@ static struct { { CTDB_RUNSTATE_UNKNOWN, "UNKNOWN" }, { CTDB_RUNSTATE_INIT, "INIT" }, { CTDB_RUNSTATE_SETUP, "SETUP" }, + { CTDB_RUNSTATE_FIRST_RECOVERY, "FIRST_RECOVERY" }, { CTDB_RUNSTATE_STARTUP, "STARTUP" }, { CTDB_RUNSTATE_RUNNING, "RUNNING" }, { CTDB_RUNSTATE_SHUTDOWN, "SHUTDOWN" }, diff --git a/config/ctdb.init b/config/ctdb.init index 70dcfa64..2ceb45ff 100755 --- a/config/ctdb.init +++ b/config/ctdb.init @@ -220,7 +220,7 @@ wait_until_ready () { _timeout="${1:-10}" # default is 10 seconds _count=0 - while ! ctdb runstate startup running >/dev/null 2>&1 ; do + while ! ctdb runstate first_recovery startup running >/dev/null 2>&1 ; do if [ $_count -ge $_timeout ] ; then return 1 fi diff --git a/doc/ctdb.1.xml b/doc/ctdb.1.xml index 7242f3aa..ce83a3ef 100644 --- a/doc/ctdb.1.xml +++ b/doc/ctdb.1.xml @@ -382,7 +382,7 @@ response from 3 time=0.000114 sec (2 clients) - runstate [setup|startup|running] + runstate [setup|first_recovery|startup|running] Print the runstate of the specified node. Runstates are used to serialise important state transitions in CTDB, particularly diff --git a/include/ctdb_private.h b/include/ctdb_private.h index 2698785f..eadd9637 100644 --- a/include/ctdb_private.h +++ b/include/ctdb_private.h @@ -440,6 +440,7 @@ enum ctdb_runstate { CTDB_RUNSTATE_UNKNOWN, CTDB_RUNSTATE_INIT, CTDB_RUNSTATE_SETUP, + CTDB_RUNSTATE_FIRST_RECOVERY, CTDB_RUNSTATE_STARTUP, CTDB_RUNSTATE_RUNNING, CTDB_RUNSTATE_SHUTDOWN, diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c index edbc34c7..32c30243 100644 --- a/server/ctdb_daemon.c +++ b/server/ctdb_daemon.c @@ -1037,7 +1037,7 @@ static void ctdb_setup_event_callback(struct ctdb_context *ctdb, int status, } ctdb_run_notification_script(ctdb, "setup"); - ctdb_set_runstate(ctdb, CTDB_RUNSTATE_STARTUP); + ctdb_set_runstate(ctdb, CTDB_RUNSTATE_FIRST_RECOVERY); /* tell all other nodes we've just started up */ ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, diff --git a/server/ctdb_monitor.c b/server/ctdb_monitor.c index 1e556e00..1608804c 100644 --- a/server/ctdb_monitor.c +++ b/server/ctdb_monitor.c @@ -307,7 +307,6 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve } ctdb->db_persistent_check_errors = 0; - DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n")); event_add_timed(ctdb->ev, ctdb->monitor->monitor_context, timeval_current(), ctdb_check_health, ctdb); @@ -323,6 +322,14 @@ static void ctdb_check_health(struct event_context *ev, struct timed_event *te, struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context); int ret = 0; + if (ctdb->runstate < CTDB_RUNSTATE_STARTUP) { + DEBUG(DEBUG_NOTICE,("Not yet in startup runstate. Wait one more second\n")); + event_add_timed(ctdb->ev, ctdb->monitor->monitor_context, + timeval_current_ofs(1, 0), + ctdb_check_health, ctdb); + return; + } + if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL || (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->runstate == CTDB_RUNSTATE_RUNNING)) { @@ -333,6 +340,7 @@ static void ctdb_check_health(struct event_context *ev, struct timed_event *te, } if (ctdb->runstate == CTDB_RUNSTATE_STARTUP) { + DEBUG(DEBUG_NOTICE,("Recoveries finished. Running the \"startup\" event.\n")); ret = ctdb_event_script_callback(ctdb, ctdb->monitor->monitor_context, ctdb_startup_callback, ctdb, false, diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c index 7fd0ac36..6506f8bb 100644 --- a/server/ctdb_recover.c +++ b/server/ctdb_recover.c @@ -911,6 +911,10 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo talloc_free(state); gettimeofday(&ctdb->last_recovery_finished, NULL); + + if (ctdb->runstate == CTDB_RUNSTATE_FIRST_RECOVERY) { + ctdb_set_runstate(ctdb, CTDB_RUNSTATE_STARTUP); + } } /* diff --git a/tools/ctdb.c b/tools/ctdb.c index e21e8450..48ca93ba 100644 --- a/tools/ctdb.c +++ b/tools/ctdb.c @@ -5847,7 +5847,7 @@ static const struct { { "status", control_status, true, false, "show node status" }, { "uptime", control_uptime, true, false, "show node uptime" }, { "ping", control_ping, true, false, "ping all nodes" }, - { "runstate", control_runstate, true, false, "get/check runstate of a node", "[setup|startup|running]" }, + { "runstate", control_runstate, true, false, "get/check runstate of a node", "[setup|first_recovery|startup|running]" }, { "getvar", control_getvar, true, false, "get a tunable variable", ""}, { "setvar", control_setvar, true, false, "set a tunable variable", " "}, { "listvars", control_listvars, true, false, "list tunable variables"}, -- 2.34.1