You should have received a copy of the GNU General Public License
along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
-#include "includes.h"
-#include "tdb.h"
+#include "replace.h"
#include "system/time.h"
#include "system/network.h"
#include "system/filesys.h"
#include "system/wait.h"
-#include "../include/ctdb_private.h"
-#include "lib/util/dlinklist.h"
+
+#include <talloc.h>
+#include <tevent.h>
+#include <tdb.h>
+
#include "lib/tdb_wrap/tdb_wrap.h"
+#include "lib/util/dlinklist.h"
+#include "lib/util/debug.h"
+#include "lib/util/time.h"
+#include "lib/util/util_process.h"
+
+#include "ctdb_private.h"
+#include "ctdb_client.h"
+#include "common/system.h"
+#include "common/common.h"
+#include "common/logging.h"
+
+#include "ctdb_cluster_mutex.h"
int
ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
return 0;
}
-int
+int
ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
{
struct ctdb_vnn_map_wire *map = (struct ctdb_vnn_map_wire *)indata.dptr;
- int i;
- for(i=1; i<=NUM_DB_PRIORITIES; i++) {
- if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
- DEBUG(DEBUG_ERR,("Attempt to set vnnmap when not frozen\n"));
- return -1;
- }
+ if (ctdb->recovery_mode != CTDB_RECOVERY_ACTIVE) {
+ DEBUG(DEBUG_ERR, ("Attempt to set vnnmap when not in recovery\n"));
+ return -1;
}
talloc_free(ctdb->vnn_map);
{
uint32_t i, len;
struct ctdb_db_context *ctdb_db;
- struct ctdb_dbid_map *dbid_map;
+ struct ctdb_dbid_map_old *dbid_map;
CHECK_CONTROL_DATA_SIZE(0);
}
- outdata->dsize = offsetof(struct ctdb_dbid_map, dbs) + sizeof(dbid_map->dbs[0])*len;
+ outdata->dsize = offsetof(struct ctdb_dbid_map_old, dbs) + sizeof(dbid_map->dbs[0])*len;
outdata->dptr = (unsigned char *)talloc_zero_size(outdata, outdata->dsize);
if (!outdata->dptr) {
DEBUG(DEBUG_ALERT, (__location__ " Failed to allocate dbmap array\n"));
exit(1);
}
- dbid_map = (struct ctdb_dbid_map *)outdata->dptr;
+ dbid_map = (struct ctdb_dbid_map_old *)outdata->dptr;
dbid_map->num = len;
for (i=0,ctdb_db=ctdb->db_list;ctdb_db;i++,ctdb_db=ctdb_db->next){
- dbid_map->dbs[i].dbid = ctdb_db->db_id;
+ dbid_map->dbs[i].db_id = ctdb_db->db_id;
if (ctdb_db->persistent != 0) {
dbid_map->dbs[i].flags |= CTDB_DB_FLAGS_PERSISTENT;
}
static int traverse_pulldb(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
{
struct pulldb_data *params = (struct pulldb_data *)p;
- struct ctdb_rec_data *rec;
+ struct ctdb_rec_data_old *rec;
struct ctdb_context *ctdb = params->ctdb;
struct ctdb_db_context *ctdb_db = params->ctdb_db;
*/
int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
{
- struct ctdb_control_pulldb *pull;
+ struct ctdb_pulldb *pull;
struct ctdb_db_context *ctdb_db;
struct pulldb_data params;
struct ctdb_marshall_buffer *reply;
- pull = (struct ctdb_control_pulldb *)indata.dptr;
-
+ pull = (struct ctdb_pulldb *)indata.dptr;
+
ctdb_db = find_ctdb_db(ctdb, pull->db_id);
if (!ctdb_db) {
DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", pull->db_id));
return -1;
}
- if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
- DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_pull_db when not frozen\n"));
+ if (!ctdb_db_frozen(ctdb_db)) {
+ DEBUG(DEBUG_ERR,
+ ("rejecting ctdb_control_pull_db when not frozen\n"));
return -1;
}
ctdb_db->db_name, ctdb_db->unhealthy_reason));
}
- if (ctdb_lockall_mark_prio(ctdb, ctdb_db->priority) != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
+ if (ctdb_lockdb_mark(ctdb_db) != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entire db - failing\n"));
return -1;
}
if (tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_pulldb, ¶ms) == -1) {
DEBUG(DEBUG_ERR,(__location__ " Failed to get traverse db '%s'\n", ctdb_db->db_name));
- ctdb_lockall_unmark_prio(ctdb, ctdb_db->priority);
+ ctdb_lockdb_unmark(ctdb_db);
talloc_free(params.pulldata);
return -1;
}
- ctdb_lockall_unmark_prio(ctdb, ctdb_db->priority);
+ ctdb_lockdb_unmark(ctdb_db);
outdata->dptr = (uint8_t *)params.pulldata;
outdata->dsize = params.len;
return 0;
}
+struct db_pull_state {
+ struct ctdb_context *ctdb;
+ struct ctdb_db_context *ctdb_db;
+ struct ctdb_marshall_buffer *recs;
+ uint32_t pnn;
+ uint64_t srvid;
+ uint32_t num_records;
+};
+
+static int traverse_db_pull(struct tdb_context *tdb, TDB_DATA key,
+ TDB_DATA data, void *private_data)
+{
+ struct db_pull_state *state = (struct db_pull_state *)private_data;
+ struct ctdb_marshall_buffer *recs;
+
+ recs = ctdb_marshall_add(state->ctdb, state->recs,
+ state->ctdb_db->db_id, 0, key, NULL, data);
+ if (recs == NULL) {
+ TALLOC_FREE(state->recs);
+ return -1;
+ }
+ state->recs = recs;
+
+ if (talloc_get_size(state->recs) >=
+ state->ctdb->tunable.rec_buffer_size_limit) {
+ TDB_DATA buffer;
+ int ret;
+
+ buffer = ctdb_marshall_finish(state->recs);
+ ret = ctdb_daemon_send_message(state->ctdb, state->pnn,
+ state->srvid, buffer);
+ if (ret != 0) {
+ TALLOC_FREE(state->recs);
+ return -1;
+ }
+
+ state->num_records += state->recs->count;
+ TALLOC_FREE(state->recs);
+ }
+
+ return 0;
+}
+
+int32_t ctdb_control_db_pull(struct ctdb_context *ctdb,
+ struct ctdb_req_control_old *c,
+ TDB_DATA indata, TDB_DATA *outdata)
+{
+ struct ctdb_pulldb_ext *pulldb_ext;
+ struct ctdb_db_context *ctdb_db;
+ struct db_pull_state state;
+ int ret;
+
+ pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr;
+
+ ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id);
+ if (ctdb_db == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n",
+ pulldb_ext->db_id));
+ return -1;
+ }
+
+ if (!ctdb_db_frozen(ctdb_db)) {
+ DEBUG(DEBUG_ERR,
+ ("rejecting ctdb_control_pull_db when not frozen\n"));
+ return -1;
+ }
+
+ if (ctdb_db->unhealthy_reason) {
+ /* this is just a warning, as the tdb should be empty anyway */
+ DEBUG(DEBUG_WARNING,
+ ("db(%s) unhealty in ctdb_control_db_pull: %s\n",
+ ctdb_db->db_name, ctdb_db->unhealthy_reason));
+ }
+
+ state.ctdb = ctdb;
+ state.ctdb_db = ctdb_db;
+ state.recs = NULL;
+ state.pnn = c->hdr.srcnode;
+ state.srvid = pulldb_ext->srvid;
+ state.num_records = 0;
+
+ if (ctdb_lockdb_mark(ctdb_db) != 0) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Failed to get lock on entire db - failing\n"));
+ return -1;
+ }
+
+ ret = tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_db_pull, &state);
+ if (ret == -1) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Failed to get traverse db '%s'\n",
+ ctdb_db->db_name));
+ ctdb_lockdb_unmark(ctdb_db);
+ return -1;
+ }
+
+ /* Last few records */
+ if (state.recs != NULL) {
+ TDB_DATA buffer;
+
+ buffer = ctdb_marshall_finish(state.recs);
+ ret = ctdb_daemon_send_message(state.ctdb, state.pnn,
+ state.srvid, buffer);
+ if (ret != 0) {
+ TALLOC_FREE(state.recs);
+ ctdb_lockdb_unmark(ctdb_db);
+ return -1;
+ }
+
+ state.num_records += state.recs->count;
+ TALLOC_FREE(state.recs);
+ }
+
+ ctdb_lockdb_unmark(ctdb_db);
+
+ outdata->dptr = talloc_size(outdata, sizeof(uint32_t));
+ if (outdata->dptr == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
+ return -1;
+ }
+
+ memcpy(outdata->dptr, (uint8_t *)&state.num_records, sizeof(uint32_t));
+ outdata->dsize = sizeof(uint32_t);
+
+ return 0;
+}
+
/*
push a bunch of records into a ltdb, filtering by rsn
*/
struct ctdb_marshall_buffer *reply = (struct ctdb_marshall_buffer *)indata.dptr;
struct ctdb_db_context *ctdb_db;
int i, ret;
- struct ctdb_rec_data *rec;
+ struct ctdb_rec_data_old *rec;
if (indata.dsize < offsetof(struct ctdb_marshall_buffer, data)) {
DEBUG(DEBUG_ERR,(__location__ " invalid data in pulldb reply\n"));
return -1;
}
- if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
- DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_push_db when not frozen\n"));
+ if (!ctdb_db_frozen(ctdb_db)) {
+ DEBUG(DEBUG_ERR,
+ ("rejecting ctdb_control_push_db when not frozen\n"));
return -1;
}
- if (ctdb_lockall_mark_prio(ctdb, ctdb_db->priority) != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
+ if (ctdb_lockdb_mark(ctdb_db) != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entire db - failing\n"));
return -1;
}
- rec = (struct ctdb_rec_data *)&reply->data[0];
+ rec = (struct ctdb_rec_data_old *)&reply->data[0];
DEBUG(DEBUG_INFO,("starting push of %u records for dbid 0x%x\n",
reply->count, reply->db_id));
goto failed;
}
- rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec);
+ rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec);
}
DEBUG(DEBUG_DEBUG,("finished push of %u records for dbid 0x%x\n",
}
}
- ctdb_lockall_unmark_prio(ctdb, ctdb_db->priority);
+ ctdb_lockdb_unmark(ctdb_db);
return 0;
failed:
- ctdb_lockall_unmark_prio(ctdb, ctdb_db->priority);
+ ctdb_lockdb_unmark(ctdb_db);
return -1;
}
-struct ctdb_set_recmode_state {
+struct db_push_state {
struct ctdb_context *ctdb;
- struct ctdb_req_control *c;
- uint32_t recmode;
- int fd[2];
- struct timed_event *te;
- struct fd_event *fde;
- pid_t child;
- struct timeval start_time;
+ struct ctdb_db_context *ctdb_db;
+ uint64_t srvid;
+ uint32_t num_records;
+ bool failed;
};
-/*
- called if our set_recmode child times out. this would happen if
- ctdb_recovery_lock() would block.
- */
-static void ctdb_set_recmode_timeout(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data)
+static void db_push_msg_handler(uint64_t srvid, TDB_DATA indata,
+ void *private_data)
{
- struct ctdb_set_recmode_state *state = talloc_get_type(private_data,
- struct ctdb_set_recmode_state);
-
- /* we consider this a success, not a failure, as we failed to
- set the recovery lock which is what we wanted. This can be
- caused by the cluster filesystem being very slow to
- arbitrate locks immediately after a node failure.
- */
- DEBUG(DEBUG_ERR,(__location__ " set_recmode child process hung/timedout CFS slow to grant locks? (allowing recmode set anyway)\n"));
- state->ctdb->recovery_mode = state->recmode;
- ctdb_request_control_reply(state->ctdb, state->c, NULL, 0, NULL);
- talloc_free(state);
-}
+ struct db_push_state *state = talloc_get_type(
+ private_data, struct db_push_state);
+ struct ctdb_marshall_buffer *recs;
+ struct ctdb_rec_data_old *rec;
+ int i, ret;
+ if (state->failed) {
+ return;
+ }
-/* when we free the recmode state we must kill any child process.
-*/
-static int set_recmode_destructor(struct ctdb_set_recmode_state *state)
+ recs = (struct ctdb_marshall_buffer *)indata.dptr;
+ rec = (struct ctdb_rec_data_old *)&recs->data[0];
+
+ DEBUG(DEBUG_INFO, ("starting push of %u records for dbid 0x%x\n",
+ recs->count, recs->db_id));
+
+ for (i=0; i<recs->count; i++) {
+ TDB_DATA key, data;
+ struct ctdb_ltdb_header *hdr;
+
+ key.dptr = &rec->data[0];
+ key.dsize = rec->keylen;
+ data.dptr = &rec->data[key.dsize];
+ data.dsize = rec->datalen;
+
+ if (data.dsize < sizeof(struct ctdb_ltdb_header)) {
+ DEBUG(DEBUG_CRIT,(__location__ " bad ltdb record\n"));
+ goto failed;
+ }
+
+ hdr = (struct ctdb_ltdb_header *)data.dptr;
+ /* Strip off any read only record flags.
+ * All readonly records are revoked implicitely by a recovery.
+ */
+ hdr->flags &= ~CTDB_REC_RO_FLAGS;
+
+ data.dptr += sizeof(*hdr);
+ data.dsize -= sizeof(*hdr);
+
+ ret = ctdb_ltdb_store(state->ctdb_db, key, hdr, data);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Unable to store record\n"));
+ goto failed;
+ }
+
+ rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec);
+ }
+
+ DEBUG(DEBUG_DEBUG, ("finished push of %u records for dbid 0x%x\n",
+ recs->count, recs->db_id));
+
+ state->num_records += recs->count;
+ return;
+
+failed:
+ state->failed = true;
+}
+
+int32_t ctdb_control_db_push_start(struct ctdb_context *ctdb, TDB_DATA indata)
{
- double l = timeval_elapsed(&state->start_time);
+ struct ctdb_pulldb_ext *pulldb_ext;
+ struct ctdb_db_context *ctdb_db;
+ struct db_push_state *state;
+ int ret;
+
+ pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr;
+
+ ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id);
+ if (ctdb_db == NULL) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Unknown db 0x%08x\n", pulldb_ext->db_id));
+ return -1;
+ }
+
+ if (!ctdb_db_frozen(ctdb_db)) {
+ DEBUG(DEBUG_ERR,
+ ("rejecting ctdb_control_db_push_start when not frozen\n"));
+ return -1;
+ }
+
+ if (ctdb_db->push_started) {
+ DEBUG(DEBUG_WARNING,
+ (__location__ " DB push already started for %s\n",
+ ctdb_db->db_name));
+
+ /* De-register old state */
+ state = (struct db_push_state *)ctdb_db->push_state;
+ if (state != NULL) {
+ srvid_deregister(ctdb->srv, state->srvid, state);
+ talloc_free(state);
+ ctdb_db->push_state = NULL;
+ }
+ }
- CTDB_UPDATE_RECLOCK_LATENCY(state->ctdb, "daemon reclock", reclock.ctdbd, l);
+ state = talloc_zero(ctdb_db, struct db_push_state);
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
+ return -1;
+ }
- if (state->fd[0] != -1) {
- state->fd[0] = -1;
+ state->ctdb = ctdb;
+ state->ctdb_db = ctdb_db;
+ state->srvid = pulldb_ext->srvid;
+ state->failed = false;
+
+ ret = srvid_register(ctdb->srv, state, state->srvid,
+ db_push_msg_handler, state);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Failed to register srvid for db push\n"));
+ talloc_free(state);
+ return -1;
}
- if (state->fd[1] != -1) {
- state->fd[1] = -1;
+
+ if (ctdb_lockdb_mark(ctdb_db) != 0) {
+ DEBUG(DEBUG_ERR,
+ (__location__ " Failed to get lock on entire db - failing\n"));
+ srvid_deregister(ctdb->srv, state->srvid, state);
+ talloc_free(state);
+ return -1;
}
- ctdb_kill(state->ctdb, state->child, SIGKILL);
+
+ ctdb_db->push_started = true;
+ ctdb_db->push_state = state;
+
return 0;
}
-/* this is called when the client process has completed ctdb_recovery_lock()
- and has written data back to us through the pipe.
-*/
-static void set_recmode_handler(struct event_context *ev, struct fd_event *fde,
- uint16_t flags, void *private_data)
+int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb,
+ TDB_DATA indata, TDB_DATA *outdata)
{
- struct ctdb_set_recmode_state *state= talloc_get_type(private_data,
- struct ctdb_set_recmode_state);
- char c = 0;
- int ret;
+ uint32_t db_id;
+ struct ctdb_db_context *ctdb_db;
+ struct db_push_state *state;
- /* we got a response from our child process so we can abort the
- timeout.
- */
- talloc_free(state->te);
- state->te = NULL;
+ db_id = *(uint32_t *)indata.dptr;
+ ctdb_db = find_ctdb_db(ctdb, db_id);
+ if (ctdb_db == NULL) {
+ DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", db_id));
+ return -1;
+ }
- /* If, as expected, the child was unable to take the recovery
- * lock then it will have written 0 into the pipe, so
- * continue. However, any other value (e.g. 1) indicates that
- * it was able to take the recovery lock when it should have
- * been held by the recovery daemon on the recovery master.
- */
- ret = sys_read(state->fd[0], &c, 1);
- if (ret != 1 || c != 0) {
- ctdb_request_control_reply(
- state->ctdb, state->c, NULL, -1,
- "Took recovery lock from daemon during recovery - probably a cluster filesystem lock coherence problem");
+ if (!ctdb_db_frozen(ctdb_db)) {
+ DEBUG(DEBUG_ERR,
+ ("rejecting ctdb_control_db_push_confirm when not frozen\n"));
+ return -1;
+ }
+
+ if (!ctdb_db->push_started) {
+ DEBUG(DEBUG_ERR, (__location__ " DB push not started\n"));
+ return -1;
+ }
+
+ if (ctdb_db->readonly) {
+ DEBUG(DEBUG_ERR,
+ ("Clearing the tracking database for dbid 0x%x\n",
+ ctdb_db->db_id));
+ if (tdb_wipe_all(ctdb_db->rottdb) != 0) {
+ DEBUG(DEBUG_ERR,
+ ("Failed to wipe tracking database for 0x%x."
+ " Dropping read-only delegation support\n",
+ ctdb_db->db_id));
+ ctdb_db->readonly = false;
+ tdb_close(ctdb_db->rottdb);
+ ctdb_db->rottdb = NULL;
+ ctdb_db->readonly = false;
+ }
+
+ while (ctdb_db->revokechild_active != NULL) {
+ talloc_free(ctdb_db->revokechild_active);
+ }
+ }
+
+ ctdb_lockdb_unmark(ctdb_db);
+
+ state = (struct db_push_state *)ctdb_db->push_state;
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " Missing push db state\n"));
+ return -1;
+ }
+
+ srvid_deregister(ctdb->srv, state->srvid, state);
+
+ outdata->dptr = talloc_size(outdata, sizeof(uint32_t));
+ if (outdata->dptr == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
talloc_free(state);
- return;
+ ctdb_db->push_state = NULL;
+ return -1;
}
- state->ctdb->recovery_mode = state->recmode;
+ memcpy(outdata->dptr, (uint8_t *)&state->num_records, sizeof(uint32_t));
+ outdata->dsize = sizeof(uint32_t);
+
+ talloc_free(state);
+ ctdb_db->push_started = false;
+ ctdb_db->push_state = NULL;
+
+ return 0;
+}
+
+struct set_recmode_state {
+ struct ctdb_context *ctdb;
+ struct ctdb_req_control_old *c;
+};
+
+static void set_recmode_handler(char status,
+ double latency,
+ void *private_data)
+{
+ struct set_recmode_state *state = talloc_get_type_abort(
+ private_data, struct set_recmode_state);
+ int s = 0;
+ const char *err = NULL;
+
+ switch (status) {
+ case '0':
+ /* Mutex taken */
+ DEBUG(DEBUG_ERR,
+ ("ERROR: Daemon able to take recovery lock on \"%s\" during recovery\n",
+ state->ctdb->recovery_lock));
+ s = -1;
+ err = "Took recovery lock from daemon during recovery - probably a cluster filesystem lock coherence problem";
+ break;
+
+ case '1':
+ /* Contention */
+ DEBUG(DEBUG_DEBUG, (__location__ " Recovery lock check OK\n"));
+ state->ctdb->recovery_mode = CTDB_RECOVERY_NORMAL;
+ ctdb_process_deferred_attach(state->ctdb);
- /* release any deferred attach calls from clients */
- if (state->recmode == CTDB_RECOVERY_NORMAL) {
+ s = 0;
+
+ CTDB_UPDATE_RECLOCK_LATENCY(state->ctdb, "daemon reclock",
+ reclock.ctdbd, latency);
+ break;
+
+ case '2':
+ /* Timeout. Consider this a success, not a failure,
+ * as we failed to set the recovery lock which is what
+ * we wanted. This can be caused by the cluster
+ * filesystem being very slow to arbitrate locks
+ * immediately after a node failure. */
+ DEBUG(DEBUG_WARNING,
+ (__location__
+ "Time out getting recovery lock, allowing recmode set anyway\n"));
+ state->ctdb->recovery_mode = CTDB_RECOVERY_NORMAL;
ctdb_process_deferred_attach(state->ctdb);
+
+ s = 0;
+ break;
+
+ default:
+ DEBUG(DEBUG_ERR,
+ ("Unexpected error when testing recovery lock\n"));
+ s = -1;
+ err = "Unexpected error when testing recovery lock";
}
- ctdb_request_control_reply(state->ctdb, state->c, NULL, 0, NULL);
+ ctdb_request_control_reply(state->ctdb, state->c, NULL, s, err);
talloc_free(state);
- return;
}
static void
-ctdb_drop_all_ips_event(struct event_context *ev, struct timed_event *te,
- struct timeval t, void *private_data)
+ctdb_drop_all_ips_event(struct tevent_context *ev, struct tevent_timer *te,
+ struct timeval t, void *private_data)
{
struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
ctdb->release_ips_ctx = talloc_new(ctdb);
CTDB_NO_MEMORY(ctdb, ctdb->release_ips_ctx);
- event_add_timed(ctdb->ev, ctdb->release_ips_ctx, timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0), ctdb_drop_all_ips_event, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->release_ips_ctx,
+ timeval_current_ofs(ctdb->tunable.recovery_drop_all_ips, 0),
+ ctdb_drop_all_ips_event, ctdb);
return 0;
}
set the recovery mode
*/
int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
- struct ctdb_req_control *c,
+ struct ctdb_req_control_old *c,
TDB_DATA indata, bool *async_reply,
const char **errormsg)
{
uint32_t recmode = *(uint32_t *)indata.dptr;
- int i, ret;
- struct ctdb_set_recmode_state *state;
- pid_t parent = getpid();
+ int i;
+ struct ctdb_db_context *ctdb_db;
+ struct set_recmode_state *state;
+ struct ctdb_cluster_mutex_handle *h;
/* if we enter recovery but stay in recovery for too long
we will eventually drop all our ip addresses
return 0;
}
- /* some special handling when ending recovery mode */
+ /* From this point: recmode == CTDB_RECOVERY_NORMAL
+ *
+ * Therefore, what follows is special handling when setting
+ * recovery mode back to normal */
+
+ for (ctdb_db = ctdb->db_list; ctdb_db != NULL; ctdb_db = ctdb_db->next) {
+ if (ctdb_db->generation != ctdb->vnn_map->generation) {
+ DEBUG(DEBUG_ERR,
+ ("Inconsistent DB generation %u for %s\n",
+ ctdb_db->generation, ctdb_db->db_name));
+ DEBUG(DEBUG_ERR, ("Recovery mode set to ACTIVE\n"));
+ return -1;
+ }
+ }
/* force the databases to thaw */
for (i=1; i<=NUM_DB_PRIORITIES; i++) {
- if (ctdb->freeze_handles[i] != NULL) {
+ if (ctdb_db_prio_frozen(ctdb, i)) {
ctdb_control_thaw(ctdb, i, false);
}
}
- state = talloc(ctdb, struct ctdb_set_recmode_state);
- CTDB_NO_MEMORY(ctdb, state);
-
- state->start_time = timeval_current();
- state->fd[0] = -1;
- state->fd[1] = -1;
-
- /* release any deferred attach calls from clients */
- if (recmode == CTDB_RECOVERY_NORMAL) {
- ctdb_process_deferred_attach(ctdb);
- }
-
- if (ctdb->recovery_lock_file == NULL) {
+ if (ctdb->recovery_lock == NULL) {
/* Not using recovery lock file */
- ctdb->recovery_mode = recmode;
+ ctdb->recovery_mode = CTDB_RECOVERY_NORMAL;
+ ctdb_process_deferred_attach(ctdb);
return 0;
}
- /* For the rest of what needs to be done, we need to do this in
- a child process since
- 1, the call to ctdb_recovery_lock() can block if the cluster
- filesystem is in the process of recovery.
- */
- ret = pipe(state->fd);
- if (ret != 0) {
- talloc_free(state);
- DEBUG(DEBUG_CRIT,(__location__ " Failed to open pipe for set_recmode child\n"));
+ state = talloc_zero(ctdb, struct set_recmode_state);
+ if (state == NULL) {
+ DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
return -1;
}
+ state->ctdb = ctdb;
+ state->c = NULL;
- state->child = ctdb_fork(ctdb);
- if (state->child == (pid_t)-1) {
- close(state->fd[0]);
- close(state->fd[1]);
+ h = ctdb_cluster_mutex(state, ctdb, ctdb->recovery_lock, 5,
+ set_recmode_handler, state, NULL, NULL);
+ if (h == NULL) {
talloc_free(state);
return -1;
}
- if (state->child == 0) {
- char cc = 0;
- close(state->fd[0]);
-
- ctdb_set_process_name("ctdb_recmode");
- debug_extra = talloc_asprintf(NULL, "set_recmode:");
- /* Daemon should not be able to get the recover lock,
- * as it should be held by the recovery master */
- if (ctdb_recovery_lock(ctdb)) {
- DEBUG(DEBUG_ERR,
- ("ERROR: Daemon able to take recovery lock on \"%s\" during recovery\n",
- ctdb->recovery_lock_file));
- ctdb_recovery_unlock(ctdb);
- cc = 1;
- }
-
- sys_write(state->fd[1], &cc, 1);
- /* make sure we die when our parent dies */
- while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
- sleep(5);
- sys_write(state->fd[1], &cc, 1);
- }
- _exit(0);
- }
- close(state->fd[1]);
- set_close_on_exec(state->fd[0]);
-
- state->fd[1] = -1;
-
- talloc_set_destructor(state, set_recmode_destructor);
-
- DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d for setrecmode\n", state->fd[0]));
-
- state->te = event_add_timed(ctdb->ev, state, timeval_current_ofs(5, 0),
- ctdb_set_recmode_timeout, state);
-
- state->fde = event_add_fd(ctdb->ev, state, state->fd[0],
- EVENT_FD_READ,
- set_recmode_handler,
- (void *)state);
-
- if (state->fde == NULL) {
- talloc_free(state);
- return -1;
- }
- tevent_fd_set_auto_close(state->fde);
-
- state->ctdb = ctdb;
- state->recmode = recmode;
- state->c = talloc_steal(state, c);
-
+ state->c = talloc_steal(state, c);
*async_reply = true;
return 0;
}
-bool ctdb_recovery_have_lock(struct ctdb_context *ctdb)
-{
- return ctdb->recovery_lock_fd != -1;
-}
-
-/*
- try and get the recovery lock in shared storage - should only work
- on the recovery master recovery daemon. Anywhere else is a bug
- */
-bool ctdb_recovery_lock(struct ctdb_context *ctdb)
-{
- struct flock lock;
-
- ctdb->recovery_lock_fd = open(ctdb->recovery_lock_file,
- O_RDWR|O_CREAT, 0600);
- if (ctdb->recovery_lock_fd == -1) {
- DEBUG(DEBUG_ERR,
- ("ctdb_recovery_lock: Unable to open %s - (%s)\n",
- ctdb->recovery_lock_file, strerror(errno)));
- return false;
- }
-
- set_close_on_exec(ctdb->recovery_lock_fd);
-
- lock.l_type = F_WRLCK;
- lock.l_whence = SEEK_SET;
- lock.l_start = 0;
- lock.l_len = 1;
- lock.l_pid = 0;
-
- if (fcntl(ctdb->recovery_lock_fd, F_SETLK, &lock) != 0) {
- int saved_errno = errno;
- close(ctdb->recovery_lock_fd);
- ctdb->recovery_lock_fd = -1;
- /* Fail silently on these errors, since they indicate
- * lock contention, but log an error for any other
- * failure. */
- if (saved_errno != EACCES &&
- saved_errno != EAGAIN) {
- DEBUG(DEBUG_ERR,("ctdb_recovery_lock: Failed to get "
- "recovery lock on '%s' - (%s)\n",
- ctdb->recovery_lock_file,
- strerror(saved_errno)));
- }
- return false;
- }
-
- return true;
-}
-
-void ctdb_recovery_unlock(struct ctdb_context *ctdb)
-{
- if (ctdb->recovery_lock_fd != -1) {
- DEBUG(DEBUG_NOTICE, ("Releasing recovery lock\n"));
- close(ctdb->recovery_lock_fd);
- ctdb->recovery_lock_fd = -1;
- }
-}
-
/*
delete a record as part of the vacuum process
only delete if we are not lmaster or dmaster, and our rsn is <= the provided rsn
when the function returns)
or !0 is the record still exists in the tdb after returning.
*/
-static int delete_tdb_record(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, struct ctdb_rec_data *rec)
+static int delete_tdb_record(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, struct ctdb_rec_data_old *rec)
{
TDB_DATA key, data, data2;
struct ctdb_ltdb_header *hdr, *hdr2;
struct recovery_callback_state {
- struct ctdb_req_control *c;
+ struct ctdb_req_control_old *c;
};
recovery has finished
*/
int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb,
- struct ctdb_req_control *c,
+ struct ctdb_req_control_old *c,
bool *async_reply)
{
int ret;
talloc_free(state);
}
-/*
- run the startrecovery eventscript
- */
-int32_t ctdb_control_start_recovery(struct ctdb_context *ctdb,
- struct ctdb_req_control *c,
- bool *async_reply)
+static void run_start_recovery_event(struct ctdb_context *ctdb,
+ struct recovery_callback_state *state)
{
int ret;
- struct recovery_callback_state *state;
-
- DEBUG(DEBUG_NOTICE,(__location__ " startrecovery eventscript has been invoked\n"));
- gettimeofday(&ctdb->last_recovery_started, NULL);
-
- state = talloc(ctdb, struct recovery_callback_state);
- CTDB_NO_MEMORY(ctdb, state);
-
- state->c = talloc_steal(state, c);
ctdb_disable_monitoring(ctdb);
ret = ctdb_event_script_callback(ctdb, state,
- ctdb_start_recovery_callback,
+ ctdb_start_recovery_callback,
state,
CTDB_EVENT_START_RECOVERY,
"%s", "");
if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to start recovery\n"));
+ DEBUG(DEBUG_ERR,("Unable to run startrecovery event\n"));
+ ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
talloc_free(state);
- return -1;
+ return;
+ }
+
+ return;
+}
+
+static bool reclock_strings_equal(const char *a, const char *b)
+{
+ return (a == NULL && b == NULL) ||
+ (a != NULL && b != NULL && strcmp(a, b) == 0);
+}
+
+static void start_recovery_reclock_callback(struct ctdb_context *ctdb,
+ int32_t status,
+ TDB_DATA data,
+ const char *errormsg,
+ void *private_data)
+{
+ struct recovery_callback_state *state = talloc_get_type_abort(
+ private_data, struct recovery_callback_state);
+ const char *local = ctdb->recovery_lock;
+ const char *remote = NULL;
+
+ if (status != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " GET_RECLOCK failed\n"));
+ ctdb_request_control_reply(ctdb, state->c, NULL,
+ status, errormsg);
+ talloc_free(state);
+ return;
+ }
+
+ /* Check reclock consistency */
+ if (data.dsize > 0) {
+ /* Ensure NUL-termination */
+ data.dptr[data.dsize-1] = '\0';
+ remote = (const char *)data.dptr;
+ }
+ if (! reclock_strings_equal(local, remote)) {
+ /* Inconsistent */
+ ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
+ DEBUG(DEBUG_ERR,
+ ("Recovery lock configuration inconsistent: "
+ "recmaster has %s, this node has %s, shutting down\n",
+ remote == NULL ? "NULL" : remote,
+ local == NULL ? "NULL" : local));
+ talloc_free(state);
+ ctdb_shutdown_sequence(ctdb, 1);
+ }
+ DEBUG(DEBUG_INFO,
+ ("Recovery lock consistency check successful\n"));
+
+ run_start_recovery_event(ctdb, state);
+}
+
+/* Check recovery lock consistency and run eventscripts for the
+ * "startrecovery" event */
+int32_t ctdb_control_start_recovery(struct ctdb_context *ctdb,
+ struct ctdb_req_control_old *c,
+ bool *async_reply)
+{
+ int ret;
+ struct recovery_callback_state *state;
+ uint32_t recmaster = c->hdr.srcnode;
+
+ DEBUG(DEBUG_NOTICE, ("Running startrecovery event\n"));
+ gettimeofday(&ctdb->last_recovery_started, NULL);
+
+ state = talloc(ctdb, struct recovery_callback_state);
+ CTDB_NO_MEMORY(ctdb, state);
+
+ state->c = c;
+
+ /* Although the recovery master sent this node a start
+ * recovery control, this node might still think the recovery
+ * master is disconnected. In this case defer the recovery
+ * lock consistency check. */
+ if (ctdb->nodes[recmaster]->flags & NODE_FLAGS_DISCONNECTED) {
+ run_start_recovery_event(ctdb, state);
+ } else {
+ /* Ask the recovery master about its reclock setting */
+ ret = ctdb_daemon_send_control(ctdb,
+ recmaster,
+ 0,
+ CTDB_CONTROL_GET_RECLOCK_FILE,
+ 0, 0,
+ tdb_null,
+ start_recovery_reclock_callback,
+ state);
+
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " GET_RECLOCK failed\n"));
+ talloc_free(state);
+ return -1;
+ }
}
/* tell the control that we will be reply asynchronously */
+ state->c = talloc_steal(state, c);
*async_reply = true;
+
return 0;
}
struct ctdb_marshall_buffer *reply = (struct ctdb_marshall_buffer *)indata.dptr;
struct ctdb_db_context *ctdb_db;
int i;
- struct ctdb_rec_data *rec;
+ struct ctdb_rec_data_old *rec;
struct ctdb_marshall_buffer *records;
if (indata.dsize < offsetof(struct ctdb_marshall_buffer, data)) {
records->db_id = ctdb_db->db_id;
- rec = (struct ctdb_rec_data *)&reply->data[0];
+ rec = (struct ctdb_rec_data_old *)&reply->data[0];
for (i=0;i<reply->count;i++) {
TDB_DATA key, data;
memcpy(old_size+(uint8_t *)records, rec, rec->length);
}
- rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec);
+ rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec);
}
*/
static int store_tdb_record(struct ctdb_context *ctdb,
struct ctdb_db_context *ctdb_db,
- struct ctdb_rec_data *rec)
+ struct ctdb_rec_data_old *rec)
{
TDB_DATA key, data, data2;
struct ctdb_ltdb_header *hdr, *hdr2;
struct ctdb_marshall_buffer *reply = (struct ctdb_marshall_buffer *)indata.dptr;
struct ctdb_db_context *ctdb_db;
int i;
- struct ctdb_rec_data *rec;
+ struct ctdb_rec_data_old *rec;
struct ctdb_marshall_buffer *records;
if (indata.dsize < offsetof(struct ctdb_marshall_buffer, data)) {
}
records->db_id = ctdb_db->db_id;
- rec = (struct ctdb_rec_data *)&reply->data[0];
+ rec = (struct ctdb_rec_data_old *)&reply->data[0];
for (i=0; i<reply->count; i++) {
TDB_DATA key, data;
memcpy(old_size+(uint8_t *)records, rec, rec->length);
}
- rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec);
+ rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec);
}
*outdata = ctdb_marshall_finish(records);
If we havent been pinged for a while we assume the recovery
daemon is inoperable and we restart.
*/
-static void ctdb_recd_ping_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
+static void ctdb_recd_ping_timeout(struct tevent_context *ev,
+ struct tevent_timer *te,
+ struct timeval t, void *p)
{
struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
uint32_t *count = talloc_get_type(ctdb->recd_ping_count, uint32_t);
if (*count < ctdb->tunable.recd_ping_failcount) {
(*count)++;
- event_add_timed(ctdb->ev, ctdb->recd_ping_count,
- timeval_current_ofs(ctdb->tunable.recd_ping_timeout, 0),
- ctdb_recd_ping_timeout, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->recd_ping_count,
+ timeval_current_ofs(ctdb->tunable.recd_ping_timeout, 0),
+ ctdb_recd_ping_timeout, ctdb);
return;
}
CTDB_NO_MEMORY(ctdb, ctdb->recd_ping_count);
if (ctdb->tunable.recd_ping_timeout != 0) {
- event_add_timed(ctdb->ev, ctdb->recd_ping_count,
- timeval_current_ofs(ctdb->tunable.recd_ping_timeout, 0),
- ctdb_recd_ping_timeout, ctdb);
+ tevent_add_timer(ctdb->ev, ctdb->recd_ping_count,
+ timeval_current_ofs(ctdb->tunable.recd_ping_timeout, 0),
+ ctdb_recd_ping_timeout, ctdb);
}
return 0;
if (ctdb->pnn != new_recmaster && ctdb->recovery_master == ctdb->pnn) {
DEBUG(DEBUG_NOTICE,
- ("This node (%u) is no longer the recovery master\n", ctdb->pnn));
+ ("Remote node (%u) is now the recovery master\n",
+ new_recmaster));
}
if (ctdb->pnn == new_recmaster && ctdb->recovery_master != new_recmaster) {
DEBUG(DEBUG_NOTICE,
- ("This node (%u) is now the recovery master\n", ctdb->pnn));
+ ("This node (%u) is now the recovery master\n",
+ ctdb->pnn));
}
ctdb->recovery_master = new_recmaster;
git.samba.org / obnox / samba / samba-obnox.git / blobdiff