protocol design and packet details
*/
#include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
#include "lib/tdb/include/tdb.h"
#include "lib/util/dlinklist.h"
#include "system/network.h"
int msglen, len;
if (ctdb->methods == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to send error. Transport is DOWN\n"));
+ DEBUG(DEBUG_INFO,(__location__ " Failed to send error. Transport is DOWN\n"));
return;
}
}
if (ctdb->methods == NULL) {
- ctdb_fatal(ctdb, "ctdb_send_dmaster_reply cant update dmaster sicne transport is down");
+ ctdb_fatal(ctdb, "ctdb_send_dmaster_reply cant update dmaster since transport is down");
return;
}
talloc_free(r);
}
-static void ctdb_hold_back_key(struct ctdb_db_context *db, TDB_DATA key)
-{
- size_t num_keys;
- uint8_t **tmp;
-
- DEBUG(DEBUG_INFO, ("Holding back key %08x\n", ctdb_hash(&key)));
-
- num_keys = talloc_array_length(db->holdback_keys);
- tmp = talloc_realloc(db, db->holdback_keys, uint8_t *, num_keys+1);
- if (tmp == NULL) {
- DEBUG(DEBUG_ERR, ("talloc_realloc failed\n"));
- return;
- }
- db->holdback_keys = tmp;
-
- db->holdback_keys[num_keys] = (uint8_t *)talloc_memdup(
- db->holdback_keys, key.dptr, key.dsize);
- if (db->holdback_keys[num_keys] == NULL) {
- DEBUG(DEBUG_ERR, ("talloc_memdup failed\n"));
- db->holdback_keys = talloc_realloc(db, db->holdback_keys,
- uint8_t *, num_keys);
- }
-}
-
/*
called when a CTDB_REPLY_DMASTER packet comes in, or when the lmaster
gets a CTDB_REQUEST_DMASTER for itself. We become the dmaster.
struct ctdb_call_state *state;
struct ctdb_context *ctdb = ctdb_db->ctdb;
struct ctdb_ltdb_header header;
+ int ret;
- DEBUG(DEBUG_INFO,("pnn %u dmaster response %08x\n", ctdb->pnn, ctdb_hash(&key)));
+ DEBUG(DEBUG_DEBUG,("pnn %u dmaster response %08x\n", ctdb->pnn, ctdb_hash(&key)));
ZERO_STRUCT(header);
header.rsn = rsn + 1;
if (ctdb_ltdb_store(ctdb_db, key, &header, data) != 0) {
ctdb_fatal(ctdb, "ctdb_reply_dmaster store failed\n");
- ctdb_ltdb_unlock(ctdb_db, key);
+
+ ret = ctdb_ltdb_unlock(ctdb_db, key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
return;
}
if (state == NULL) {
DEBUG(DEBUG_ERR,("pnn %u Invalid reqid %u in ctdb_become_dmaster from node %u\n",
ctdb->pnn, hdr->reqid, hdr->srcnode));
- ctdb_ltdb_unlock(ctdb_db, key);
+
+ ret = ctdb_ltdb_unlock(ctdb_db, key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
+ return;
+ }
+
+ if (key.dsize != state->call->key.dsize || memcmp(key.dptr, state->call->key.dptr, key.dsize)) {
+ DEBUG(DEBUG_ERR, ("Got bogus DMASTER packet reqid:%u from node %u. Key does not match key held in matching idr.\n", hdr->reqid, hdr->srcnode));
+
+ ret = ctdb_ltdb_unlock(ctdb_db, key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
return;
}
if (hdr->reqid != state->reqid) {
/* we found a record but it was the wrong one */
DEBUG(DEBUG_ERR, ("Dropped orphan in ctdb_become_dmaster with reqid:%u\n from node %u", hdr->reqid, hdr->srcnode));
- ctdb_ltdb_unlock(ctdb_db, key);
+
+ ret = ctdb_ltdb_unlock(ctdb_db, key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
return;
}
- ctdb_hold_back_key(ctdb_db, key);
-
- ctdb_call_local(ctdb_db, state->call, &header, state, &data, ctdb->pnn);
+ ctdb_call_local(ctdb_db, state->call, &header, state, &data);
- ctdb_ltdb_unlock(ctdb_db, state->call->key);
+ ret = ctdb_ltdb_unlock(ctdb_db, state->call->key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
state->state = CTDB_CALL_DONE;
if (state->async.fn) {
ctdb_fatal(ctdb, "ctdb_req_dmaster to non-lmaster");
}
- DEBUG(DEBUG_INFO,("pnn %u dmaster request on %08x for %u from %u\n",
+ DEBUG(DEBUG_DEBUG,("pnn %u dmaster request on %08x for %u from %u\n",
ctdb->pnn, ctdb_hash(&key), c->dmaster, c->hdr.srcnode));
/* its a protocol error if the sending node is not the current dmaster */
(unsigned long long)c->rsn, (unsigned long long)header.rsn, c->hdr.reqid,
(key.dsize >= 4)?(*(uint32_t *)key.dptr):0));
if (header.rsn != 0 || header.dmaster != ctdb->pnn) {
- ctdb_fatal(ctdb, "ctdb_req_dmaster from non-master");
+ DEBUG(DEBUG_ERR,("ctdb_req_dmaster from non-master. Force a recovery.\n"));
+
+ ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
return;
}
}
ctdb_become_dmaster(ctdb_db, hdr, key, data, c->rsn);
} else {
ctdb_send_dmaster_reply(ctdb_db, &header, key, data, c->dmaster, hdr->reqid);
- ctdb_ltdb_unlock(ctdb_db, key);
- }
-}
-
-/*
- * Did we just pull the dmaster of the record for a client fetch_lock,
- * so should we hold it back a while and thus give our client the
- * chance to do its own tdb_lock?
- */
-
-static bool ctdb_held_back(struct ctdb_db_context *db, TDB_DATA key,
- struct ctdb_req_header *hdr)
-{
- int i;
- size_t num_keys = talloc_array_length(db->holdback_keys);
- size_t num_hdrs;
- struct ctdb_req_header **tmp;
-
- for (i=0; i<num_keys; i++) {
- uint8_t *hold_key = db->holdback_keys[i];
- size_t keylength = talloc_array_length(hold_key);
- if ((keylength == key.dsize)
- && (memcmp(hold_key, key.dptr, keylength) == 0)) {
- break;
+ ret = ctdb_ltdb_unlock(ctdb_db, key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
}
}
- if (i == num_keys) {
- return false;
- }
- DEBUG(DEBUG_DEBUG, ("holding back record %08x after migration\n",
- ctdb_hash(&key)));
-
- num_hdrs = talloc_array_length(db->held_back);
-
- tmp = talloc_realloc(db, db->held_back, struct ctdb_req_header *,
- num_hdrs + 1);
- if (tmp == NULL) {
- DEBUG(DEBUG_ERR, (__location__ "talloc_realloc failed\n"));
- return false;
- }
- db->held_back = tmp;
- db->held_back[num_hdrs] = talloc_move(db->held_back, &hdr);
- return true;
}
+
/*
called when a CTDB_REQ_CALL packet comes in
*/
struct ctdb_db_context *ctdb_db;
if (ctdb->methods == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed ctdb_request_call. Transport is DOWN\n"));
+ DEBUG(DEBUG_INFO,(__location__ " Failed ctdb_request_call. Transport is DOWN\n"));
return;
}
if (header.dmaster != ctdb->pnn) {
talloc_free(data.dptr);
ctdb_call_send_redirect(ctdb, call->key, c, &header);
- ctdb_ltdb_unlock(ctdb_db, call->key);
- return;
- }
-
- if (c->hopcount > ctdb->statistics.max_hop_count) {
- ctdb->statistics.max_hop_count = c->hopcount;
- }
- if ((c->flags & CTDB_IMMEDIATE_MIGRATION)
- && (ctdb_held_back(ctdb_db, call->key, hdr))) {
- talloc_free(data.dptr);
- ctdb_ltdb_unlock(ctdb_db, call->key);
+ ret = ctdb_ltdb_unlock(ctdb_db, call->key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
return;
}
- /* if this nodes has done enough consecutive calls on the same record
- then give them the record
- or if the node requested an immediate migration
- */
- if ( c->hdr.srcnode != ctdb->pnn &&
- ((header.laccessor == c->hdr.srcnode
- && header.lacount >= ctdb->tunable.max_lacount)
- || (c->flags & CTDB_IMMEDIATE_MIGRATION)) ) {
+ CTDB_UPDATE_STAT(ctdb, max_hop_count, c->hopcount);
+
+ /* Try if possible to migrate the record off to the caller node.
+ * From the clients perspective a fetch of the data is just as
+ * expensive as a migration.
+ */
+ if (c->hdr.srcnode != ctdb->pnn) {
if (ctdb_db->transaction_active) {
DEBUG(DEBUG_INFO, (__location__ " refusing migration"
" of key %s while transaction is active\n",
(char *)call->key.dptr));
} else {
- DEBUG(DEBUG_INFO,("pnn %u starting migration of %08x to %u\n",
+ DEBUG(DEBUG_DEBUG,("pnn %u starting migration of %08x to %u\n",
ctdb->pnn, ctdb_hash(&(call->key)), c->hdr.srcnode));
ctdb_call_send_dmaster(ctdb_db, c, &header, &(call->key), &data);
talloc_free(data.dptr);
- ctdb_ltdb_unlock(ctdb_db, call->key);
+
+ ret = ctdb_ltdb_unlock(ctdb_db, call->key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
return;
}
}
- ctdb_call_local(ctdb_db, call, &header, hdr, &data, c->hdr.srcnode);
+ ctdb_call_local(ctdb_db, call, &header, hdr, &data);
- ctdb_ltdb_unlock(ctdb_db, call->key);
+ ret = ctdb_ltdb_unlock(ctdb_db, call->key);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " ctdb_ltdb_unlock() failed with error %d\n", ret));
+ }
len = offsetof(struct ctdb_reply_call, data) + call->reply_data.dsize;
r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CALL, len,
*(state->call) = *call;
state->ctdb_db = ctdb_db;
- ret = ctdb_call_local(ctdb_db, state->call, header, state, data, ctdb->pnn);
+ ret = ctdb_call_local(ctdb_db, state->call, header, state, data);
event_add_timed(ctdb->ev, state, timeval_zero(), call_local_trigger, state);
struct ctdb_context *ctdb = ctdb_db->ctdb;
if (ctdb->methods == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed send packet. Transport is down\n"));
+ DEBUG(DEBUG_INFO,(__location__ " Failed send packet. Transport is down\n"));
return NULL;
}
struct ctdb_req_keepalive *r;
if (ctdb->methods == NULL) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to send keepalive. Transport is DOWN\n"));
+ DEBUG(DEBUG_INFO,(__location__ " Failed to send keepalive. Transport is DOWN\n"));
return;
}
r->hdr.destnode = destnode;
r->hdr.reqid = 0;
- ctdb->statistics.keepalive_packets_sent++;
+ CTDB_INCREMENT_STAT(ctdb, keepalive_packets_sent);
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(r);
}
-
-static void ctdb_holdback_cleanup(struct event_context *ev,
- struct timed_event *te,
- struct timeval t, void *private_data)
-{
- struct ctdb_context *ctdb = talloc_get_type(private_data,
- struct ctdb_context);
- struct ctdb_db_context *ctdb_db;
-
- DEBUG(DEBUG_INFO, ("running ctdb_holdback_cleanup\n"));
-
- if (te != ctdb->holdback_cleanup_te) {
- ctdb_fatal(ctdb, "te != ctdb->holdback_cleanup_te");
- }
-
- for (ctdb_db=ctdb->db_list; ctdb_db; ctdb_db=ctdb_db->next) {
- size_t i, num_heldback;
-
- talloc_free(ctdb_db->holdback_keys);
- ctdb_db->holdback_keys = NULL;
-
- num_heldback = talloc_array_length(ctdb_db->held_back);
- for (i=0; i<num_heldback; i++) {
- ctdb_queue_packet(ctdb, ctdb_db->held_back[i]);
- }
- talloc_free(ctdb_db->held_back);
- ctdb_db->held_back = NULL;
- }
-
- ctdb->holdback_cleanup_te = event_add_timed(
- ctdb->ev, ctdb, timeval_current_ofs(
- 0, ctdb->tunable.holdback_cleanup_interval * 1000),
- ctdb_holdback_cleanup, ctdb);
-}
-
-void ctdb_start_holdback_cleanup(struct ctdb_context *ctdb)
-{
- ctdb->holdback_cleanup_te = event_add_timed(
- ctdb->ev, ctdb, timeval_current_ofs(
- 0, ctdb->tunable.holdback_cleanup_interval * 1000),
- ctdb_holdback_cleanup, ctdb);
-
- CTDB_NO_MEMORY_FATAL(ctdb, ctdb->holdback_cleanup_te);
-
- DEBUG(DEBUG_NOTICE,("Holdback cleanup has been started\n"));
-}
-
-void ctdb_stop_holdback_cleanup(struct ctdb_context *ctdb)
-{
- talloc_free(ctdb->holdback_cleanup_te);
- ctdb->holdback_cleanup_te = NULL;
-}
-
-int32_t ctdb_control_gotit(struct ctdb_context *ctdb, TDB_DATA indata)
-{
- struct ctdb_control_gotit *c =
- (struct ctdb_control_gotit *)indata.dptr;
- struct ctdb_db_context *db;
- size_t i, num_keys, num_heldback;
- TDB_DATA key;
-
- if (indata.dsize < sizeof(struct ctdb_control_gotit)) {
- DEBUG(DEBUG_ERR, (__location__ "Invalid data size %d\n",
- (int)indata.dsize));
- return -1;
- }
- db = find_ctdb_db(ctdb, c->db_id);
- if (db == NULL) {
- DEBUG(DEBUG_ERR, ("Unknown db_id 0x%x in ctdb_reply_dmaster\n",
- (int)c->db_id));
- return -1;
- }
-
- key.dptr = c->key;
- key.dsize = indata.dsize - offsetof(struct ctdb_control_gotit, key);
-
- num_keys = talloc_array_length(db->holdback_keys);
- for (i=0; i<num_keys; i++) {
- uint8_t *hold_key = db->holdback_keys[i];
- size_t keylength = talloc_array_length(hold_key);
-
- if ((keylength == key.dsize)
- && (memcmp(hold_key, key.dptr, keylength) == 0)) {
- break;
- }
- }
- if (i == num_keys) {
- /*
- * ctdb_holdback_cleanup has kicked in. This is okay,
- * we will just potentially have to retry.
- */
- return 0;
- }
-
- talloc_free(db->holdback_keys[i]);
- if (i < num_keys-1) {
- db->holdback_keys[i] = db->holdback_keys[num_keys-1];
- }
- db->holdback_keys = talloc_realloc(db, db->holdback_keys, uint8_t *,
- num_keys-1);
-
- num_heldback = talloc_array_length(db->held_back);
- for (i=0; i<num_heldback; i++) {
- ctdb_queue_packet(ctdb, db->held_back[i]);
- }
- talloc_free(db->held_back);
- db->held_back = NULL;
- return 0;
-}
git.samba.org / sahlberg / ctdb.git / blobdiff