X-Git-Url: http://git.samba.org/?a=blobdiff_plain;f=tools%2Fctdb.c;h=80d196ac6492339b00c12a601ecd49d1d222a503;hb=775bd75d4cbc5ee3bf6007ea71aa70af9581ee07;hp=5e4b9edd1d4b9d6b5b3457cbe0acd616e56736b1;hpb=e1ca391136eab60b586eaa263bed33219285c0bd;p=sahlberg%2Fctdb.git

diff --git a/tools/ctdb.c b/tools/ctdb.c
index 5e4b9edd..80d196ac 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -1290,6 +1290,7 @@ static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn
 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
 {
 	uint32_t pnn;
+	int ret, retries = 0;
 	ctdb_sock_addr addr;
 
 	if (argc < 2) {
@@ -1308,8 +1309,16 @@ static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv
 		return -1;
 	}
 
-	if (move_ip(ctdb, &addr, pnn) != 0) {
-		DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
+	do {
+		ret = move_ip(ctdb, &addr, pnn);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Wait 3 second and try again.\n", pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Giving up.\n", pnn));
 		return -1;
 	}
 
@@ -1616,7 +1625,7 @@ again:
 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 {
 	int i, ret;
-	int len;
+	int len, retries = 0;
 	uint32_t pnn;
 	unsigned mask;
 	ctdb_sock_addr addr;
@@ -1662,9 +1671,16 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 	pub->len   = strlen(argv[1])+1;
 	memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
 
-	ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
+	do {
+		ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
 	if (ret != 0) {
-		DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u\n", options.pnn));
+		DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
 		talloc_free(tmp_ctx);
 		return ret;
 	}
@@ -1676,14 +1692,31 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 		pnn  = ips->ips[i].pnn;
 	}
 
-	if (move_ip(ctdb, &addr, pnn) != 0) {
-		DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
-		return -1;
+	do {
+		ret = move_ip(ctdb, &addr, pnn);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR,("Failed to move ip to node %d. wait 3 seconds and try again.\n", pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Giving up.\n", pnn));
+		talloc_free(tmp_ctx);
+		return ret;
 	}
 
-	ret = control_ipreallocate(ctdb, argc, argv);
+	do {
+		ret = control_ipreallocate(ctdb, argc, argv);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Wait 3 seconds and try again.\n", options.pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
 	if (ret != 0) {
-		DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
+		DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Giving up.\n", options.pnn));
+		talloc_free(tmp_ctx);
 		return ret;
 	}
 
@@ -1767,6 +1800,7 @@ static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **a
 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
 {
 	int i, ret;
+	int retries = 0;
 	ctdb_sock_addr addr;
 	struct ctdb_control_ip_iface pub;
 	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
@@ -1813,8 +1847,16 @@ static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
 	if (ips->ips[i].pnn == options.pnn) {
 		ret = find_other_host_for_public_ip(ctdb, &addr);
 		if (ret != -1) {
-			if (move_ip(ctdb, &addr, ret) != 0) {
-				DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", ret));
+			do {
+				ret = move_ip(ctdb, &addr, ret);
+				if (ret != 0) {
+					DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Wait 3 seconds and try again.\n", options.pnn));
+					sleep(3);
+					retries++;
+				}
+			} while (retries < 5 && ret != 0);
+			if (ret != 0) {
+				DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Giving up.\n", options.pnn));
 				return -1;
 			}
 		}