4 Copyright (C) Ronnie Sahlberg 2007
5 Copyright (C) Andrew Tridgell 2007
6 Copyright (C) Martin Schwenke 2011
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #include "system/network.h"
23 #include "system/filesys.h"
24 #include "system/time.h"
25 #include "system/wait.h"
30 #include "lib/util/dlinklist.h"
31 #include "lib/util/debug.h"
32 #include "lib/util/samba_util.h"
34 #include "ctdb_private.h"
35 #include "ctdb_client.h"
36 #include "ctdb_logging.h"
38 #include "common/rb_tree.h"
39 #include "common/reqid.h"
40 #include "common/system.h"
41 #include "common/common.h"
44 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
46 #define CTDB_ARP_INTERVAL 1
47 #define CTDB_ARP_REPEAT 3
49 /* Flags used in IP allocation algorithms. */
53 enum ctdb_runstate runstate;
56 struct ctdb_interface {
57 struct ctdb_interface *prev, *next;
63 static const char *ctdb_vnn_iface_string(const struct ctdb_vnn *vnn)
66 return vnn->iface->name;
72 static int ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
74 struct ctdb_interface *i;
76 /* Verify that we dont have an entry for this ip yet */
77 for (i=ctdb->ifaces;i;i=i->next) {
78 if (strcmp(i->name, iface) == 0) {
83 /* create a new structure for this interface */
84 i = talloc_zero(ctdb, struct ctdb_interface);
85 CTDB_NO_MEMORY_FATAL(ctdb, i);
86 i->name = talloc_strdup(i, iface);
87 CTDB_NO_MEMORY(ctdb, i->name);
91 DLIST_ADD(ctdb->ifaces, i);
96 static bool vnn_has_interface_with_name(struct ctdb_vnn *vnn,
101 for (n = 0; vnn->ifaces[n] != NULL; n++) {
102 if (strcmp(name, vnn->ifaces[n]) == 0) {
110 /* If any interfaces now have no possible IPs then delete them. This
111 * implementation is naive (i.e. simple) rather than clever
112 * (i.e. complex). Given that this is run on delip and that operation
113 * is rare, this doesn't need to be efficient - it needs to be
114 * foolproof. One alternative is reference counting, where the logic
115 * is distributed and can, therefore, be broken in multiple places.
116 * Another alternative is to build a red-black tree of interfaces that
117 * can have addresses (by walking ctdb->vnn and ctdb->single_ip_vnn
118 * once) and then walking ctdb->ifaces once and deleting those not in
119 * the tree. Let's go to one of those if the naive implementation
120 * causes problems... :-)
122 static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
123 struct ctdb_vnn *vnn)
125 struct ctdb_interface *i, *next;
127 /* For each interface, check if there's an IP using it. */
128 for (i = ctdb->ifaces; i != NULL; i = next) {
133 /* Only consider interfaces named in the given VNN. */
134 if (!vnn_has_interface_with_name(vnn, i->name)) {
138 /* Is the "single IP" on this interface? */
139 if ((ctdb->single_ip_vnn != NULL) &&
140 (ctdb->single_ip_vnn->ifaces[0] != NULL) &&
141 (strcmp(i->name, ctdb->single_ip_vnn->ifaces[0]) == 0)) {
142 /* Found, next interface please... */
145 /* Search for a vnn with this interface. */
147 for (tv=ctdb->vnn; tv; tv=tv->next) {
148 if (vnn_has_interface_with_name(tv, i->name)) {
155 /* None of the VNNs are using this interface. */
156 DLIST_REMOVE(ctdb->ifaces, i);
163 static struct ctdb_interface *ctdb_find_iface(struct ctdb_context *ctdb,
166 struct ctdb_interface *i;
168 for (i=ctdb->ifaces;i;i=i->next) {
169 if (strcmp(i->name, iface) == 0) {
177 static struct ctdb_interface *ctdb_vnn_best_iface(struct ctdb_context *ctdb,
178 struct ctdb_vnn *vnn)
181 struct ctdb_interface *cur = NULL;
182 struct ctdb_interface *best = NULL;
184 for (i=0; vnn->ifaces[i]; i++) {
186 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
200 if (cur->references < best->references) {
209 static int32_t ctdb_vnn_assign_iface(struct ctdb_context *ctdb,
210 struct ctdb_vnn *vnn)
212 struct ctdb_interface *best = NULL;
215 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
216 "still assigned to iface '%s'\n",
217 ctdb_addr_to_str(&vnn->public_address),
218 ctdb_vnn_iface_string(vnn)));
222 best = ctdb_vnn_best_iface(ctdb, vnn);
224 DEBUG(DEBUG_ERR, (__location__ " public address '%s' "
225 "cannot assign to iface any iface\n",
226 ctdb_addr_to_str(&vnn->public_address)));
232 vnn->pnn = ctdb->pnn;
234 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
235 "now assigned to iface '%s' refs[%d]\n",
236 ctdb_addr_to_str(&vnn->public_address),
237 ctdb_vnn_iface_string(vnn),
242 static void ctdb_vnn_unassign_iface(struct ctdb_context *ctdb,
243 struct ctdb_vnn *vnn)
245 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
246 "now unassigned (old iface '%s' refs[%d])\n",
247 ctdb_addr_to_str(&vnn->public_address),
248 ctdb_vnn_iface_string(vnn),
249 vnn->iface?vnn->iface->references:0));
251 vnn->iface->references--;
254 if (vnn->pnn == ctdb->pnn) {
259 static bool ctdb_vnn_available(struct ctdb_context *ctdb,
260 struct ctdb_vnn *vnn)
264 if (vnn->delete_pending) {
268 if (vnn->iface && vnn->iface->link_up) {
272 for (i=0; vnn->ifaces[i]; i++) {
273 struct ctdb_interface *cur;
275 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
288 struct ctdb_takeover_arp {
289 struct ctdb_context *ctdb;
292 struct ctdb_tcp_array *tcparray;
293 struct ctdb_vnn *vnn;
298 lists of tcp endpoints
300 struct ctdb_tcp_list {
301 struct ctdb_tcp_list *prev, *next;
302 struct ctdb_connection connection;
306 list of clients to kill on IP release
308 struct ctdb_client_ip {
309 struct ctdb_client_ip *prev, *next;
310 struct ctdb_context *ctdb;
317 send a gratuitous arp
319 static void ctdb_control_send_arp(struct tevent_context *ev,
320 struct tevent_timer *te,
321 struct timeval t, void *private_data)
323 struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
324 struct ctdb_takeover_arp);
326 struct ctdb_tcp_array *tcparray;
327 const char *iface = ctdb_vnn_iface_string(arp->vnn);
329 ret = ctdb_sys_send_arp(&arp->addr, iface);
331 DEBUG(DEBUG_CRIT,(__location__ " sending of arp failed on iface '%s' (%s)\n",
332 iface, strerror(errno)));
335 tcparray = arp->tcparray;
337 for (i=0;i<tcparray->num;i++) {
338 struct ctdb_connection *tcon;
340 tcon = &tcparray->connections[i];
341 DEBUG(DEBUG_INFO,("sending tcp tickle ack for %u->%s:%u\n",
342 (unsigned)ntohs(tcon->dst.ip.sin_port),
343 ctdb_addr_to_str(&tcon->src),
344 (unsigned)ntohs(tcon->src.ip.sin_port)));
345 ret = ctdb_sys_send_tcp(
350 DEBUG(DEBUG_CRIT,(__location__ " Failed to send tcp tickle ack for %s\n",
351 ctdb_addr_to_str(&tcon->src)));
358 if (arp->count == CTDB_ARP_REPEAT) {
363 tevent_add_timer(arp->ctdb->ev, arp->vnn->takeover_ctx,
364 timeval_current_ofs(CTDB_ARP_INTERVAL, 100000),
365 ctdb_control_send_arp, arp);
368 static int32_t ctdb_announce_vnn_iface(struct ctdb_context *ctdb,
369 struct ctdb_vnn *vnn)
371 struct ctdb_takeover_arp *arp;
372 struct ctdb_tcp_array *tcparray;
374 if (!vnn->takeover_ctx) {
375 vnn->takeover_ctx = talloc_new(vnn);
376 if (!vnn->takeover_ctx) {
381 arp = talloc_zero(vnn->takeover_ctx, struct ctdb_takeover_arp);
387 arp->addr = vnn->public_address;
390 tcparray = vnn->tcp_array;
392 /* add all of the known tcp connections for this IP to the
393 list of tcp connections to send tickle acks for */
394 arp->tcparray = talloc_steal(arp, tcparray);
396 vnn->tcp_array = NULL;
397 vnn->tcp_update_needed = true;
400 tevent_add_timer(arp->ctdb->ev, vnn->takeover_ctx,
401 timeval_zero(), ctdb_control_send_arp, arp);
406 struct takeover_callback_state {
407 struct ctdb_req_control_old *c;
408 ctdb_sock_addr *addr;
409 struct ctdb_vnn *vnn;
412 struct ctdb_do_takeip_state {
413 struct ctdb_req_control_old *c;
414 struct ctdb_vnn *vnn;
418 called when takeip event finishes
420 static void ctdb_do_takeip_callback(struct ctdb_context *ctdb, int status,
423 struct ctdb_do_takeip_state *state =
424 talloc_get_type(private_data, struct ctdb_do_takeip_state);
429 struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
431 if (status == -ETIME) {
434 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
435 ctdb_addr_to_str(&state->vnn->public_address),
436 ctdb_vnn_iface_string(state->vnn)));
437 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
439 node->flags |= NODE_FLAGS_UNHEALTHY;
444 if (ctdb->do_checkpublicip) {
446 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
448 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
455 data.dptr = (uint8_t *)ctdb_addr_to_str(&state->vnn->public_address);
456 data.dsize = strlen((char *)data.dptr) + 1;
457 DEBUG(DEBUG_INFO,(__location__ " sending TAKE_IP for '%s'\n", data.dptr));
459 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_TAKE_IP, data);
462 /* the control succeeded */
463 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
468 static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
470 state->vnn->update_in_flight = false;
475 take over an ip address
477 static int32_t ctdb_do_takeip(struct ctdb_context *ctdb,
478 struct ctdb_req_control_old *c,
479 struct ctdb_vnn *vnn)
482 struct ctdb_do_takeip_state *state;
484 if (vnn->update_in_flight) {
485 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
486 "update for this IP already in flight\n",
487 ctdb_addr_to_str(&vnn->public_address),
488 vnn->public_netmask_bits));
492 ret = ctdb_vnn_assign_iface(ctdb, vnn);
494 DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
495 "assign a usable interface\n",
496 ctdb_addr_to_str(&vnn->public_address),
497 vnn->public_netmask_bits));
501 state = talloc(vnn, struct ctdb_do_takeip_state);
502 CTDB_NO_MEMORY(ctdb, state);
504 state->c = talloc_steal(ctdb, c);
507 vnn->update_in_flight = true;
508 talloc_set_destructor(state, ctdb_takeip_destructor);
510 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
511 ctdb_addr_to_str(&vnn->public_address),
512 vnn->public_netmask_bits,
513 ctdb_vnn_iface_string(vnn)));
515 ret = ctdb_event_script_callback(ctdb,
517 ctdb_do_takeip_callback,
521 ctdb_vnn_iface_string(vnn),
522 ctdb_addr_to_str(&vnn->public_address),
523 vnn->public_netmask_bits);
526 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
527 ctdb_addr_to_str(&vnn->public_address),
528 ctdb_vnn_iface_string(vnn)));
536 struct ctdb_do_updateip_state {
537 struct ctdb_req_control_old *c;
538 struct ctdb_interface *old;
539 struct ctdb_vnn *vnn;
543 called when updateip event finishes
545 static void ctdb_do_updateip_callback(struct ctdb_context *ctdb, int status,
548 struct ctdb_do_updateip_state *state =
549 talloc_get_type(private_data, struct ctdb_do_updateip_state);
553 if (status == -ETIME) {
556 DEBUG(DEBUG_ERR,(__location__ " Failed to move IP %s from interface %s to %s\n",
557 ctdb_addr_to_str(&state->vnn->public_address),
559 ctdb_vnn_iface_string(state->vnn)));
562 * All we can do is reset the old interface
563 * and let the next run fix it
565 ctdb_vnn_unassign_iface(ctdb, state->vnn);
566 state->vnn->iface = state->old;
567 state->vnn->iface->references++;
569 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
574 if (ctdb->do_checkpublicip) {
576 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
578 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
585 /* the control succeeded */
586 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
591 static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
593 state->vnn->update_in_flight = false;
598 update (move) an ip address
600 static int32_t ctdb_do_updateip(struct ctdb_context *ctdb,
601 struct ctdb_req_control_old *c,
602 struct ctdb_vnn *vnn)
605 struct ctdb_do_updateip_state *state;
606 struct ctdb_interface *old = vnn->iface;
607 const char *new_name;
609 if (vnn->update_in_flight) {
610 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
611 "update for this IP already in flight\n",
612 ctdb_addr_to_str(&vnn->public_address),
613 vnn->public_netmask_bits));
617 ctdb_vnn_unassign_iface(ctdb, vnn);
618 ret = ctdb_vnn_assign_iface(ctdb, vnn);
620 DEBUG(DEBUG_ERR,("update of IP %s/%u failed to "
621 "assin a usable interface (old iface '%s')\n",
622 ctdb_addr_to_str(&vnn->public_address),
623 vnn->public_netmask_bits,
628 new_name = ctdb_vnn_iface_string(vnn);
629 if (old->name != NULL && new_name != NULL && !strcmp(old->name, new_name)) {
630 /* A benign update from one interface onto itself.
631 * no need to run the eventscripts in this case, just return
634 ctdb_request_control_reply(ctdb, c, NULL, 0, NULL);
638 state = talloc(vnn, struct ctdb_do_updateip_state);
639 CTDB_NO_MEMORY(ctdb, state);
641 state->c = talloc_steal(ctdb, c);
645 vnn->update_in_flight = true;
646 talloc_set_destructor(state, ctdb_updateip_destructor);
648 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
649 "interface %s to %s\n",
650 ctdb_addr_to_str(&vnn->public_address),
651 vnn->public_netmask_bits,
655 ret = ctdb_event_script_callback(ctdb,
657 ctdb_do_updateip_callback,
659 CTDB_EVENT_UPDATE_IP,
663 ctdb_addr_to_str(&vnn->public_address),
664 vnn->public_netmask_bits);
666 DEBUG(DEBUG_ERR,(__location__ " Failed update IP %s from interface %s to %s\n",
667 ctdb_addr_to_str(&vnn->public_address),
668 old->name, new_name));
677 Find the vnn of the node that has a public ip address
678 returns -1 if the address is not known as a public address
680 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
682 struct ctdb_vnn *vnn;
684 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
685 if (ctdb_same_ip(&vnn->public_address, addr)) {
694 take over an ip address
696 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
697 struct ctdb_req_control_old *c,
702 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
703 struct ctdb_vnn *vnn;
704 bool have_ip = false;
705 bool do_updateip = false;
706 bool do_takeip = false;
707 struct ctdb_interface *best_iface = NULL;
709 if (pip->pnn != ctdb->pnn) {
710 DEBUG(DEBUG_ERR,(__location__" takeoverip called for an ip '%s' "
711 "with pnn %d, but we're node %d\n",
712 ctdb_addr_to_str(&pip->addr),
713 pip->pnn, ctdb->pnn));
717 /* update out vnn list */
718 vnn = find_public_ip_vnn(ctdb, &pip->addr);
720 DEBUG(DEBUG_INFO,("takeoverip called for an ip '%s' that is not a public address\n",
721 ctdb_addr_to_str(&pip->addr)));
725 if (ctdb->tunable.disable_ip_failover == 0 && ctdb->do_checkpublicip) {
726 have_ip = ctdb_sys_have_ip(&pip->addr);
728 best_iface = ctdb_vnn_best_iface(ctdb, vnn);
729 if (best_iface == NULL) {
730 DEBUG(DEBUG_ERR,("takeoverip of IP %s/%u failed to find"
731 "a usable interface (old %s, have_ip %d)\n",
732 ctdb_addr_to_str(&vnn->public_address),
733 vnn->public_netmask_bits,
734 ctdb_vnn_iface_string(vnn),
739 if (vnn->iface == NULL && vnn->pnn == -1 && have_ip && best_iface != NULL) {
740 DEBUG(DEBUG_ERR,("Taking over newly created ip\n"));
745 if (vnn->iface == NULL && have_ip) {
746 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
747 "but we have no interface assigned, has someone manually configured it? Ignore for now.\n",
748 ctdb_addr_to_str(&vnn->public_address)));
752 if (vnn->pnn != ctdb->pnn && have_ip && vnn->pnn != -1) {
753 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
754 "and we have it on iface[%s], but it was assigned to node %d"
755 "and we are node %d, banning ourself\n",
756 ctdb_addr_to_str(&vnn->public_address),
757 ctdb_vnn_iface_string(vnn), vnn->pnn, ctdb->pnn));
762 if (vnn->pnn == -1 && have_ip) {
763 vnn->pnn = ctdb->pnn;
764 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
765 "and we already have it on iface[%s], update local daemon\n",
766 ctdb_addr_to_str(&vnn->public_address),
767 ctdb_vnn_iface_string(vnn)));
772 if (vnn->iface != best_iface) {
773 if (!vnn->iface->link_up) {
775 } else if (vnn->iface->references > (best_iface->references + 1)) {
776 /* only move when the rebalance gains something */
784 ctdb_vnn_unassign_iface(ctdb, vnn);
791 ret = ctdb_do_takeip(ctdb, c, vnn);
795 } else if (do_updateip) {
796 ret = ctdb_do_updateip(ctdb, c, vnn);
802 * The interface is up and the kernel known the ip
805 DEBUG(DEBUG_INFO,("Redundant takeover of IP %s/%u on interface %s (ip already held)\n",
806 ctdb_addr_to_str(&pip->addr),
807 vnn->public_netmask_bits,
808 ctdb_vnn_iface_string(vnn)));
812 /* tell ctdb_control.c that we will be replying asynchronously */
819 kill any clients that are registered with a IP that is being released
821 static void release_kill_clients(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
823 struct ctdb_client_ip *ip;
825 DEBUG(DEBUG_INFO,("release_kill_clients for ip %s\n",
826 ctdb_addr_to_str(addr)));
828 for (ip=ctdb->client_ip_list; ip; ip=ip->next) {
829 ctdb_sock_addr tmp_addr;
832 DEBUG(DEBUG_INFO,("checking for client %u with IP %s\n",
834 ctdb_addr_to_str(&ip->addr)));
836 if (ctdb_same_ip(&tmp_addr, addr)) {
837 struct ctdb_client *client = reqid_find(ctdb->idr,
840 DEBUG(DEBUG_INFO,("matched client %u with IP %s and pid %u\n",
842 ctdb_addr_to_str(&ip->addr),
845 if (client->pid != 0) {
846 DEBUG(DEBUG_INFO,(__location__ " Killing client pid %u for IP %s on client_id %u\n",
847 (unsigned)client->pid,
848 ctdb_addr_to_str(addr),
850 kill(client->pid, SIGKILL);
856 static void do_delete_ip(struct ctdb_context *ctdb, struct ctdb_vnn *vnn)
858 DLIST_REMOVE(ctdb->vnn, vnn);
859 ctdb_vnn_unassign_iface(ctdb, vnn);
860 ctdb_remove_orphaned_ifaces(ctdb, vnn);
865 called when releaseip event finishes
867 static void release_ip_callback(struct ctdb_context *ctdb, int status,
870 struct takeover_callback_state *state =
871 talloc_get_type(private_data, struct takeover_callback_state);
874 if (status == -ETIME) {
878 if (ctdb->tunable.disable_ip_failover == 0 && ctdb->do_checkpublicip) {
879 if (ctdb_sys_have_ip(state->addr)) {
881 ("IP %s still hosted during release IP callback, failing\n",
882 ctdb_addr_to_str(state->addr)));
883 ctdb_request_control_reply(ctdb, state->c,
890 /* send a message to all clients of this node telling them
891 that the cluster has been reconfigured and they should
892 release any sockets on this IP */
893 data.dptr = (uint8_t *)talloc_strdup(state, ctdb_addr_to_str(state->addr));
894 CTDB_NO_MEMORY_VOID(ctdb, data.dptr);
895 data.dsize = strlen((char *)data.dptr)+1;
897 DEBUG(DEBUG_INFO,(__location__ " sending RELEASE_IP for '%s'\n", data.dptr));
899 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
901 /* kill clients that have registered with this IP */
902 release_kill_clients(ctdb, state->addr);
904 ctdb_vnn_unassign_iface(ctdb, state->vnn);
906 /* Process the IP if it has been marked for deletion */
907 if (state->vnn->delete_pending) {
908 do_delete_ip(ctdb, state->vnn);
912 /* the control succeeded */
913 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
917 static int ctdb_releaseip_destructor(struct takeover_callback_state *state)
919 if (state->vnn != NULL) {
920 state->vnn->update_in_flight = false;
926 release an ip address
928 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
929 struct ctdb_req_control_old *c,
934 struct takeover_callback_state *state;
935 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
936 struct ctdb_vnn *vnn;
939 /* update our vnn list */
940 vnn = find_public_ip_vnn(ctdb, &pip->addr);
942 DEBUG(DEBUG_INFO,("releaseip called for an ip '%s' that is not a public address\n",
943 ctdb_addr_to_str(&pip->addr)));
948 /* stop any previous arps */
949 talloc_free(vnn->takeover_ctx);
950 vnn->takeover_ctx = NULL;
952 /* Some ctdb tool commands (e.g. moveip, rebalanceip) send
953 * lazy multicast to drop an IP from any node that isn't the
954 * intended new node. The following causes makes ctdbd ignore
955 * a release for any address it doesn't host.
957 if (ctdb->tunable.disable_ip_failover == 0 && ctdb->do_checkpublicip) {
958 if (!ctdb_sys_have_ip(&pip->addr)) {
959 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
960 ctdb_addr_to_str(&pip->addr),
961 vnn->public_netmask_bits,
962 ctdb_vnn_iface_string(vnn)));
963 ctdb_vnn_unassign_iface(ctdb, vnn);
967 if (vnn->iface == NULL) {
968 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
969 ctdb_addr_to_str(&pip->addr),
970 vnn->public_netmask_bits));
975 /* There is a potential race between take_ip and us because we
976 * update the VNN via a callback that run when the
977 * eventscripts have been run. Avoid the race by allowing one
978 * update to be in flight at a time.
980 if (vnn->update_in_flight) {
981 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
982 "update for this IP already in flight\n",
983 ctdb_addr_to_str(&vnn->public_address),
984 vnn->public_netmask_bits));
988 iface = strdup(ctdb_vnn_iface_string(vnn));
990 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s node:%d\n",
991 ctdb_addr_to_str(&pip->addr),
992 vnn->public_netmask_bits,
996 state = talloc(ctdb, struct takeover_callback_state);
998 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1004 state->c = talloc_steal(state, c);
1005 state->addr = talloc(state, ctdb_sock_addr);
1006 if (state->addr == NULL) {
1007 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1008 __FILE__, __LINE__);
1013 *state->addr = pip->addr;
1016 vnn->update_in_flight = true;
1017 talloc_set_destructor(state, ctdb_releaseip_destructor);
1019 ret = ctdb_event_script_callback(ctdb,
1020 state, release_ip_callback, state,
1021 CTDB_EVENT_RELEASE_IP,
1024 ctdb_addr_to_str(&pip->addr),
1025 vnn->public_netmask_bits);
1028 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
1029 ctdb_addr_to_str(&pip->addr),
1030 ctdb_vnn_iface_string(vnn)));
1035 /* tell the control that we will be reply asynchronously */
1036 *async_reply = true;
1040 static int ctdb_add_public_address(struct ctdb_context *ctdb,
1041 ctdb_sock_addr *addr,
1042 unsigned mask, const char *ifaces,
1045 struct ctdb_vnn *vnn;
1052 tmp = strdup(ifaces);
1053 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1054 if (!ctdb_sys_check_iface_exists(iface)) {
1055 DEBUG(DEBUG_CRIT,("Interface %s does not exist. Can not add public-address : %s\n", iface, ctdb_addr_to_str(addr)));
1062 /* Verify that we dont have an entry for this ip yet */
1063 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1064 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
1065 DEBUG(DEBUG_CRIT,("Same ip '%s' specified multiple times in the public address list \n",
1066 ctdb_addr_to_str(addr)));
1071 /* create a new vnn structure for this ip address */
1072 vnn = talloc_zero(ctdb, struct ctdb_vnn);
1073 CTDB_NO_MEMORY_FATAL(ctdb, vnn);
1074 vnn->ifaces = talloc_array(vnn, const char *, num + 2);
1075 tmp = talloc_strdup(vnn, ifaces);
1076 CTDB_NO_MEMORY_FATAL(ctdb, tmp);
1077 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1078 vnn->ifaces = talloc_realloc(vnn, vnn->ifaces, const char *, num + 2);
1079 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces);
1080 vnn->ifaces[num] = talloc_strdup(vnn, iface);
1081 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces[num]);
1085 vnn->ifaces[num] = NULL;
1086 vnn->public_address = *addr;
1087 vnn->public_netmask_bits = mask;
1089 if (check_address) {
1090 if (ctdb_sys_have_ip(addr)) {
1091 DEBUG(DEBUG_ERR,("We are already hosting public address '%s'. setting PNN to ourself:%d\n", ctdb_addr_to_str(addr), ctdb->pnn));
1092 vnn->pnn = ctdb->pnn;
1096 for (i=0; vnn->ifaces[i]; i++) {
1097 ret = ctdb_add_local_iface(ctdb, vnn->ifaces[i]);
1099 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1100 "for public_address[%s]\n",
1101 vnn->ifaces[i], ctdb_addr_to_str(addr)));
1107 DLIST_ADD(ctdb->vnn, vnn);
1113 setup the public address lists from a file
1115 int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
1121 lines = file_lines_load(ctdb->public_addresses_file, &nlines, 0, ctdb);
1122 if (lines == NULL) {
1123 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
1126 while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
1130 for (i=0;i<nlines;i++) {
1132 ctdb_sock_addr addr;
1133 const char *addrstr;
1138 while ((*line == ' ') || (*line == '\t')) {
1144 if (strcmp(line, "") == 0) {
1147 tok = strtok(line, " \t");
1149 tok = strtok(NULL, " \t");
1151 if (NULL == ctdb->default_public_interface) {
1152 DEBUG(DEBUG_CRIT,("No default public interface and no interface specified at line %u of public address list\n",
1157 ifaces = ctdb->default_public_interface;
1162 if (!addrstr || !parse_ip_mask(addrstr, ifaces, &addr, &mask)) {
1163 DEBUG(DEBUG_CRIT,("Badly formed line %u in public address list\n", i+1));
1167 if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
1168 DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
1179 int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
1183 struct ctdb_vnn *svnn;
1184 struct ctdb_interface *cur = NULL;
1188 svnn = talloc_zero(ctdb, struct ctdb_vnn);
1189 CTDB_NO_MEMORY(ctdb, svnn);
1191 svnn->ifaces = talloc_array(svnn, const char *, 2);
1192 CTDB_NO_MEMORY(ctdb, svnn->ifaces);
1193 svnn->ifaces[0] = talloc_strdup(svnn->ifaces, iface);
1194 CTDB_NO_MEMORY(ctdb, svnn->ifaces[0]);
1195 svnn->ifaces[1] = NULL;
1197 ok = parse_ip(ip, iface, 0, &svnn->public_address);
1203 ret = ctdb_add_local_iface(ctdb, svnn->ifaces[0]);
1205 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1206 "for single_ip[%s]\n",
1208 ctdb_addr_to_str(&svnn->public_address)));
1213 /* assume the single public ip interface is initially "good" */
1214 cur = ctdb_find_iface(ctdb, iface);
1216 DEBUG(DEBUG_CRIT,("Can not find public interface %s used by --single-public-ip", iface));
1219 cur->link_up = true;
1221 ret = ctdb_vnn_assign_iface(ctdb, svnn);
1227 ctdb->single_ip_vnn = svnn;
1231 struct public_ip_list {
1232 struct public_ip_list *next;
1234 ctdb_sock_addr addr;
1237 /* Given a physical node, return the number of
1238 public addresses that is currently assigned to this node.
1240 static int node_ip_coverage(struct ctdb_context *ctdb, int32_t pnn,
1241 struct public_ip_list *ips)
1245 for (;ips;ips=ips->next) {
1246 if (ips->pnn == pnn) {
1254 /* Can the given node host the given IP: is the public IP known to the
1255 * node and is NOIPHOST unset?
1257 static bool can_node_host_ip(struct ctdb_context *ctdb, int32_t pnn,
1258 struct ctdb_ipflags ipflags,
1259 struct public_ip_list *ip)
1261 struct ctdb_public_ip_list_old *public_ips;
1264 if (ipflags.noiphost) {
1268 public_ips = ctdb->nodes[pnn]->available_public_ips;
1270 if (public_ips == NULL) {
1274 for (i=0; i<public_ips->num; i++) {
1275 if (ctdb_same_ip(&ip->addr, &public_ips->ips[i].addr)) {
1276 /* yes, this node can serve this public ip */
1284 static bool can_node_takeover_ip(struct ctdb_context *ctdb, int32_t pnn,
1285 struct ctdb_ipflags ipflags,
1286 struct public_ip_list *ip)
1288 if (ipflags.noiptakeover) {
1292 return can_node_host_ip(ctdb, pnn, ipflags, ip);
1295 /* search the node lists list for a node to takeover this ip.
1296 pick the node that currently are serving the least number of ips
1297 so that the ips get spread out evenly.
1299 static int find_takeover_node(struct ctdb_context *ctdb,
1300 struct ctdb_ipflags *ipflags,
1301 struct public_ip_list *ip,
1302 struct public_ip_list *all_ips)
1304 int pnn, min=0, num;
1307 numnodes = talloc_array_length(ipflags);
1309 for (i=0; i<numnodes; i++) {
1310 /* verify that this node can serve this ip */
1311 if (!can_node_takeover_ip(ctdb, i, ipflags[i], ip)) {
1312 /* no it couldnt so skip to the next node */
1316 num = node_ip_coverage(ctdb, i, all_ips);
1317 /* was this the first node we checked ? */
1329 DEBUG(DEBUG_WARNING,(__location__ " Could not find node to take over public address '%s'\n",
1330 ctdb_addr_to_str(&ip->addr)));
1340 static uint32_t *ip_key(ctdb_sock_addr *ip)
1342 static uint32_t key[IP_KEYLEN];
1344 bzero(key, sizeof(key));
1346 switch (ip->sa.sa_family) {
1348 key[3] = htonl(ip->ip.sin_addr.s_addr);
1351 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
1352 key[0] = htonl(s6_a32[0]);
1353 key[1] = htonl(s6_a32[1]);
1354 key[2] = htonl(s6_a32[2]);
1355 key[3] = htonl(s6_a32[3]);
1359 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1366 static void *add_ip_callback(void *parm, void *data)
1368 struct public_ip_list *this_ip = parm;
1369 struct public_ip_list *prev_ip = data;
1371 if (prev_ip == NULL) {
1374 if (this_ip->pnn == -1) {
1375 this_ip->pnn = prev_ip->pnn;
1381 static int getips_count_callback(void *param, void *data)
1383 struct public_ip_list **ip_list = (struct public_ip_list **)param;
1384 struct public_ip_list *new_ip = (struct public_ip_list *)data;
1386 new_ip->next = *ip_list;
1391 static struct public_ip_list *
1392 create_merged_ip_list(struct ctdb_context *ctdb)
1395 struct public_ip_list *ip_list;
1396 struct ctdb_public_ip_list_old *public_ips;
1398 if (ctdb->ip_tree != NULL) {
1399 talloc_free(ctdb->ip_tree);
1400 ctdb->ip_tree = NULL;
1402 ctdb->ip_tree = trbt_create(ctdb, 0);
1404 for (i=0;i<ctdb->num_nodes;i++) {
1405 public_ips = ctdb->nodes[i]->known_public_ips;
1407 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
1411 /* there were no public ips for this node */
1412 if (public_ips == NULL) {
1416 for (j=0;j<public_ips->num;j++) {
1417 struct public_ip_list *tmp_ip;
1419 tmp_ip = talloc_zero(ctdb->ip_tree, struct public_ip_list);
1420 CTDB_NO_MEMORY_NULL(ctdb, tmp_ip);
1421 /* Do not use information about IP addresses hosted
1422 * on other nodes, it may not be accurate */
1423 if (public_ips->ips[j].pnn == ctdb->nodes[i]->pnn) {
1424 tmp_ip->pnn = public_ips->ips[j].pnn;
1428 tmp_ip->addr = public_ips->ips[j].addr;
1429 tmp_ip->next = NULL;
1431 trbt_insertarray32_callback(ctdb->ip_tree,
1432 IP_KEYLEN, ip_key(&public_ips->ips[j].addr),
1439 trbt_traversearray32(ctdb->ip_tree, IP_KEYLEN, getips_count_callback, &ip_list);
1445 * This is the length of the longtest common prefix between the IPs.
1446 * It is calculated by XOR-ing the 2 IPs together and counting the
1447 * number of leading zeroes. The implementation means that all
1448 * addresses end up being 128 bits long.
1450 * FIXME? Should we consider IPv4 and IPv6 separately given that the
1451 * 12 bytes of 0 prefix padding will hurt the algorithm if there are
1452 * lots of nodes and IP addresses?
1454 static uint32_t ip_distance(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
1456 uint32_t ip1_k[IP_KEYLEN];
1461 uint32_t distance = 0;
1463 memcpy(ip1_k, ip_key(ip1), sizeof(ip1_k));
1465 for (i=0; i<IP_KEYLEN; i++) {
1466 x = ip1_k[i] ^ t[i];
1470 /* Count number of leading zeroes.
1471 * FIXME? This could be optimised...
1473 while ((x & (1 << 31)) == 0) {
1483 /* Calculate the IP distance for the given IP relative to IPs on the
1484 given node. The ips argument is generally the all_ips variable
1485 used in the main part of the algorithm.
1487 static uint32_t ip_distance_2_sum(ctdb_sock_addr *ip,
1488 struct public_ip_list *ips,
1491 struct public_ip_list *t;
1496 for (t=ips; t != NULL; t=t->next) {
1497 if (t->pnn != pnn) {
1501 /* Optimisation: We never calculate the distance
1502 * between an address and itself. This allows us to
1503 * calculate the effect of removing an address from a
1504 * node by simply calculating the distance between
1505 * that address and all of the exitsing addresses.
1506 * Moreover, we assume that we're only ever dealing
1507 * with addresses from all_ips so we can identify an
1508 * address via a pointer rather than doing a more
1509 * expensive address comparison. */
1510 if (&(t->addr) == ip) {
1514 d = ip_distance(ip, &(t->addr));
1515 sum += d * d; /* Cheaper than pulling in math.h :-) */
1521 /* Return the LCP2 imbalance metric for addresses currently assigned
1524 static uint32_t lcp2_imbalance(struct public_ip_list * all_ips, int pnn)
1526 struct public_ip_list *t;
1528 uint32_t imbalance = 0;
1530 for (t=all_ips; t!=NULL; t=t->next) {
1531 if (t->pnn != pnn) {
1534 /* Pass the rest of the IPs rather than the whole
1537 imbalance += ip_distance_2_sum(&(t->addr), t->next, pnn);
1543 /* Allocate any unassigned IPs just by looping through the IPs and
1544 * finding the best node for each.
1546 static void basic_allocate_unassigned(struct ctdb_context *ctdb,
1547 struct ctdb_ipflags *ipflags,
1548 struct public_ip_list *all_ips)
1550 struct public_ip_list *tmp_ip;
1552 /* loop over all ip's and find a physical node to cover for
1555 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1556 if (tmp_ip->pnn == -1) {
1557 if (find_takeover_node(ctdb, ipflags, tmp_ip, all_ips)) {
1558 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1559 ctdb_addr_to_str(&tmp_ip->addr)));
1565 /* Basic non-deterministic rebalancing algorithm.
1567 static void basic_failback(struct ctdb_context *ctdb,
1568 struct ctdb_ipflags *ipflags,
1569 struct public_ip_list *all_ips,
1573 int maxnode, maxnum, minnode, minnum, num, retries;
1574 struct public_ip_list *tmp_ip;
1576 numnodes = talloc_array_length(ipflags);
1583 /* for each ip address, loop over all nodes that can serve
1584 this ip and make sure that the difference between the node
1585 serving the most and the node serving the least ip's are
1588 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1589 if (tmp_ip->pnn == -1) {
1593 /* Get the highest and lowest number of ips's served by any
1594 valid node which can serve this ip.
1598 for (i=0; i<numnodes; i++) {
1599 /* only check nodes that can actually serve this ip */
1600 if (!can_node_takeover_ip(ctdb, i, ipflags[i], tmp_ip)) {
1601 /* no it couldnt so skip to the next node */
1605 num = node_ip_coverage(ctdb, i, all_ips);
1606 if (maxnode == -1) {
1615 if (minnode == -1) {
1625 if (maxnode == -1) {
1626 DEBUG(DEBUG_WARNING,(__location__ " Could not find maxnode. May not be able to serve ip '%s'\n",
1627 ctdb_addr_to_str(&tmp_ip->addr)));
1632 /* if the spread between the smallest and largest coverage by
1633 a node is >=2 we steal one of the ips from the node with
1634 most coverage to even things out a bit.
1635 try to do this a limited number of times since we dont
1636 want to spend too much time balancing the ip coverage.
1638 if ( (maxnum > minnum+1)
1639 && (retries < (num_ips + 5)) ){
1640 struct public_ip_list *tmp;
1642 /* Reassign one of maxnode's VNNs */
1643 for (tmp=all_ips;tmp;tmp=tmp->next) {
1644 if (tmp->pnn == maxnode) {
1645 (void)find_takeover_node(ctdb, ipflags, tmp, all_ips);
1654 static void lcp2_init(struct ctdb_context *tmp_ctx,
1655 struct ctdb_ipflags *ipflags,
1656 struct public_ip_list *all_ips,
1657 uint32_t *force_rebalance_nodes,
1658 uint32_t **lcp2_imbalances,
1659 bool **rebalance_candidates)
1662 struct public_ip_list *tmp_ip;
1664 numnodes = talloc_array_length(ipflags);
1666 *rebalance_candidates = talloc_array(tmp_ctx, bool, numnodes);
1667 CTDB_NO_MEMORY_FATAL(tmp_ctx, *rebalance_candidates);
1668 *lcp2_imbalances = talloc_array(tmp_ctx, uint32_t, numnodes);
1669 CTDB_NO_MEMORY_FATAL(tmp_ctx, *lcp2_imbalances);
1671 for (i=0; i<numnodes; i++) {
1672 (*lcp2_imbalances)[i] = lcp2_imbalance(all_ips, i);
1673 /* First step: assume all nodes are candidates */
1674 (*rebalance_candidates)[i] = true;
1677 /* 2nd step: if a node has IPs assigned then it must have been
1678 * healthy before, so we remove it from consideration. This
1679 * is overkill but is all we have because we don't maintain
1680 * state between takeover runs. An alternative would be to
1681 * keep state and invalidate it every time the recovery master
1684 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1685 if (tmp_ip->pnn != -1) {
1686 (*rebalance_candidates)[tmp_ip->pnn] = false;
1690 /* 3rd step: if a node is forced to re-balance then
1691 we allow failback onto the node */
1692 if (force_rebalance_nodes == NULL) {
1695 for (i = 0; i < talloc_array_length(force_rebalance_nodes); i++) {
1696 uint32_t pnn = force_rebalance_nodes[i];
1697 if (pnn >= numnodes) {
1699 (__location__ "unknown node %u\n", pnn));
1704 ("Forcing rebalancing of IPs to node %u\n", pnn));
1705 (*rebalance_candidates)[pnn] = true;
1709 /* Allocate any unassigned addresses using the LCP2 algorithm to find
1710 * the IP/node combination that will cost the least.
1712 static void lcp2_allocate_unassigned(struct ctdb_context *ctdb,
1713 struct ctdb_ipflags *ipflags,
1714 struct public_ip_list *all_ips,
1715 uint32_t *lcp2_imbalances)
1717 struct public_ip_list *tmp_ip;
1718 int dstnode, numnodes;
1721 uint32_t mindsum, dstdsum, dstimbl, minimbl;
1722 struct public_ip_list *minip;
1724 bool should_loop = true;
1725 bool have_unassigned = true;
1727 numnodes = talloc_array_length(ipflags);
1729 while (have_unassigned && should_loop) {
1730 should_loop = false;
1732 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1733 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES (UNASSIGNED)\n"));
1739 /* loop over each unassigned ip. */
1740 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1741 if (tmp_ip->pnn != -1) {
1745 for (dstnode=0; dstnode<numnodes; dstnode++) {
1746 /* only check nodes that can actually takeover this ip */
1747 if (!can_node_takeover_ip(ctdb, dstnode,
1750 /* no it couldnt so skip to the next node */
1754 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1755 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1756 DEBUG(DEBUG_DEBUG,(" %s -> %d [+%d]\n",
1757 ctdb_addr_to_str(&(tmp_ip->addr)),
1759 dstimbl - lcp2_imbalances[dstnode]));
1762 if ((minnode == -1) || (dstdsum < mindsum)) {
1772 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1774 /* If we found one then assign it to the given node. */
1775 if (minnode != -1) {
1776 minip->pnn = minnode;
1777 lcp2_imbalances[minnode] = minimbl;
1778 DEBUG(DEBUG_INFO,(" %s -> %d [+%d]\n",
1779 ctdb_addr_to_str(&(minip->addr)),
1784 /* There might be a better way but at least this is clear. */
1785 have_unassigned = false;
1786 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1787 if (tmp_ip->pnn == -1) {
1788 have_unassigned = true;
1793 /* We know if we have an unassigned addresses so we might as
1796 if (have_unassigned) {
1797 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1798 if (tmp_ip->pnn == -1) {
1799 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1800 ctdb_addr_to_str(&tmp_ip->addr)));
1806 /* LCP2 algorithm for rebalancing the cluster. Given a candidate node
1807 * to move IPs from, determines the best IP/destination node
1808 * combination to move from the source node.
1810 static bool lcp2_failback_candidate(struct ctdb_context *ctdb,
1811 struct ctdb_ipflags *ipflags,
1812 struct public_ip_list *all_ips,
1814 uint32_t *lcp2_imbalances,
1815 bool *rebalance_candidates)
1817 int dstnode, mindstnode, numnodes;
1818 uint32_t srcimbl, srcdsum, dstimbl, dstdsum;
1819 uint32_t minsrcimbl, mindstimbl;
1820 struct public_ip_list *minip;
1821 struct public_ip_list *tmp_ip;
1823 /* Find an IP and destination node that best reduces imbalance. */
1830 numnodes = talloc_array_length(ipflags);
1832 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1833 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES FROM %d [%d]\n",
1834 srcnode, lcp2_imbalances[srcnode]));
1836 for (tmp_ip=all_ips; tmp_ip; tmp_ip=tmp_ip->next) {
1837 /* Only consider addresses on srcnode. */
1838 if (tmp_ip->pnn != srcnode) {
1842 /* What is this IP address costing the source node? */
1843 srcdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, srcnode);
1844 srcimbl = lcp2_imbalances[srcnode] - srcdsum;
1846 /* Consider this IP address would cost each potential
1847 * destination node. Destination nodes are limited to
1848 * those that are newly healthy, since we don't want
1849 * to do gratuitous failover of IPs just to make minor
1850 * balance improvements.
1852 for (dstnode=0; dstnode<numnodes; dstnode++) {
1853 if (!rebalance_candidates[dstnode]) {
1857 /* only check nodes that can actually takeover this ip */
1858 if (!can_node_takeover_ip(ctdb, dstnode,
1859 ipflags[dstnode], tmp_ip)) {
1860 /* no it couldnt so skip to the next node */
1864 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1865 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1866 DEBUG(DEBUG_DEBUG,(" %d [%d] -> %s -> %d [+%d]\n",
1868 ctdb_addr_to_str(&(tmp_ip->addr)),
1871 if ((dstimbl < lcp2_imbalances[srcnode]) &&
1872 (dstdsum < srcdsum) && \
1873 ((mindstnode == -1) || \
1874 ((srcimbl + dstimbl) < (minsrcimbl + mindstimbl)))) {
1877 minsrcimbl = srcimbl;
1878 mindstnode = dstnode;
1879 mindstimbl = dstimbl;
1883 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1885 if (mindstnode != -1) {
1886 /* We found a move that makes things better... */
1887 DEBUG(DEBUG_INFO,("%d [%d] -> %s -> %d [+%d]\n",
1888 srcnode, minsrcimbl - lcp2_imbalances[srcnode],
1889 ctdb_addr_to_str(&(minip->addr)),
1890 mindstnode, mindstimbl - lcp2_imbalances[mindstnode]));
1893 lcp2_imbalances[srcnode] = minsrcimbl;
1894 lcp2_imbalances[mindstnode] = mindstimbl;
1895 minip->pnn = mindstnode;
1904 struct lcp2_imbalance_pnn {
1909 static int lcp2_cmp_imbalance_pnn(const void * a, const void * b)
1911 const struct lcp2_imbalance_pnn * lipa = (const struct lcp2_imbalance_pnn *) a;
1912 const struct lcp2_imbalance_pnn * lipb = (const struct lcp2_imbalance_pnn *) b;
1914 if (lipa->imbalance > lipb->imbalance) {
1916 } else if (lipa->imbalance == lipb->imbalance) {
1923 /* LCP2 algorithm for rebalancing the cluster. This finds the source
1924 * node with the highest LCP2 imbalance, and then determines the best
1925 * IP/destination node combination to move from the source node.
1927 static void lcp2_failback(struct ctdb_context *ctdb,
1928 struct ctdb_ipflags *ipflags,
1929 struct public_ip_list *all_ips,
1930 uint32_t *lcp2_imbalances,
1931 bool *rebalance_candidates)
1934 struct lcp2_imbalance_pnn * lips;
1937 numnodes = talloc_array_length(ipflags);
1940 /* Put the imbalances and nodes into an array, sort them and
1941 * iterate through candidates. Usually the 1st one will be
1942 * used, so this doesn't cost much...
1944 DEBUG(DEBUG_DEBUG,("+++++++++++++++++++++++++++++++++++++++++\n"));
1945 DEBUG(DEBUG_DEBUG,("Selecting most imbalanced node from:\n"));
1946 lips = talloc_array(ctdb, struct lcp2_imbalance_pnn, numnodes);
1947 for (i=0; i<numnodes; i++) {
1948 lips[i].imbalance = lcp2_imbalances[i];
1950 DEBUG(DEBUG_DEBUG,(" %d [%d]\n", i, lcp2_imbalances[i]));
1952 qsort(lips, numnodes, sizeof(struct lcp2_imbalance_pnn),
1953 lcp2_cmp_imbalance_pnn);
1956 for (i=0; i<numnodes; i++) {
1957 /* This means that all nodes had 0 or 1 addresses, so
1958 * can't be imbalanced.
1960 if (lips[i].imbalance == 0) {
1964 if (lcp2_failback_candidate(ctdb,
1969 rebalance_candidates)) {
1981 static void unassign_unsuitable_ips(struct ctdb_context *ctdb,
1982 struct ctdb_ipflags *ipflags,
1983 struct public_ip_list *all_ips)
1985 struct public_ip_list *tmp_ip;
1987 /* verify that the assigned nodes can serve that public ip
1988 and set it to -1 if not
1990 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1991 if (tmp_ip->pnn == -1) {
1994 if (!can_node_host_ip(ctdb, tmp_ip->pnn,
1995 ipflags[tmp_ip->pnn], tmp_ip) != 0) {
1996 /* this node can not serve this ip. */
1997 DEBUG(DEBUG_DEBUG,("Unassign IP: %s from %d\n",
1998 ctdb_addr_to_str(&(tmp_ip->addr)),
2005 static void ip_alloc_deterministic_ips(struct ctdb_context *ctdb,
2006 struct ctdb_ipflags *ipflags,
2007 struct public_ip_list *all_ips)
2009 struct public_ip_list *tmp_ip;
2012 numnodes = talloc_array_length(ipflags);
2014 DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
2015 /* Allocate IPs to nodes in a modulo fashion so that IPs will
2016 * always be allocated the same way for a specific set of
2017 * available/unavailable nodes.
2020 for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
2021 tmp_ip->pnn = i % numnodes;
2024 /* IP failback doesn't make sense with deterministic
2025 * IPs, since the modulo step above implicitly fails
2026 * back IPs to their "home" node.
2028 if (1 == ctdb->tunable.no_ip_failback) {
2029 DEBUG(DEBUG_WARNING, ("WARNING: 'NoIPFailback' set but ignored - incompatible with 'DeterministicIPs\n"));
2032 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2034 basic_allocate_unassigned(ctdb, ipflags, all_ips);
2036 /* No failback here! */
2039 static void ip_alloc_nondeterministic_ips(struct ctdb_context *ctdb,
2040 struct ctdb_ipflags *ipflags,
2041 struct public_ip_list *all_ips)
2043 /* This should be pushed down into basic_failback. */
2044 struct public_ip_list *tmp_ip;
2046 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2050 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2052 basic_allocate_unassigned(ctdb, ipflags, all_ips);
2054 /* If we don't want IPs to fail back then don't rebalance IPs. */
2055 if (1 == ctdb->tunable.no_ip_failback) {
2059 /* Now, try to make sure the ip adresses are evenly distributed
2062 basic_failback(ctdb, ipflags, all_ips, num_ips);
2065 static void ip_alloc_lcp2(struct ctdb_context *ctdb,
2066 struct ctdb_ipflags *ipflags,
2067 struct public_ip_list *all_ips,
2068 uint32_t *force_rebalance_nodes)
2070 uint32_t *lcp2_imbalances;
2071 bool *rebalance_candidates;
2072 int numnodes, num_rebalance_candidates, i;
2074 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2076 unassign_unsuitable_ips(ctdb, ipflags, all_ips);
2078 lcp2_init(tmp_ctx, ipflags, all_ips,force_rebalance_nodes,
2079 &lcp2_imbalances, &rebalance_candidates);
2081 lcp2_allocate_unassigned(ctdb, ipflags, all_ips, lcp2_imbalances);
2083 /* If we don't want IPs to fail back then don't rebalance IPs. */
2084 if (1 == ctdb->tunable.no_ip_failback) {
2088 /* It is only worth continuing if we have suitable target
2089 * nodes to transfer IPs to. This check is much cheaper than
2092 numnodes = talloc_array_length(ipflags);
2093 num_rebalance_candidates = 0;
2094 for (i=0; i<numnodes; i++) {
2095 if (rebalance_candidates[i]) {
2096 num_rebalance_candidates++;
2099 if (num_rebalance_candidates == 0) {
2103 /* Now, try to make sure the ip adresses are evenly distributed
2106 lcp2_failback(ctdb, ipflags, all_ips,
2107 lcp2_imbalances, rebalance_candidates);
2110 talloc_free(tmp_ctx);
2113 static bool all_nodes_are_disabled(struct ctdb_node_map_old *nodemap)
2117 for (i=0;i<nodemap->num;i++) {
2118 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
2119 /* Found one completely healthy node */
2127 /* The calculation part of the IP allocation algorithm. */
2128 static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
2129 struct ctdb_ipflags *ipflags,
2130 struct public_ip_list **all_ips_p,
2131 uint32_t *force_rebalance_nodes)
2133 /* since nodes only know about those public addresses that
2134 can be served by that particular node, no single node has
2135 a full list of all public addresses that exist in the cluster.
2136 Walk over all node structures and create a merged list of
2137 all public addresses that exist in the cluster.
2139 keep the tree of ips around as ctdb->ip_tree
2141 *all_ips_p = create_merged_ip_list(ctdb);
2143 if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
2144 ip_alloc_lcp2(ctdb, ipflags, *all_ips_p, force_rebalance_nodes);
2145 } else if (1 == ctdb->tunable.deterministic_public_ips) {
2146 ip_alloc_deterministic_ips(ctdb, ipflags, *all_ips_p);
2148 ip_alloc_nondeterministic_ips(ctdb, ipflags, *all_ips_p);
2151 /* at this point ->pnn is the node which will own each IP
2152 or -1 if there is no node that can cover this ip
2158 struct get_tunable_callback_data {
2159 const char *tunable;
2164 static void get_tunable_callback(struct ctdb_context *ctdb, uint32_t pnn,
2165 int32_t res, TDB_DATA outdata,
2168 struct get_tunable_callback_data *cd =
2169 (struct get_tunable_callback_data *)callback;
2173 /* Already handled in fail callback */
2177 if (outdata.dsize != sizeof(uint32_t)) {
2178 DEBUG(DEBUG_ERR,("Wrong size of returned data when reading \"%s\" tunable from node %d. Expected %d bytes but received %d bytes\n",
2179 cd->tunable, pnn, (int)sizeof(uint32_t),
2180 (int)outdata.dsize));
2185 size = talloc_array_length(cd->out);
2187 DEBUG(DEBUG_ERR,("Got %s reply from node %d but nodemap only has %d entries\n",
2188 cd->tunable, pnn, size));
2193 cd->out[pnn] = *(uint32_t *)outdata.dptr;
2196 static void get_tunable_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2197 int32_t res, TDB_DATA outdata,
2200 struct get_tunable_callback_data *cd =
2201 (struct get_tunable_callback_data *)callback;
2206 ("Timed out getting tunable \"%s\" from node %d\n",
2212 DEBUG(DEBUG_WARNING,
2213 ("Tunable \"%s\" not implemented on node %d\n",
2218 ("Unexpected error getting tunable \"%s\" from node %d\n",
2224 static uint32_t *get_tunable_from_nodes(struct ctdb_context *ctdb,
2225 TALLOC_CTX *tmp_ctx,
2226 struct ctdb_node_map_old *nodemap,
2227 const char *tunable,
2228 uint32_t default_value)
2231 struct ctdb_control_get_tunable *t;
2234 struct get_tunable_callback_data callback_data;
2237 tvals = talloc_array(tmp_ctx, uint32_t, nodemap->num);
2238 CTDB_NO_MEMORY_NULL(ctdb, tvals);
2239 for (i=0; i<nodemap->num; i++) {
2240 tvals[i] = default_value;
2243 callback_data.out = tvals;
2244 callback_data.tunable = tunable;
2245 callback_data.fatal = false;
2247 data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen(tunable) + 1;
2248 data.dptr = talloc_size(tmp_ctx, data.dsize);
2249 t = (struct ctdb_control_get_tunable *)data.dptr;
2250 t->length = strlen(tunable)+1;
2251 memcpy(t->name, tunable, t->length);
2252 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2253 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_TUNABLE,
2254 nodes, 0, TAKEOVER_TIMEOUT(),
2256 get_tunable_callback,
2257 get_tunable_fail_callback,
2258 &callback_data) != 0) {
2259 if (callback_data.fatal) {
2265 talloc_free(data.dptr);
2270 struct get_runstate_callback_data {
2271 enum ctdb_runstate *out;
2275 static void get_runstate_callback(struct ctdb_context *ctdb, uint32_t pnn,
2276 int32_t res, TDB_DATA outdata,
2277 void *callback_data)
2279 struct get_runstate_callback_data *cd =
2280 (struct get_runstate_callback_data *)callback_data;
2284 /* Already handled in fail callback */
2288 if (outdata.dsize != sizeof(uint32_t)) {
2289 DEBUG(DEBUG_ERR,("Wrong size of returned data when getting runstate from node %d. Expected %d bytes but received %d bytes\n",
2290 pnn, (int)sizeof(uint32_t),
2291 (int)outdata.dsize));
2296 size = talloc_array_length(cd->out);
2298 DEBUG(DEBUG_ERR,("Got reply from node %d but nodemap only has %d entries\n",
2303 cd->out[pnn] = (enum ctdb_runstate)*(uint32_t *)outdata.dptr;
2306 static void get_runstate_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2307 int32_t res, TDB_DATA outdata,
2310 struct get_runstate_callback_data *cd =
2311 (struct get_runstate_callback_data *)callback;
2316 ("Timed out getting runstate from node %d\n", pnn));
2320 DEBUG(DEBUG_WARNING,
2321 ("Error getting runstate from node %d - assuming runstates not supported\n",
2326 static enum ctdb_runstate * get_runstate_from_nodes(struct ctdb_context *ctdb,
2327 TALLOC_CTX *tmp_ctx,
2328 struct ctdb_node_map_old *nodemap,
2329 enum ctdb_runstate default_value)
2332 enum ctdb_runstate *rs;
2333 struct get_runstate_callback_data callback_data;
2336 rs = talloc_array(tmp_ctx, enum ctdb_runstate, nodemap->num);
2337 CTDB_NO_MEMORY_NULL(ctdb, rs);
2338 for (i=0; i<nodemap->num; i++) {
2339 rs[i] = default_value;
2342 callback_data.out = rs;
2343 callback_data.fatal = false;
2345 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2346 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_RUNSTATE,
2347 nodes, 0, TAKEOVER_TIMEOUT(),
2349 get_runstate_callback,
2350 get_runstate_fail_callback,
2351 &callback_data) != 0) {
2352 if (callback_data.fatal) {
2362 /* Set internal flags for IP allocation:
2364 * Set NOIPTAKOVER ip flags from per-node NoIPTakeover tunable
2365 * Set NOIPHOST ip flag for each INACTIVE node
2366 * if all nodes are disabled:
2367 * Set NOIPHOST ip flags from per-node NoIPHostOnAllDisabled tunable
2369 * Set NOIPHOST ip flags for disabled nodes
2371 static struct ctdb_ipflags *
2372 set_ipflags_internal(struct ctdb_context *ctdb,
2373 TALLOC_CTX *tmp_ctx,
2374 struct ctdb_node_map_old *nodemap,
2375 uint32_t *tval_noiptakeover,
2376 uint32_t *tval_noiphostonalldisabled,
2377 enum ctdb_runstate *runstate)
2380 struct ctdb_ipflags *ipflags;
2382 /* Clear IP flags - implicit due to talloc_zero */
2383 ipflags = talloc_zero_array(tmp_ctx, struct ctdb_ipflags, nodemap->num);
2384 CTDB_NO_MEMORY_NULL(ctdb, ipflags);
2386 for (i=0;i<nodemap->num;i++) {
2387 /* Can not take IPs on node with NoIPTakeover set */
2388 if (tval_noiptakeover[i] != 0) {
2389 ipflags[i].noiptakeover = true;
2392 /* Can not host IPs on node not in RUNNING state */
2393 if (runstate[i] != CTDB_RUNSTATE_RUNNING) {
2394 ipflags[i].noiphost = true;
2397 /* Can not host IPs on INACTIVE node */
2398 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2399 ipflags[i].noiphost = true;
2401 /* Remember the runstate */
2402 ipflags[i].runstate = runstate[i];
2405 if (all_nodes_are_disabled(nodemap)) {
2406 /* If all nodes are disabled, can not host IPs on node
2407 * with NoIPHostOnAllDisabled set
2409 for (i=0;i<nodemap->num;i++) {
2410 if (tval_noiphostonalldisabled[i] != 0) {
2411 ipflags[i].noiphost = true;
2415 /* If some nodes are not disabled, then can not host
2416 * IPs on DISABLED node
2418 for (i=0;i<nodemap->num;i++) {
2419 if (nodemap->nodes[i].flags & NODE_FLAGS_DISABLED) {
2420 ipflags[i].noiphost = true;
2428 static struct ctdb_ipflags *set_ipflags(struct ctdb_context *ctdb,
2429 TALLOC_CTX *tmp_ctx,
2430 struct ctdb_node_map_old *nodemap)
2432 uint32_t *tval_noiptakeover;
2433 uint32_t *tval_noiphostonalldisabled;
2434 struct ctdb_ipflags *ipflags;
2435 enum ctdb_runstate *runstate;
2438 tval_noiptakeover = get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
2440 if (tval_noiptakeover == NULL) {
2444 tval_noiphostonalldisabled =
2445 get_tunable_from_nodes(ctdb, tmp_ctx, nodemap,
2446 "NoIPHostOnAllDisabled", 0);
2447 if (tval_noiphostonalldisabled == NULL) {
2448 /* Caller frees tmp_ctx */
2452 /* Any nodes where CTDB_CONTROL_GET_RUNSTATE is not supported
2453 * will default to CTDB_RUNSTATE_RUNNING. This ensures
2454 * reasonable behaviour on a mixed cluster during upgrade.
2456 runstate = get_runstate_from_nodes(ctdb, tmp_ctx, nodemap,
2457 CTDB_RUNSTATE_RUNNING);
2458 if (runstate == NULL) {
2459 /* Caller frees tmp_ctx */
2463 ipflags = set_ipflags_internal(ctdb, tmp_ctx, nodemap,
2465 tval_noiphostonalldisabled,
2468 talloc_free(tval_noiptakeover);
2469 talloc_free(tval_noiphostonalldisabled);
2470 talloc_free(runstate);
2475 struct iprealloc_callback_data {
2478 client_async_callback fail_callback;
2479 void *fail_callback_data;
2480 struct ctdb_node_map_old *nodemap;
2483 static void iprealloc_fail_callback(struct ctdb_context *ctdb, uint32_t pnn,
2484 int32_t res, TDB_DATA outdata,
2488 struct iprealloc_callback_data *cd =
2489 (struct iprealloc_callback_data *)callback;
2491 numnodes = talloc_array_length(cd->retry_nodes);
2492 if (pnn > numnodes) {
2494 ("ipreallocated failure from node %d, "
2495 "but only %d nodes in nodemap\n",
2500 /* Can't run the "ipreallocated" event on a INACTIVE node */
2501 if (cd->nodemap->nodes[pnn].flags & NODE_FLAGS_INACTIVE) {
2502 DEBUG(DEBUG_WARNING,
2503 ("ipreallocated failed on inactive node %d, ignoring\n",
2510 /* If the control timed out then that's a real error,
2511 * so call the real fail callback
2513 if (cd->fail_callback) {
2514 cd->fail_callback(ctdb, pnn, res, outdata,
2515 cd->fail_callback_data);
2517 DEBUG(DEBUG_WARNING,
2518 ("iprealloc timed out but no callback registered\n"));
2522 /* If not a timeout then either the ipreallocated
2523 * eventscript (or some setup) failed. This might
2524 * have failed because the IPREALLOCATED control isn't
2525 * implemented - right now there is no way of knowing
2526 * because the error codes are all folded down to -1.
2527 * Consider retrying using EVENTSCRIPT control...
2529 DEBUG(DEBUG_WARNING,
2530 ("ipreallocated failure from node %d, flagging retry\n",
2532 cd->retry_nodes[pnn] = true;
2537 struct takeover_callback_data {
2539 client_async_callback fail_callback;
2540 void *fail_callback_data;
2541 struct ctdb_node_map_old *nodemap;
2544 static void takeover_run_fail_callback(struct ctdb_context *ctdb,
2545 uint32_t node_pnn, int32_t res,
2546 TDB_DATA outdata, void *callback_data)
2548 struct takeover_callback_data *cd =
2549 talloc_get_type_abort(callback_data,
2550 struct takeover_callback_data);
2553 for (i = 0; i < cd->nodemap->num; i++) {
2554 if (node_pnn == cd->nodemap->nodes[i].pnn) {
2559 if (i == cd->nodemap->num) {
2560 DEBUG(DEBUG_ERR, (__location__ " invalid PNN %u\n", node_pnn));
2564 if (!cd->node_failed[i]) {
2565 cd->node_failed[i] = true;
2566 cd->fail_callback(ctdb, node_pnn, res, outdata,
2567 cd->fail_callback_data);
2572 make any IP alias changes for public addresses that are necessary
2574 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map_old *nodemap,
2575 uint32_t *force_rebalance_nodes,
2576 client_async_callback fail_callback, void *callback_data)
2579 struct ctdb_public_ip ip;
2581 struct public_ip_list *all_ips, *tmp_ip;
2583 struct timeval timeout;
2584 struct client_async_data *async_data;
2585 struct ctdb_client_control_state *state;
2586 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2587 struct ctdb_ipflags *ipflags;
2588 struct takeover_callback_data *takeover_data;
2589 struct iprealloc_callback_data iprealloc_data;
2594 * ip failover is completely disabled, just send out the
2595 * ipreallocated event.
2597 if (ctdb->tunable.disable_ip_failover != 0) {
2601 ipflags = set_ipflags(ctdb, tmp_ctx, nodemap);
2602 if (ipflags == NULL) {
2603 DEBUG(DEBUG_ERR,("Failed to set IP flags - aborting takeover run\n"));
2604 talloc_free(tmp_ctx);
2608 /* Short-circuit IP allocation if no nodes are in the RUNNING
2609 * runstate yet, since no nodes will be able to host IPs */
2610 can_host_ips = false;
2611 for (i=0; i<nodemap->num; i++) {
2612 if (ipflags[i].runstate == CTDB_RUNSTATE_RUNNING) {
2613 can_host_ips = true;
2616 if (!can_host_ips) {
2617 DEBUG(DEBUG_WARNING,("No nodes available to host public IPs yet\n"));
2621 /* Do the IP reassignment calculations */
2622 ctdb_takeover_run_core(ctdb, ipflags, &all_ips, force_rebalance_nodes);
2624 /* Now tell all nodes to release any public IPs should not
2625 * host. This will be a NOOP on nodes that don't currently
2626 * hold the given IP.
2628 takeover_data = talloc_zero(tmp_ctx, struct takeover_callback_data);
2629 CTDB_NO_MEMORY_FATAL(ctdb, takeover_data);
2631 takeover_data->node_failed = talloc_zero_array(tmp_ctx,
2632 bool, nodemap->num);
2633 CTDB_NO_MEMORY_FATAL(ctdb, takeover_data->node_failed);
2634 takeover_data->fail_callback = fail_callback;
2635 takeover_data->fail_callback_data = callback_data;
2636 takeover_data->nodemap = nodemap;
2638 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2639 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2641 async_data->fail_callback = takeover_run_fail_callback;
2642 async_data->callback_data = takeover_data;
2644 ZERO_STRUCT(ip); /* Avoid valgrind warnings for union */
2646 /* Send a RELEASE_IP to all nodes that should not be hosting
2647 * each IP. For each IP, all but one of these will be
2648 * redundant. However, the redundant ones are used to tell
2649 * nodes which node should be hosting the IP so that commands
2650 * like "ctdb ip" can display a particular nodes idea of who
2651 * is hosting what. */
2652 for (i=0;i<nodemap->num;i++) {
2653 /* don't talk to unconnected nodes, but do talk to banned nodes */
2654 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2658 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2659 if (tmp_ip->pnn == nodemap->nodes[i].pnn) {
2660 /* This node should be serving this
2661 vnn so dont tell it to release the ip
2665 ip.pnn = tmp_ip->pnn;
2666 ip.addr = tmp_ip->addr;
2668 timeout = TAKEOVER_TIMEOUT();
2669 data.dsize = sizeof(ip);
2670 data.dptr = (uint8_t *)&ip;
2671 state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
2672 0, CTDB_CONTROL_RELEASE_IP, 0,
2675 if (state == NULL) {
2676 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_RELEASE_IP to node %u\n", nodemap->nodes[i].pnn));
2677 talloc_free(tmp_ctx);
2681 ctdb_client_async_add(async_data, state);
2684 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2685 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_RELEASE_IP failed\n"));
2686 talloc_free(tmp_ctx);
2689 talloc_free(async_data);
2692 /* For each IP, send a TAKOVER_IP to the node that should be
2693 * hosting it. Many of these will often be redundant (since
2694 * the allocation won't have changed) but they can be useful
2695 * to recover from inconsistencies. */
2696 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2697 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2699 async_data->fail_callback = fail_callback;
2700 async_data->callback_data = callback_data;
2702 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2703 if (tmp_ip->pnn == -1) {
2704 /* this IP won't be taken over */
2708 ip.pnn = tmp_ip->pnn;
2709 ip.addr = tmp_ip->addr;
2711 timeout = TAKEOVER_TIMEOUT();
2712 data.dsize = sizeof(ip);
2713 data.dptr = (uint8_t *)&ip;
2714 state = ctdb_control_send(ctdb, tmp_ip->pnn,
2715 0, CTDB_CONTROL_TAKEOVER_IP, 0,
2716 data, async_data, &timeout, NULL);
2717 if (state == NULL) {
2718 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_TAKEOVER_IP to node %u\n", tmp_ip->pnn));
2719 talloc_free(tmp_ctx);
2723 ctdb_client_async_add(async_data, state);
2725 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2726 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_TAKEOVER_IP failed\n"));
2727 talloc_free(tmp_ctx);
2733 * Tell all nodes to run eventscripts to process the
2734 * "ipreallocated" event. This can do a lot of things,
2735 * including restarting services to reconfigure them if public
2736 * IPs have moved. Once upon a time this event only used to
2739 retry_data = talloc_zero_array(tmp_ctx, bool, nodemap->num);
2740 CTDB_NO_MEMORY_FATAL(ctdb, retry_data);
2741 iprealloc_data.retry_nodes = retry_data;
2742 iprealloc_data.retry_count = 0;
2743 iprealloc_data.fail_callback = fail_callback;
2744 iprealloc_data.fail_callback_data = callback_data;
2745 iprealloc_data.nodemap = nodemap;
2747 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2748 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_IPREALLOCATED,
2749 nodes, 0, TAKEOVER_TIMEOUT(),
2751 NULL, iprealloc_fail_callback,
2754 /* If the control failed then we should retry to any
2755 * nodes flagged by iprealloc_fail_callback using the
2756 * EVENTSCRIPT control. This is a best-effort at
2757 * backward compatiblity when running a mixed cluster
2758 * where some nodes have not yet been upgraded to
2759 * support the IPREALLOCATED control.
2761 DEBUG(DEBUG_WARNING,
2762 ("Retry ipreallocated to some nodes using eventscript control\n"));
2764 nodes = talloc_array(tmp_ctx, uint32_t,
2765 iprealloc_data.retry_count);
2766 CTDB_NO_MEMORY_FATAL(ctdb, nodes);
2769 for (i=0; i<nodemap->num; i++) {
2770 if (iprealloc_data.retry_nodes[i]) {
2776 data.dptr = discard_const("ipreallocated");
2777 data.dsize = strlen((char *)data.dptr) + 1;
2778 ret = ctdb_client_async_control(ctdb,
2779 CTDB_CONTROL_RUN_EVENTSCRIPTS,
2780 nodes, 0, TAKEOVER_TIMEOUT(),
2782 NULL, fail_callback,
2785 DEBUG(DEBUG_ERR, (__location__ " failed to send control to run eventscripts with \"ipreallocated\"\n"));
2789 talloc_free(tmp_ctx);
2795 destroy a ctdb_client_ip structure
2797 static int ctdb_client_ip_destructor(struct ctdb_client_ip *ip)
2799 DEBUG(DEBUG_DEBUG,("destroying client tcp for %s:%u (client_id %u)\n",
2800 ctdb_addr_to_str(&ip->addr),
2801 ntohs(ip->addr.ip.sin_port),
2804 DLIST_REMOVE(ip->ctdb->client_ip_list, ip);
2809 called by a client to inform us of a TCP connection that it is managing
2810 that should tickled with an ACK when IP takeover is done
2812 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
2815 struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
2816 struct ctdb_connection *tcp_sock = NULL;
2817 struct ctdb_tcp_list *tcp;
2818 struct ctdb_connection t;
2821 struct ctdb_client_ip *ip;
2822 struct ctdb_vnn *vnn;
2823 ctdb_sock_addr addr;
2825 /* If we don't have public IPs, tickles are useless */
2826 if (ctdb->vnn == NULL) {
2830 tcp_sock = (struct ctdb_connection *)indata.dptr;
2832 addr = tcp_sock->src;
2833 ctdb_canonicalize_ip(&addr, &tcp_sock->src);
2834 addr = tcp_sock->dst;
2835 ctdb_canonicalize_ip(&addr, &tcp_sock->dst);
2838 memcpy(&addr, &tcp_sock->dst, sizeof(addr));
2839 vnn = find_public_ip_vnn(ctdb, &addr);
2841 switch (addr.sa.sa_family) {
2843 if (ntohl(addr.ip.sin_addr.s_addr) != INADDR_LOOPBACK) {
2844 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public address.\n",
2845 ctdb_addr_to_str(&addr)));
2849 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public ipv6 address.\n",
2850 ctdb_addr_to_str(&addr)));
2853 DEBUG(DEBUG_ERR,(__location__ " Unknown family type %d\n", addr.sa.sa_family));
2859 if (vnn->pnn != ctdb->pnn) {
2860 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
2861 ctdb_addr_to_str(&addr),
2862 client_id, client->pid));
2863 /* failing this call will tell smbd to die */
2867 ip = talloc(client, struct ctdb_client_ip);
2868 CTDB_NO_MEMORY(ctdb, ip);
2872 ip->client_id = client_id;
2873 talloc_set_destructor(ip, ctdb_client_ip_destructor);
2874 DLIST_ADD(ctdb->client_ip_list, ip);
2876 tcp = talloc(client, struct ctdb_tcp_list);
2877 CTDB_NO_MEMORY(ctdb, tcp);
2879 tcp->connection.src = tcp_sock->src;
2880 tcp->connection.dst = tcp_sock->dst;
2882 DLIST_ADD(client->tcp_list, tcp);
2884 t.src = tcp_sock->src;
2885 t.dst = tcp_sock->dst;
2887 data.dptr = (uint8_t *)&t;
2888 data.dsize = sizeof(t);
2890 switch (addr.sa.sa_family) {
2892 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2893 (unsigned)ntohs(tcp_sock->dst.ip.sin_port),
2894 ctdb_addr_to_str(&tcp_sock->src),
2895 (unsigned)ntohs(tcp_sock->src.ip.sin_port), client_id, client->pid));
2898 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2899 (unsigned)ntohs(tcp_sock->dst.ip6.sin6_port),
2900 ctdb_addr_to_str(&tcp_sock->src),
2901 (unsigned)ntohs(tcp_sock->src.ip6.sin6_port), client_id, client->pid));
2904 DEBUG(DEBUG_ERR,(__location__ " Unknown family %d\n", addr.sa.sa_family));
2908 /* tell all nodes about this tcp connection */
2909 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
2910 CTDB_CONTROL_TCP_ADD,
2911 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
2913 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
2921 find a tcp address on a list
2923 static struct ctdb_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
2924 struct ctdb_connection *tcp)
2928 if (array == NULL) {
2932 for (i=0;i<array->num;i++) {
2933 if (ctdb_same_sockaddr(&array->connections[i].src, &tcp->src) &&
2934 ctdb_same_sockaddr(&array->connections[i].dst, &tcp->dst)) {
2935 return &array->connections[i];
2944 called by a daemon to inform us of a TCP connection that one of its
2945 clients managing that should tickled with an ACK when IP takeover is
2948 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed)
2950 struct ctdb_connection *p = (struct ctdb_connection *)indata.dptr;
2951 struct ctdb_tcp_array *tcparray;
2952 struct ctdb_connection tcp;
2953 struct ctdb_vnn *vnn;
2955 /* If we don't have public IPs, tickles are useless */
2956 if (ctdb->vnn == NULL) {
2960 vnn = find_public_ip_vnn(ctdb, &p->dst);
2962 DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
2963 ctdb_addr_to_str(&p->dst)));
2969 tcparray = vnn->tcp_array;
2971 /* If this is the first tickle */
2972 if (tcparray == NULL) {
2973 tcparray = talloc(vnn, struct ctdb_tcp_array);
2974 CTDB_NO_MEMORY(ctdb, tcparray);
2975 vnn->tcp_array = tcparray;
2978 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_connection));
2979 CTDB_NO_MEMORY(ctdb, tcparray->connections);
2981 tcparray->connections[tcparray->num].src = p->src;
2982 tcparray->connections[tcparray->num].dst = p->dst;
2985 if (tcp_update_needed) {
2986 vnn->tcp_update_needed = true;
2992 /* Do we already have this tickle ?*/
2995 if (ctdb_tcp_find(tcparray, &tcp) != NULL) {
2996 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
2997 ctdb_addr_to_str(&tcp.dst),
2998 ntohs(tcp.dst.ip.sin_port),
3003 /* A new tickle, we must add it to the array */
3004 tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
3005 struct ctdb_connection,
3007 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3009 tcparray->connections[tcparray->num].src = p->src;
3010 tcparray->connections[tcparray->num].dst = p->dst;
3013 DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
3014 ctdb_addr_to_str(&tcp.dst),
3015 ntohs(tcp.dst.ip.sin_port),
3018 if (tcp_update_needed) {
3019 vnn->tcp_update_needed = true;
3027 called by a daemon to inform us of a TCP connection that one of its
3028 clients managing that should tickled with an ACK when IP takeover is
3031 static void ctdb_remove_connection(struct ctdb_context *ctdb, struct ctdb_connection *conn)
3033 struct ctdb_connection *tcpp;
3034 struct ctdb_vnn *vnn = find_public_ip_vnn(ctdb, &conn->dst);
3037 DEBUG(DEBUG_ERR,(__location__ " unable to find public address %s\n",
3038 ctdb_addr_to_str(&conn->dst)));
3042 /* if the array is empty we cant remove it
3043 and we dont need to do anything
3045 if (vnn->tcp_array == NULL) {
3046 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist (array is empty) %s:%u\n",
3047 ctdb_addr_to_str(&conn->dst),
3048 ntohs(conn->dst.ip.sin_port)));
3053 /* See if we know this connection
3054 if we dont know this connection then we dont need to do anything
3056 tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
3058 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist %s:%u\n",
3059 ctdb_addr_to_str(&conn->dst),
3060 ntohs(conn->dst.ip.sin_port)));
3065 /* We need to remove this entry from the array.
3066 Instead of allocating a new array and copying data to it
3067 we cheat and just copy the last entry in the existing array
3068 to the entry that is to be removed and just shring the
3071 *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
3072 vnn->tcp_array->num--;
3074 /* If we deleted the last entry we also need to remove the entire array
3076 if (vnn->tcp_array->num == 0) {
3077 talloc_free(vnn->tcp_array);
3078 vnn->tcp_array = NULL;
3081 vnn->tcp_update_needed = true;
3083 DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
3084 ctdb_addr_to_str(&conn->src),
3085 ntohs(conn->src.ip.sin_port)));
3090 called by a daemon to inform us of a TCP connection that one of its
3091 clients used are no longer needed in the tickle database
3093 int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
3095 struct ctdb_connection *conn = (struct ctdb_connection *)indata.dptr;
3097 /* If we don't have public IPs, tickles are useless */
3098 if (ctdb->vnn == NULL) {
3102 ctdb_remove_connection(ctdb, conn);
3109 Called when another daemon starts - causes all tickles for all
3110 public addresses we are serving to be sent to the new node on the
3111 next check. This actually causes the next scheduled call to
3112 tdb_update_tcp_tickles() to update all nodes. This is simple and
3113 doesn't require careful error handling.
3115 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t pnn)
3117 struct ctdb_vnn *vnn;
3119 DEBUG(DEBUG_INFO, ("Received startup control from node %lu\n",
3120 (unsigned long) pnn));
3122 for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
3123 vnn->tcp_update_needed = true;
3131 called when a client structure goes away - hook to remove
3132 elements from the tcp_list in all daemons
3134 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
3136 while (client->tcp_list) {
3137 struct ctdb_tcp_list *tcp = client->tcp_list;
3138 DLIST_REMOVE(client->tcp_list, tcp);
3139 ctdb_remove_connection(client->ctdb, &tcp->connection);
3144 void ctdb_release_all_ips(struct ctdb_context *ctdb)
3146 struct ctdb_vnn *vnn;
3149 if (ctdb->tunable.disable_ip_failover == 1) {
3153 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3154 if (!ctdb_sys_have_ip(&vnn->public_address)) {
3155 ctdb_vnn_unassign_iface(ctdb, vnn);
3162 /* Don't allow multiple releases at once. Some code,
3163 * particularly ctdb_tickle_sentenced_connections() is
3165 if (vnn->update_in_flight) {
3166 DEBUG(DEBUG_WARNING,
3168 " Not releasing IP %s/%u on interface %s, an update is already in progess\n",
3169 ctdb_addr_to_str(&vnn->public_address),
3170 vnn->public_netmask_bits,
3171 ctdb_vnn_iface_string(vnn)));
3174 vnn->update_in_flight = true;
3176 DEBUG(DEBUG_INFO,("Release of IP %s/%u on interface %s node:-1\n",
3177 ctdb_addr_to_str(&vnn->public_address),
3178 vnn->public_netmask_bits,
3179 ctdb_vnn_iface_string(vnn)));
3181 ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
3182 ctdb_vnn_iface_string(vnn),
3183 ctdb_addr_to_str(&vnn->public_address),
3184 vnn->public_netmask_bits);
3185 release_kill_clients(ctdb, &vnn->public_address);
3186 ctdb_vnn_unassign_iface(ctdb, vnn);
3187 vnn->update_in_flight = false;
3191 DEBUG(DEBUG_NOTICE,(__location__ " Released %d public IPs\n", count));
3196 get list of public IPs
3198 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
3199 struct ctdb_req_control_old *c, TDB_DATA *outdata)
3202 struct ctdb_public_ip_list_old *ips;
3203 struct ctdb_vnn *vnn;
3204 bool only_available = false;
3206 if (c->flags & CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE) {
3207 only_available = true;
3210 /* count how many public ip structures we have */
3212 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3216 len = offsetof(struct ctdb_public_ip_list_old, ips) +
3217 num*sizeof(struct ctdb_public_ip);
3218 ips = talloc_zero_size(outdata, len);
3219 CTDB_NO_MEMORY(ctdb, ips);
3222 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3223 if (only_available && !ctdb_vnn_available(ctdb, vnn)) {
3226 ips->ips[i].pnn = vnn->pnn;
3227 ips->ips[i].addr = vnn->public_address;
3231 len = offsetof(struct ctdb_public_ip_list_old, ips) +
3232 i*sizeof(struct ctdb_public_ip);
3234 outdata->dsize = len;
3235 outdata->dptr = (uint8_t *)ips;
3241 int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
3242 struct ctdb_req_control_old *c,
3247 ctdb_sock_addr *addr;
3248 struct ctdb_public_ip_info_old *info;
3249 struct ctdb_vnn *vnn;
3251 addr = (ctdb_sock_addr *)indata.dptr;
3253 vnn = find_public_ip_vnn(ctdb, addr);
3255 /* if it is not a public ip it could be our 'single ip' */
3256 if (ctdb->single_ip_vnn) {
3257 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, addr)) {
3258 vnn = ctdb->single_ip_vnn;
3263 DEBUG(DEBUG_ERR,(__location__ " Could not get public ip info, "
3264 "'%s'not a public address\n",
3265 ctdb_addr_to_str(addr)));
3269 /* count how many public ip structures we have */
3271 for (;vnn->ifaces[num];) {
3275 len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
3276 num*sizeof(struct ctdb_iface);
3277 info = talloc_zero_size(outdata, len);
3278 CTDB_NO_MEMORY(ctdb, info);
3280 info->ip.addr = vnn->public_address;
3281 info->ip.pnn = vnn->pnn;
3282 info->active_idx = 0xFFFFFFFF;
3284 for (i=0; vnn->ifaces[i]; i++) {
3285 struct ctdb_interface *cur;
3287 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
3289 DEBUG(DEBUG_CRIT, (__location__ " internal error iface[%s] unknown\n",
3293 if (vnn->iface == cur) {
3294 info->active_idx = i;
3296 strncpy(info->ifaces[i].name, cur->name, sizeof(info->ifaces[i].name)-1);
3297 info->ifaces[i].link_state = cur->link_up;
3298 info->ifaces[i].references = cur->references;
3301 len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
3302 i*sizeof(struct ctdb_iface);
3304 outdata->dsize = len;
3305 outdata->dptr = (uint8_t *)info;
3310 int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
3311 struct ctdb_req_control_old *c,
3315 struct ctdb_control_get_ifaces *ifaces;
3316 struct ctdb_interface *cur;
3318 /* count how many public ip structures we have */
3320 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3324 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3325 num*sizeof(struct ctdb_iface);
3326 ifaces = talloc_zero_size(outdata, len);
3327 CTDB_NO_MEMORY(ctdb, ifaces);
3330 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3331 strcpy(ifaces->ifaces[i].name, cur->name);
3332 ifaces->ifaces[i].link_state = cur->link_up;
3333 ifaces->ifaces[i].references = cur->references;
3337 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3338 i*sizeof(struct ctdb_iface);
3340 outdata->dsize = len;
3341 outdata->dptr = (uint8_t *)ifaces;
3346 int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
3347 struct ctdb_req_control_old *c,
3350 struct ctdb_iface *info;
3351 struct ctdb_interface *iface;
3352 bool link_up = false;
3354 info = (struct ctdb_iface *)indata.dptr;
3356 if (info->name[CTDB_IFACE_SIZE] != '\0') {
3357 int len = strnlen(info->name, CTDB_IFACE_SIZE);
3358 DEBUG(DEBUG_ERR, (__location__ " name[%*.*s] not terminated\n",
3359 len, len, info->name));
3363 switch (info->link_state) {
3371 DEBUG(DEBUG_ERR, (__location__ " link_state[%u] invalid\n",
3372 (unsigned int)info->link_state));
3376 if (info->references != 0) {
3377 DEBUG(DEBUG_ERR, (__location__ " references[%u] should be 0\n",
3378 (unsigned int)info->references));
3382 iface = ctdb_find_iface(ctdb, info->name);
3383 if (iface == NULL) {
3387 if (link_up == iface->link_up) {
3391 DEBUG(iface->link_up?DEBUG_ERR:DEBUG_NOTICE,
3392 ("iface[%s] has changed it's link status %s => %s\n",
3394 iface->link_up?"up":"down",
3395 link_up?"up":"down"));
3397 iface->link_up = link_up;
3403 structure containing the listening socket and the list of tcp connections
3404 that the ctdb daemon is to kill
3406 struct ctdb_kill_tcp {
3407 struct ctdb_vnn *vnn;
3408 struct ctdb_context *ctdb;
3410 struct tevent_fd *fde;
3411 trbt_tree_t *connections;
3416 a tcp connection that is to be killed
3418 struct ctdb_killtcp_con {
3419 ctdb_sock_addr src_addr;
3420 ctdb_sock_addr dst_addr;
3422 struct ctdb_kill_tcp *killtcp;
3425 /* this function is used to create a key to represent this socketpair
3426 in the killtcp tree.
3427 this key is used to insert and lookup matching socketpairs that are
3428 to be tickled and RST
3430 #define KILLTCP_KEYLEN 10
3431 static uint32_t *killtcp_key(ctdb_sock_addr *src, ctdb_sock_addr *dst)
3433 static uint32_t key[KILLTCP_KEYLEN];
3435 bzero(key, sizeof(key));
3437 if (src->sa.sa_family != dst->sa.sa_family) {
3438 DEBUG(DEBUG_ERR, (__location__ " ERROR, different families passed :%u vs %u\n", src->sa.sa_family, dst->sa.sa_family));
3442 switch (src->sa.sa_family) {
3444 key[0] = dst->ip.sin_addr.s_addr;
3445 key[1] = src->ip.sin_addr.s_addr;
3446 key[2] = dst->ip.sin_port;
3447 key[3] = src->ip.sin_port;
3450 uint32_t *dst6_addr32 =
3451 (uint32_t *)&(dst->ip6.sin6_addr.s6_addr);
3452 uint32_t *src6_addr32 =
3453 (uint32_t *)&(src->ip6.sin6_addr.s6_addr);
3454 key[0] = dst6_addr32[3];
3455 key[1] = src6_addr32[3];
3456 key[2] = dst6_addr32[2];
3457 key[3] = src6_addr32[2];
3458 key[4] = dst6_addr32[1];
3459 key[5] = src6_addr32[1];
3460 key[6] = dst6_addr32[0];
3461 key[7] = src6_addr32[0];
3462 key[8] = dst->ip6.sin6_port;
3463 key[9] = src->ip6.sin6_port;
3467 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", src->sa.sa_family));
3475 called when we get a read event on the raw socket
3477 static void capture_tcp_handler(struct tevent_context *ev,
3478 struct tevent_fd *fde,
3479 uint16_t flags, void *private_data)
3481 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3482 struct ctdb_killtcp_con *con;
3483 ctdb_sock_addr src, dst;
3484 uint32_t ack_seq, seq;
3486 if (!(flags & TEVENT_FD_READ)) {
3490 if (ctdb_sys_read_tcp_packet(killtcp->capture_fd,
3491 killtcp->private_data,
3493 &ack_seq, &seq) != 0) {
3494 /* probably a non-tcp ACK packet */
3498 /* check if we have this guy in our list of connections
3501 con = trbt_lookuparray32(killtcp->connections,
3502 KILLTCP_KEYLEN, killtcp_key(&src, &dst));
3504 /* no this was some other packet we can just ignore */
3508 /* This one has been tickled !
3509 now reset him and remove him from the list.
3511 DEBUG(DEBUG_INFO, ("sending a tcp reset to kill connection :%d -> %s:%d\n",
3512 ntohs(con->dst_addr.ip.sin_port),
3513 ctdb_addr_to_str(&con->src_addr),
3514 ntohs(con->src_addr.ip.sin_port)));
3516 ctdb_sys_send_tcp(&con->dst_addr, &con->src_addr, ack_seq, seq, 1);
3521 /* when traversing the list of all tcp connections to send tickle acks to
3522 (so that we can capture the ack coming back and kill the connection
3524 this callback is called for each connection we are currently trying to kill
3526 static int tickle_connection_traverse(void *param, void *data)
3528 struct ctdb_killtcp_con *con = talloc_get_type(data, struct ctdb_killtcp_con);
3530 /* have tried too many times, just give up */
3531 if (con->count >= 5) {
3532 /* can't delete in traverse: reparent to delete_cons */
3533 talloc_steal(param, con);
3537 /* othervise, try tickling it again */
3540 (ctdb_sock_addr *)&con->dst_addr,
3541 (ctdb_sock_addr *)&con->src_addr,
3548 called every second until all sentenced connections have been reset
3550 static void ctdb_tickle_sentenced_connections(struct tevent_context *ev,
3551 struct tevent_timer *te,
3552 struct timeval t, void *private_data)
3554 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3555 void *delete_cons = talloc_new(NULL);
3557 /* loop over all connections sending tickle ACKs */
3558 trbt_traversearray32(killtcp->connections, KILLTCP_KEYLEN, tickle_connection_traverse, delete_cons);
3560 /* now we've finished traverse, it's safe to do deletion. */
3561 talloc_free(delete_cons);
3563 /* If there are no more connections to kill we can remove the
3564 entire killtcp structure
3566 if ( (killtcp->connections == NULL) ||
3567 (killtcp->connections->root == NULL) ) {
3568 talloc_free(killtcp);
3572 /* try tickling them again in a seconds time
3574 tevent_add_timer(killtcp->ctdb->ev, killtcp,
3575 timeval_current_ofs(1, 0),
3576 ctdb_tickle_sentenced_connections, killtcp);
3580 destroy the killtcp structure
3582 static int ctdb_killtcp_destructor(struct ctdb_kill_tcp *killtcp)
3584 struct ctdb_vnn *tmpvnn;
3586 /* verify that this vnn is still active */
3587 for (tmpvnn = killtcp->ctdb->vnn; tmpvnn; tmpvnn = tmpvnn->next) {
3588 if (tmpvnn == killtcp->vnn) {
3593 if (tmpvnn == NULL) {
3597 if (killtcp->vnn->killtcp != killtcp) {
3601 killtcp->vnn->killtcp = NULL;
3607 /* nothing fancy here, just unconditionally replace any existing
3608 connection structure with the new one.
3610 dont even free the old one if it did exist, that one is talloc_stolen
3611 by the same node in the tree anyway and will be deleted when the new data
3614 static void *add_killtcp_callback(void *parm, void *data)
3620 add a tcp socket to the list of connections we want to RST
3622 static int ctdb_killtcp_add_connection(struct ctdb_context *ctdb,
3626 ctdb_sock_addr src, dst;
3627 struct ctdb_kill_tcp *killtcp;
3628 struct ctdb_killtcp_con *con;
3629 struct ctdb_vnn *vnn;
3631 ctdb_canonicalize_ip(s, &src);
3632 ctdb_canonicalize_ip(d, &dst);
3634 vnn = find_public_ip_vnn(ctdb, &dst);
3636 vnn = find_public_ip_vnn(ctdb, &src);
3639 /* if it is not a public ip it could be our 'single ip' */
3640 if (ctdb->single_ip_vnn) {
3641 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, &dst)) {
3642 vnn = ctdb->single_ip_vnn;
3647 DEBUG(DEBUG_ERR,(__location__ " Could not killtcp, not a public address\n"));
3651 killtcp = vnn->killtcp;
3653 /* If this is the first connection to kill we must allocate
3656 if (killtcp == NULL) {
3657 killtcp = talloc_zero(vnn, struct ctdb_kill_tcp);
3658 CTDB_NO_MEMORY(ctdb, killtcp);
3661 killtcp->ctdb = ctdb;
3662 killtcp->capture_fd = -1;
3663 killtcp->connections = trbt_create(killtcp, 0);
3665 vnn->killtcp = killtcp;
3666 talloc_set_destructor(killtcp, ctdb_killtcp_destructor);
3671 /* create a structure that describes this connection we want to
3672 RST and store it in killtcp->connections
3674 con = talloc(killtcp, struct ctdb_killtcp_con);
3675 CTDB_NO_MEMORY(ctdb, con);
3676 con->src_addr = src;
3677 con->dst_addr = dst;
3679 con->killtcp = killtcp;
3682 trbt_insertarray32_callback(killtcp->connections,
3683 KILLTCP_KEYLEN, killtcp_key(&con->dst_addr, &con->src_addr),
3684 add_killtcp_callback, con);
3687 If we dont have a socket to listen on yet we must create it
3689 if (killtcp->capture_fd == -1) {
3690 const char *iface = ctdb_vnn_iface_string(vnn);
3691 killtcp->capture_fd = ctdb_sys_open_capture_socket(iface, &killtcp->private_data);
3692 if (killtcp->capture_fd == -1) {
3693 DEBUG(DEBUG_CRIT,(__location__ " Failed to open capturing "
3694 "socket on iface '%s' for killtcp (%s)\n",
3695 iface, strerror(errno)));
3701 if (killtcp->fde == NULL) {
3702 killtcp->fde = tevent_add_fd(ctdb->ev, killtcp,
3703 killtcp->capture_fd,
3705 capture_tcp_handler, killtcp);
3706 tevent_fd_set_auto_close(killtcp->fde);
3708 /* We also need to set up some events to tickle all these connections
3709 until they are all reset
3711 tevent_add_timer(ctdb->ev, killtcp, timeval_current_ofs(1, 0),
3712 ctdb_tickle_sentenced_connections, killtcp);
3715 /* tickle him once now */
3724 talloc_free(vnn->killtcp);
3725 vnn->killtcp = NULL;
3730 kill a TCP connection.
3732 int32_t ctdb_control_kill_tcp(struct ctdb_context *ctdb, TDB_DATA indata)
3734 struct ctdb_connection *killtcp = (struct ctdb_connection *)indata.dptr;
3736 return ctdb_killtcp_add_connection(ctdb, &killtcp->src, &killtcp->dst);
3740 called by a daemon to inform us of the entire list of TCP tickles for
3741 a particular public address.
3742 this control should only be sent by the node that is currently serving
3743 that public address.
3745 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
3747 struct ctdb_tickle_list_old *list = (struct ctdb_tickle_list_old *)indata.dptr;
3748 struct ctdb_tcp_array *tcparray;
3749 struct ctdb_vnn *vnn;
3751 /* We must at least have tickles.num or else we cant verify the size
3752 of the received data blob
3754 if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)) {
3755 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list. Not enough data for the tickle.num field\n"));
3759 /* verify that the size of data matches what we expect */
3760 if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)
3761 + sizeof(struct ctdb_connection) * list->num) {
3762 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list\n"));
3766 DEBUG(DEBUG_INFO, ("Received tickle update for public address %s\n",
3767 ctdb_addr_to_str(&list->addr)));
3769 vnn = find_public_ip_vnn(ctdb, &list->addr);
3771 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
3772 ctdb_addr_to_str(&list->addr)));
3777 /* remove any old ticklelist we might have */
3778 talloc_free(vnn->tcp_array);
3779 vnn->tcp_array = NULL;
3781 tcparray = talloc(vnn, struct ctdb_tcp_array);
3782 CTDB_NO_MEMORY(ctdb, tcparray);
3784 tcparray->num = list->num;
3786 tcparray->connections = talloc_array(tcparray, struct ctdb_connection, tcparray->num);
3787 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3789 memcpy(tcparray->connections, &list->connections[0],
3790 sizeof(struct ctdb_connection)*tcparray->num);
3792 /* We now have a new fresh tickle list array for this vnn */
3793 vnn->tcp_array = tcparray;
3799 called to return the full list of tickles for the puclic address associated
3800 with the provided vnn
3802 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
3804 ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
3805 struct ctdb_tickle_list_old *list;
3806 struct ctdb_tcp_array *tcparray;
3808 struct ctdb_vnn *vnn;
3810 vnn = find_public_ip_vnn(ctdb, addr);
3812 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
3813 ctdb_addr_to_str(addr)));
3818 tcparray = vnn->tcp_array;
3820 num = tcparray->num;
3825 outdata->dsize = offsetof(struct ctdb_tickle_list_old, connections)
3826 + sizeof(struct ctdb_connection) * num;
3828 outdata->dptr = talloc_size(outdata, outdata->dsize);
3829 CTDB_NO_MEMORY(ctdb, outdata->dptr);
3830 list = (struct ctdb_tickle_list_old *)outdata->dptr;
3835 memcpy(&list->connections[0], tcparray->connections,
3836 sizeof(struct ctdb_connection) * num);
3844 set the list of all tcp tickles for a public address
3846 static int ctdb_send_set_tcp_tickles_for_ip(struct ctdb_context *ctdb,
3847 ctdb_sock_addr *addr,
3848 struct ctdb_tcp_array *tcparray)
3852 struct ctdb_tickle_list_old *list;
3855 num = tcparray->num;
3860 data.dsize = offsetof(struct ctdb_tickle_list_old, connections) +
3861 sizeof(struct ctdb_connection) * num;
3862 data.dptr = talloc_size(ctdb, data.dsize);
3863 CTDB_NO_MEMORY(ctdb, data.dptr);
3865 list = (struct ctdb_tickle_list_old *)data.dptr;
3869 memcpy(&list->connections[0], tcparray->connections, sizeof(struct ctdb_connection) * num);
3872 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
3873 CTDB_CONTROL_SET_TCP_TICKLE_LIST,
3874 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
3876 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
3880 talloc_free(data.dptr);
3887 perform tickle updates if required
3889 static void ctdb_update_tcp_tickles(struct tevent_context *ev,
3890 struct tevent_timer *te,
3891 struct timeval t, void *private_data)
3893 struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
3895 struct ctdb_vnn *vnn;
3897 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3898 /* we only send out updates for public addresses that
3901 if (ctdb->pnn != vnn->pnn) {
3904 /* We only send out the updates if we need to */
3905 if (!vnn->tcp_update_needed) {
3908 ret = ctdb_send_set_tcp_tickles_for_ip(ctdb,
3909 &vnn->public_address,
3912 DEBUG(DEBUG_ERR,("Failed to send the tickle update for public address %s\n",
3913 ctdb_addr_to_str(&vnn->public_address)));
3916 ("Sent tickle update for public address %s\n",
3917 ctdb_addr_to_str(&vnn->public_address)));
3918 vnn->tcp_update_needed = false;
3922 tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
3923 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
3924 ctdb_update_tcp_tickles, ctdb);
3928 start periodic update of tcp tickles
3930 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
3932 ctdb->tickle_update_context = talloc_new(ctdb);
3934 tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
3935 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
3936 ctdb_update_tcp_tickles, ctdb);
3942 struct control_gratious_arp {
3943 struct ctdb_context *ctdb;
3944 ctdb_sock_addr addr;
3950 send a control_gratuitous arp
3952 static void send_gratious_arp(struct tevent_context *ev,
3953 struct tevent_timer *te,
3954 struct timeval t, void *private_data)
3957 struct control_gratious_arp *arp = talloc_get_type(private_data,
3958 struct control_gratious_arp);
3960 ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
3962 DEBUG(DEBUG_ERR,(__location__ " sending of gratious arp on iface '%s' failed (%s)\n",
3963 arp->iface, strerror(errno)));
3968 if (arp->count == CTDB_ARP_REPEAT) {
3973 tevent_add_timer(arp->ctdb->ev, arp,
3974 timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
3975 send_gratious_arp, arp);
3982 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
3984 struct ctdb_addr_info_old *gratious_arp = (struct ctdb_addr_info_old *)indata.dptr;
3985 struct control_gratious_arp *arp;
3987 /* verify the size of indata */
3988 if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
3989 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
3990 (unsigned)indata.dsize,
3991 (unsigned)offsetof(struct ctdb_addr_info_old, iface)));
3995 ( offsetof(struct ctdb_addr_info_old, iface)
3996 + gratious_arp->len ) ){
3998 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
3999 "but should be %u bytes\n",
4000 (unsigned)indata.dsize,
4001 (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+gratious_arp->len)));
4006 arp = talloc(ctdb, struct control_gratious_arp);
4007 CTDB_NO_MEMORY(ctdb, arp);
4010 arp->addr = gratious_arp->addr;
4011 arp->iface = talloc_strdup(arp, gratious_arp->iface);
4012 CTDB_NO_MEMORY(ctdb, arp->iface);
4015 tevent_add_timer(arp->ctdb->ev, arp,
4016 timeval_zero(), send_gratious_arp, arp);
4021 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
4023 struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
4026 /* verify the size of indata */
4027 if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
4028 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
4032 ( offsetof(struct ctdb_addr_info_old, iface)
4035 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
4036 "but should be %u bytes\n",
4037 (unsigned)indata.dsize,
4038 (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
4042 DEBUG(DEBUG_NOTICE,("Add IP %s\n", ctdb_addr_to_str(&pub->addr)));
4044 ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
4047 DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
4054 struct delete_ip_callback_state {
4055 struct ctdb_req_control_old *c;
4059 called when releaseip event finishes for del_public_address
4061 static void delete_ip_callback(struct ctdb_context *ctdb,
4062 int32_t status, TDB_DATA data,
4063 const char *errormsg,
4066 struct delete_ip_callback_state *state =
4067 talloc_get_type(private_data, struct delete_ip_callback_state);
4069 /* If release failed then fail. */
4070 ctdb_request_control_reply(ctdb, state->c, NULL, status, errormsg);
4071 talloc_free(private_data);
4074 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb,
4075 struct ctdb_req_control_old *c,
4076 TDB_DATA indata, bool *async_reply)
4078 struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
4079 struct ctdb_vnn *vnn;
4081 /* verify the size of indata */
4082 if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
4083 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
4087 ( offsetof(struct ctdb_addr_info_old, iface)
4090 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
4091 "but should be %u bytes\n",
4092 (unsigned)indata.dsize,
4093 (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
4097 DEBUG(DEBUG_NOTICE,("Delete IP %s\n", ctdb_addr_to_str(&pub->addr)));
4099 /* walk over all public addresses until we find a match */
4100 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
4101 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
4102 if (vnn->pnn == ctdb->pnn) {
4103 struct delete_ip_callback_state *state;
4104 struct ctdb_public_ip *ip;
4108 vnn->delete_pending = true;
4110 state = talloc(ctdb,
4111 struct delete_ip_callback_state);
4112 CTDB_NO_MEMORY(ctdb, state);
4115 ip = talloc(state, struct ctdb_public_ip);
4118 (__location__ " Out of memory\n"));
4123 ip->addr = pub->addr;
4125 data.dsize = sizeof(struct ctdb_public_ip);
4126 data.dptr = (unsigned char *)ip;
4128 ret = ctdb_daemon_send_control(ctdb,
4131 CTDB_CONTROL_RELEASE_IP,
4138 (__location__ "Unable to send "
4139 "CTDB_CONTROL_RELEASE_IP\n"));
4144 state->c = talloc_steal(state, c);
4145 *async_reply = true;
4147 /* This IP is not hosted on the
4148 * current node so just delete it
4150 do_delete_ip(ctdb, vnn);
4157 DEBUG(DEBUG_ERR,("Delete IP of unknown public IP address %s\n",
4158 ctdb_addr_to_str(&pub->addr)));
4163 struct ipreallocated_callback_state {
4164 struct ctdb_req_control_old *c;
4167 static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
4168 int status, void *p)
4170 struct ipreallocated_callback_state *state =
4171 talloc_get_type(p, struct ipreallocated_callback_state);
4175 (" \"ipreallocated\" event script failed (status %d)\n",
4177 if (status == -ETIME) {
4178 ctdb_ban_self(ctdb);
4182 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
4186 /* A control to run the ipreallocated event */
4187 int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
4188 struct ctdb_req_control_old *c,
4192 struct ipreallocated_callback_state *state;
4194 state = talloc(ctdb, struct ipreallocated_callback_state);
4195 CTDB_NO_MEMORY(ctdb, state);
4197 DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
4199 ret = ctdb_event_script_callback(ctdb, state,
4200 ctdb_ipreallocated_callback, state,
4201 CTDB_EVENT_IPREALLOCATED,
4205 DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
4210 /* tell the control that we will be reply asynchronously */
4211 state->c = talloc_steal(state, c);
4212 *async_reply = true;
4218 /* This function is called from the recovery daemon to verify that a remote
4219 node has the expected ip allocation.
4220 This is verified against ctdb->ip_tree
4222 int verify_remote_ip_allocation(struct ctdb_context *ctdb,
4223 struct ctdb_public_ip_list_old *ips,
4226 struct public_ip_list *tmp_ip;
4229 if (ctdb->ip_tree == NULL) {
4230 /* dont know the expected allocation yet, assume remote node
4239 for (i=0; i<ips->num; i++) {
4240 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ips->ips[i].addr));
4241 if (tmp_ip == NULL) {
4242 DEBUG(DEBUG_ERR,("Node %u has new or unknown public IP %s\n", pnn, ctdb_addr_to_str(&ips->ips[i].addr)));
4246 if (tmp_ip->pnn == -1 || ips->ips[i].pnn == -1) {
4250 if (tmp_ip->pnn != ips->ips[i].pnn) {
4252 ("Inconsistent IP allocation - node %u thinks %s is held by node %u while it is assigned to node %u\n",
4254 ctdb_addr_to_str(&ips->ips[i].addr),
4255 ips->ips[i].pnn, tmp_ip->pnn));
4263 int update_ip_assignment_tree(struct ctdb_context *ctdb, struct ctdb_public_ip *ip)
4265 struct public_ip_list *tmp_ip;
4267 /* IP tree is never built if DisableIPFailover is set */
4268 if (ctdb->tunable.disable_ip_failover != 0) {
4272 if (ctdb->ip_tree == NULL) {
4273 DEBUG(DEBUG_ERR,("No ctdb->ip_tree yet. Failed to update ip assignment\n"));
4277 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ip->addr));
4278 if (tmp_ip == NULL) {
4279 DEBUG(DEBUG_ERR,(__location__ " Could not find record for address %s, update ip\n", ctdb_addr_to_str(&ip->addr)));
4283 DEBUG(DEBUG_NOTICE,("Updated ip assignment tree for ip : %s from node %u to node %u\n", ctdb_addr_to_str(&ip->addr), tmp_ip->pnn, ip->pnn));
4284 tmp_ip->pnn = ip->pnn;
4289 void clear_ip_assignment_tree(struct ctdb_context *ctdb)
4291 TALLOC_FREE(ctdb->ip_tree);
4294 struct ctdb_reloadips_handle {
4295 struct ctdb_context *ctdb;
4296 struct ctdb_req_control_old *c;
4300 struct tevent_fd *fde;
4303 static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
4305 if (h == h->ctdb->reload_ips) {
4306 h->ctdb->reload_ips = NULL;
4309 ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
4312 ctdb_kill(h->ctdb, h->child, SIGKILL);
4316 static void ctdb_reloadips_timeout_event(struct tevent_context *ev,
4317 struct tevent_timer *te,
4318 struct timeval t, void *private_data)
4320 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
4325 static void ctdb_reloadips_child_handler(struct tevent_context *ev,
4326 struct tevent_fd *fde,
4327 uint16_t flags, void *private_data)
4329 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
4334 ret = sys_read(h->fd[0], &res, 1);
4335 if (ret < 1 || res != 0) {
4336 DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
4344 static int ctdb_reloadips_child(struct ctdb_context *ctdb)
4346 TALLOC_CTX *mem_ctx = talloc_new(NULL);
4347 struct ctdb_public_ip_list_old *ips;
4348 struct ctdb_vnn *vnn;
4349 struct client_async_data *async_data;
4350 struct timeval timeout;
4352 struct ctdb_client_control_state *state;
4356 CTDB_NO_MEMORY(ctdb, mem_ctx);
4358 /* Read IPs from local node */
4359 ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(),
4360 CTDB_CURRENT_NODE, mem_ctx, &ips);
4363 ("Unable to fetch public IPs from local node\n"));
4364 talloc_free(mem_ctx);
4368 /* Read IPs file - this is safe since this is a child process */
4370 if (ctdb_set_public_addresses(ctdb, false) != 0) {
4371 DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
4372 talloc_free(mem_ctx);
4376 async_data = talloc_zero(mem_ctx, struct client_async_data);
4377 CTDB_NO_MEMORY(ctdb, async_data);
4379 /* Compare IPs between node and file for IPs to be deleted */
4380 for (i = 0; i < ips->num; i++) {
4382 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4383 if (ctdb_same_ip(&vnn->public_address,
4384 &ips->ips[i].addr)) {
4385 /* IP is still in file */
4391 /* Delete IP ips->ips[i] */
4392 struct ctdb_addr_info_old *pub;
4395 ("IP %s no longer configured, deleting it\n",
4396 ctdb_addr_to_str(&ips->ips[i].addr)));
4398 pub = talloc_zero(mem_ctx, struct ctdb_addr_info_old);
4399 CTDB_NO_MEMORY(ctdb, pub);
4401 pub->addr = ips->ips[i].addr;
4405 timeout = TAKEOVER_TIMEOUT();
4407 data.dsize = offsetof(struct ctdb_addr_info_old,
4409 data.dptr = (uint8_t *)pub;
4411 state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
4412 CTDB_CONTROL_DEL_PUBLIC_IP,
4413 0, data, async_data,
4415 if (state == NULL) {
4418 " failed sending CTDB_CONTROL_DEL_PUBLIC_IP\n"));
4422 ctdb_client_async_add(async_data, state);
4426 /* Compare IPs between node and file for IPs to be added */
4428 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4429 for (i = 0; i < ips->num; i++) {
4430 if (ctdb_same_ip(&vnn->public_address,
4431 &ips->ips[i].addr)) {
4432 /* IP already on node */
4436 if (i == ips->num) {
4437 /* Add IP ips->ips[i] */
4438 struct ctdb_addr_info_old *pub;
4439 const char *ifaces = NULL;
4444 ("New IP %s configured, adding it\n",
4445 ctdb_addr_to_str(&vnn->public_address)));
4447 uint32_t pnn = ctdb_get_pnn(ctdb);
4449 data.dsize = sizeof(pnn);
4450 data.dptr = (uint8_t *)&pnn;
4452 ret = ctdb_client_send_message(
4454 CTDB_BROADCAST_CONNECTED,
4455 CTDB_SRVID_REBALANCE_NODE,
4458 DEBUG(DEBUG_WARNING,
4459 ("Failed to send message to force node reallocation - IPs may be unbalanced\n"));
4465 ifaces = vnn->ifaces[0];
4467 while (vnn->ifaces[iface] != NULL) {
4468 ifaces = talloc_asprintf(vnn, "%s,%s", ifaces,
4469 vnn->ifaces[iface]);
4473 len = strlen(ifaces) + 1;
4474 pub = talloc_zero_size(mem_ctx,
4475 offsetof(struct ctdb_addr_info_old, iface) + len);
4476 CTDB_NO_MEMORY(ctdb, pub);
4478 pub->addr = vnn->public_address;
4479 pub->mask = vnn->public_netmask_bits;
4481 memcpy(&pub->iface[0], ifaces, pub->len);
4483 timeout = TAKEOVER_TIMEOUT();
4485 data.dsize = offsetof(struct ctdb_addr_info_old,
4487 data.dptr = (uint8_t *)pub;
4489 state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
4490 CTDB_CONTROL_ADD_PUBLIC_IP,
4491 0, data, async_data,
4493 if (state == NULL) {
4496 " failed sending CTDB_CONTROL_ADD_PUBLIC_IP\n"));
4500 ctdb_client_async_add(async_data, state);
4504 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
4505 DEBUG(DEBUG_ERR,(__location__ " Add/delete IPs failed\n"));
4509 talloc_free(mem_ctx);
4513 talloc_free(mem_ctx);
4517 /* This control is sent to force the node to re-read the public addresses file
4518 and drop any addresses we should nnot longer host, and add new addresses
4519 that we are now able to host
4521 int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control_old *c, bool *async_reply)
4523 struct ctdb_reloadips_handle *h;
4524 pid_t parent = getpid();
4526 if (ctdb->reload_ips != NULL) {
4527 talloc_free(ctdb->reload_ips);
4528 ctdb->reload_ips = NULL;
4531 h = talloc(ctdb, struct ctdb_reloadips_handle);
4532 CTDB_NO_MEMORY(ctdb, h);
4537 if (pipe(h->fd) == -1) {
4538 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
4543 h->child = ctdb_fork(ctdb);
4544 if (h->child == (pid_t)-1) {
4545 DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
4553 if (h->child == 0) {
4554 signed char res = 0;
4557 debug_extra = talloc_asprintf(NULL, "reloadips:");
4559 ctdb_set_process_name("ctdb_reloadips");
4560 if (switch_from_server_to_client(ctdb, "reloadips-child") != 0) {
4561 DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
4564 res = ctdb_reloadips_child(ctdb);
4566 DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
4570 sys_write(h->fd[1], &res, 1);
4571 /* make sure we die when our parent dies */
4572 while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
4578 h->c = talloc_steal(h, c);
4581 set_close_on_exec(h->fd[0]);
4583 talloc_set_destructor(h, ctdb_reloadips_destructor);
4586 h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ,
4587 ctdb_reloadips_child_handler, (void *)h);
4588 tevent_fd_set_auto_close(h->fde);
4590 tevent_add_timer(ctdb->ev, h, timeval_current_ofs(120, 0),
4591 ctdb_reloadips_timeout_event, h);
4593 /* we reply later */
4594 *async_reply = true;