4 Copyright (C) Ronnie Sahlberg 2007
5 Copyright (C) Andrew Tridgell 2007
6 Copyright (C) Martin Schwenke 2011
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #include "lib/tdb/include/tdb.h"
23 #include "lib/util/dlinklist.h"
24 #include "system/network.h"
25 #include "system/filesys.h"
26 #include "system/wait.h"
27 #include "../include/ctdb_private.h"
28 #include "../common/rb_tree.h"
31 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
33 #define CTDB_ARP_INTERVAL 1
34 #define CTDB_ARP_REPEAT 3
37 struct ctdb_iface *prev, *next;
43 static const char *ctdb_vnn_iface_string(const struct ctdb_vnn *vnn)
46 return vnn->iface->name;
52 static int ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
56 /* Verify that we dont have an entry for this ip yet */
57 for (i=ctdb->ifaces;i;i=i->next) {
58 if (strcmp(i->name, iface) == 0) {
63 /* create a new structure for this interface */
64 i = talloc_zero(ctdb, struct ctdb_iface);
65 CTDB_NO_MEMORY_FATAL(ctdb, i);
66 i->name = talloc_strdup(i, iface);
67 CTDB_NO_MEMORY(ctdb, i->name);
69 * If link_up defaults to true then IPs can be allocated to a
70 * node during the first recovery. However, then an interface
71 * could have its link marked down during the startup event,
72 * causing the IP to move almost immediately. If link_up
73 * defaults to false then, during normal operation, IPs added
74 * to a new interface can't be assigned until a monitor cycle
75 * has occurred and marked the new interfaces up. This makes
76 * IP allocation unpredictable. The following is a neat
77 * compromise: early in startup link_up defaults to false, so
78 * IPs can't be assigned, and after startup IPs can be
79 * assigned immediately.
81 i->link_up = ctdb->done_startup;
83 DLIST_ADD(ctdb->ifaces, i);
88 static bool vnn_has_interface_with_name(struct ctdb_vnn *vnn,
93 for (n = 0; vnn->ifaces[n] != NULL; n++) {
94 if (strcmp(name, vnn->ifaces[n]) == 0) {
102 /* If any interfaces now have no possible IPs then delete them. This
103 * implementation is naive (i.e. simple) rather than clever
104 * (i.e. complex). Given that this is run on delip and that operation
105 * is rare, this doesn't need to be efficient - it needs to be
106 * foolproof. One alternative is reference counting, where the logic
107 * is distributed and can, therefore, be broken in multiple places.
108 * Another alternative is to build a red-black tree of interfaces that
109 * can have addresses (by walking ctdb->vnn and ctdb->single_ip_vnn
110 * once) and then walking ctdb->ifaces once and deleting those not in
111 * the tree. Let's go to one of those if the naive implementation
112 * causes problems... :-)
114 static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
115 struct ctdb_vnn *vnn,
118 struct ctdb_iface *i;
120 /* For each interface, check if there's an IP using it. */
121 for(i=ctdb->ifaces; i; i=i->next) {
125 /* Only consider interfaces named in the given VNN. */
126 if (!vnn_has_interface_with_name(vnn, i->name)) {
130 /* Is the "single IP" on this interface? */
131 if ((ctdb->single_ip_vnn != NULL) &&
132 (ctdb->single_ip_vnn->ifaces[0] != NULL) &&
133 (strcmp(i->name, ctdb->single_ip_vnn->ifaces[0]) == 0)) {
134 /* Found, next interface please... */
137 /* Search for a vnn with this interface. */
139 for (tv=ctdb->vnn; tv; tv=tv->next) {
140 if (vnn_has_interface_with_name(tv, i->name)) {
147 /* None of the VNNs are using this interface. */
148 DLIST_REMOVE(ctdb->ifaces, i);
149 /* Caller will free mem_ctx when convenient. */
150 talloc_steal(mem_ctx, i);
156 static struct ctdb_iface *ctdb_find_iface(struct ctdb_context *ctdb,
159 struct ctdb_iface *i;
161 /* Verify that we dont have an entry for this ip yet */
162 for (i=ctdb->ifaces;i;i=i->next) {
163 if (strcmp(i->name, iface) == 0) {
171 static struct ctdb_iface *ctdb_vnn_best_iface(struct ctdb_context *ctdb,
172 struct ctdb_vnn *vnn)
175 struct ctdb_iface *cur = NULL;
176 struct ctdb_iface *best = NULL;
178 for (i=0; vnn->ifaces[i]; i++) {
180 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
194 if (cur->references < best->references) {
203 static int32_t ctdb_vnn_assign_iface(struct ctdb_context *ctdb,
204 struct ctdb_vnn *vnn)
206 struct ctdb_iface *best = NULL;
209 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
210 "still assigned to iface '%s'\n",
211 ctdb_addr_to_str(&vnn->public_address),
212 ctdb_vnn_iface_string(vnn)));
216 best = ctdb_vnn_best_iface(ctdb, vnn);
218 DEBUG(DEBUG_ERR, (__location__ " public address '%s' "
219 "cannot assign to iface any iface\n",
220 ctdb_addr_to_str(&vnn->public_address)));
226 vnn->pnn = ctdb->pnn;
228 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
229 "now assigned to iface '%s' refs[%d]\n",
230 ctdb_addr_to_str(&vnn->public_address),
231 ctdb_vnn_iface_string(vnn),
236 static void ctdb_vnn_unassign_iface(struct ctdb_context *ctdb,
237 struct ctdb_vnn *vnn)
239 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
240 "now unassigned (old iface '%s' refs[%d])\n",
241 ctdb_addr_to_str(&vnn->public_address),
242 ctdb_vnn_iface_string(vnn),
243 vnn->iface?vnn->iface->references:0));
245 vnn->iface->references--;
248 if (vnn->pnn == ctdb->pnn) {
253 static bool ctdb_vnn_available(struct ctdb_context *ctdb,
254 struct ctdb_vnn *vnn)
258 if (vnn->iface && vnn->iface->link_up) {
262 for (i=0; vnn->ifaces[i]; i++) {
263 struct ctdb_iface *cur;
265 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
278 struct ctdb_takeover_arp {
279 struct ctdb_context *ctdb;
282 struct ctdb_tcp_array *tcparray;
283 struct ctdb_vnn *vnn;
288 lists of tcp endpoints
290 struct ctdb_tcp_list {
291 struct ctdb_tcp_list *prev, *next;
292 struct ctdb_tcp_connection connection;
296 list of clients to kill on IP release
298 struct ctdb_client_ip {
299 struct ctdb_client_ip *prev, *next;
300 struct ctdb_context *ctdb;
307 send a gratuitous arp
309 static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *te,
310 struct timeval t, void *private_data)
312 struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
313 struct ctdb_takeover_arp);
315 struct ctdb_tcp_array *tcparray;
316 const char *iface = ctdb_vnn_iface_string(arp->vnn);
318 ret = ctdb_sys_send_arp(&arp->addr, iface);
320 DEBUG(DEBUG_CRIT,(__location__ " sending of arp failed on iface '%s' (%s)\n",
321 iface, strerror(errno)));
324 tcparray = arp->tcparray;
326 for (i=0;i<tcparray->num;i++) {
327 struct ctdb_tcp_connection *tcon;
329 tcon = &tcparray->connections[i];
330 DEBUG(DEBUG_INFO,("sending tcp tickle ack for %u->%s:%u\n",
331 (unsigned)ntohs(tcon->dst_addr.ip.sin_port),
332 ctdb_addr_to_str(&tcon->src_addr),
333 (unsigned)ntohs(tcon->src_addr.ip.sin_port)));
334 ret = ctdb_sys_send_tcp(
339 DEBUG(DEBUG_CRIT,(__location__ " Failed to send tcp tickle ack for %s\n",
340 ctdb_addr_to_str(&tcon->src_addr)));
347 if (arp->count == CTDB_ARP_REPEAT) {
352 event_add_timed(arp->ctdb->ev, arp->vnn->takeover_ctx,
353 timeval_current_ofs(CTDB_ARP_INTERVAL, 100000),
354 ctdb_control_send_arp, arp);
357 static int32_t ctdb_announce_vnn_iface(struct ctdb_context *ctdb,
358 struct ctdb_vnn *vnn)
360 struct ctdb_takeover_arp *arp;
361 struct ctdb_tcp_array *tcparray;
363 if (!vnn->takeover_ctx) {
364 vnn->takeover_ctx = talloc_new(vnn);
365 if (!vnn->takeover_ctx) {
370 arp = talloc_zero(vnn->takeover_ctx, struct ctdb_takeover_arp);
376 arp->addr = vnn->public_address;
379 tcparray = vnn->tcp_array;
381 /* add all of the known tcp connections for this IP to the
382 list of tcp connections to send tickle acks for */
383 arp->tcparray = talloc_steal(arp, tcparray);
385 vnn->tcp_array = NULL;
386 vnn->tcp_update_needed = true;
389 event_add_timed(arp->ctdb->ev, vnn->takeover_ctx,
390 timeval_zero(), ctdb_control_send_arp, arp);
395 struct takeover_callback_state {
396 struct ctdb_req_control *c;
397 ctdb_sock_addr *addr;
398 struct ctdb_vnn *vnn;
401 struct ctdb_do_takeip_state {
402 struct ctdb_req_control *c;
403 struct ctdb_vnn *vnn;
407 called when takeip event finishes
409 static void ctdb_do_takeip_callback(struct ctdb_context *ctdb, int status,
412 struct ctdb_do_takeip_state *state =
413 talloc_get_type(private_data, struct ctdb_do_takeip_state);
418 struct ctdb_node *node = ctdb->nodes[ctdb->pnn];
420 if (status == -ETIME) {
423 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
424 ctdb_addr_to_str(&state->vnn->public_address),
425 ctdb_vnn_iface_string(state->vnn)));
426 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
428 node->flags |= NODE_FLAGS_UNHEALTHY;
433 if (ctdb->do_checkpublicip) {
435 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
437 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
444 data.dptr = (uint8_t *)ctdb_addr_to_str(&state->vnn->public_address);
445 data.dsize = strlen((char *)data.dptr) + 1;
446 DEBUG(DEBUG_INFO,(__location__ " sending TAKE_IP for '%s'\n", data.dptr));
448 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_TAKE_IP, data);
451 /* the control succeeded */
452 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
457 static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
459 state->vnn->update_in_flight = false;
464 take over an ip address
466 static int32_t ctdb_do_takeip(struct ctdb_context *ctdb,
467 struct ctdb_req_control *c,
468 struct ctdb_vnn *vnn)
471 struct ctdb_do_takeip_state *state;
473 if (vnn->update_in_flight) {
474 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
475 "update for this IP already in flight\n",
476 ctdb_addr_to_str(&vnn->public_address),
477 vnn->public_netmask_bits));
481 ret = ctdb_vnn_assign_iface(ctdb, vnn);
483 DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
484 "assign a usable interface\n",
485 ctdb_addr_to_str(&vnn->public_address),
486 vnn->public_netmask_bits));
490 state = talloc(vnn, struct ctdb_do_takeip_state);
491 CTDB_NO_MEMORY(ctdb, state);
493 state->c = talloc_steal(ctdb, c);
496 vnn->update_in_flight = true;
497 talloc_set_destructor(state, ctdb_takeip_destructor);
499 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
500 ctdb_addr_to_str(&vnn->public_address),
501 vnn->public_netmask_bits,
502 ctdb_vnn_iface_string(vnn)));
504 ret = ctdb_event_script_callback(ctdb,
506 ctdb_do_takeip_callback,
511 ctdb_vnn_iface_string(vnn),
512 ctdb_addr_to_str(&vnn->public_address),
513 vnn->public_netmask_bits);
516 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
517 ctdb_addr_to_str(&vnn->public_address),
518 ctdb_vnn_iface_string(vnn)));
526 struct ctdb_do_updateip_state {
527 struct ctdb_req_control *c;
528 struct ctdb_iface *old;
529 struct ctdb_vnn *vnn;
533 called when updateip event finishes
535 static void ctdb_do_updateip_callback(struct ctdb_context *ctdb, int status,
538 struct ctdb_do_updateip_state *state =
539 talloc_get_type(private_data, struct ctdb_do_updateip_state);
543 if (status == -ETIME) {
546 DEBUG(DEBUG_ERR,(__location__ " Failed to move IP %s from interface %s to %s\n",
547 ctdb_addr_to_str(&state->vnn->public_address),
549 ctdb_vnn_iface_string(state->vnn)));
552 * All we can do is reset the old interface
553 * and let the next run fix it
555 ctdb_vnn_unassign_iface(ctdb, state->vnn);
556 state->vnn->iface = state->old;
557 state->vnn->iface->references++;
559 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
564 if (ctdb->do_checkpublicip) {
566 ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
568 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
575 /* the control succeeded */
576 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
581 static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
583 state->vnn->update_in_flight = false;
588 update (move) an ip address
590 static int32_t ctdb_do_updateip(struct ctdb_context *ctdb,
591 struct ctdb_req_control *c,
592 struct ctdb_vnn *vnn)
595 struct ctdb_do_updateip_state *state;
596 struct ctdb_iface *old = vnn->iface;
597 const char *new_name;
599 if (vnn->update_in_flight) {
600 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
601 "update for this IP already in flight\n",
602 ctdb_addr_to_str(&vnn->public_address),
603 vnn->public_netmask_bits));
607 ctdb_vnn_unassign_iface(ctdb, vnn);
608 ret = ctdb_vnn_assign_iface(ctdb, vnn);
610 DEBUG(DEBUG_ERR,("update of IP %s/%u failed to "
611 "assin a usable interface (old iface '%s')\n",
612 ctdb_addr_to_str(&vnn->public_address),
613 vnn->public_netmask_bits,
618 new_name = ctdb_vnn_iface_string(vnn);
619 if (old->name != NULL && new_name != NULL && !strcmp(old->name, new_name)) {
620 /* A benign update from one interface onto itself.
621 * no need to run the eventscripts in this case, just return
624 ctdb_request_control_reply(ctdb, c, NULL, 0, NULL);
628 state = talloc(vnn, struct ctdb_do_updateip_state);
629 CTDB_NO_MEMORY(ctdb, state);
631 state->c = talloc_steal(ctdb, c);
635 vnn->update_in_flight = true;
636 talloc_set_destructor(state, ctdb_updateip_destructor);
638 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
639 "interface %s to %s\n",
640 ctdb_addr_to_str(&vnn->public_address),
641 vnn->public_netmask_bits,
645 ret = ctdb_event_script_callback(ctdb,
647 ctdb_do_updateip_callback,
650 CTDB_EVENT_UPDATE_IP,
654 ctdb_addr_to_str(&vnn->public_address),
655 vnn->public_netmask_bits);
657 DEBUG(DEBUG_ERR,(__location__ " Failed update IP %s from interface %s to %s\n",
658 ctdb_addr_to_str(&vnn->public_address),
659 old->name, new_name));
668 Find the vnn of the node that has a public ip address
669 returns -1 if the address is not known as a public address
671 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
673 struct ctdb_vnn *vnn;
675 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
676 if (ctdb_same_ip(&vnn->public_address, addr)) {
685 take over an ip address
687 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
688 struct ctdb_req_control *c,
693 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
694 struct ctdb_vnn *vnn;
695 bool have_ip = false;
696 bool do_updateip = false;
697 bool do_takeip = false;
698 struct ctdb_iface *best_iface = NULL;
700 if (pip->pnn != ctdb->pnn) {
701 DEBUG(DEBUG_ERR,(__location__" takeoverip called for an ip '%s' "
702 "with pnn %d, but we're node %d\n",
703 ctdb_addr_to_str(&pip->addr),
704 pip->pnn, ctdb->pnn));
708 /* update out vnn list */
709 vnn = find_public_ip_vnn(ctdb, &pip->addr);
711 DEBUG(DEBUG_INFO,("takeoverip called for an ip '%s' that is not a public address\n",
712 ctdb_addr_to_str(&pip->addr)));
716 if (ctdb->do_checkpublicip) {
717 have_ip = ctdb_sys_have_ip(&pip->addr);
719 best_iface = ctdb_vnn_best_iface(ctdb, vnn);
720 if (best_iface == NULL) {
721 DEBUG(DEBUG_ERR,("takeoverip of IP %s/%u failed to find"
722 "a usable interface (old %s, have_ip %d)\n",
723 ctdb_addr_to_str(&vnn->public_address),
724 vnn->public_netmask_bits,
725 ctdb_vnn_iface_string(vnn),
730 if (vnn->iface == NULL && vnn->pnn == -1 && have_ip && best_iface != NULL) {
731 DEBUG(DEBUG_ERR,("Taking over newly created ip\n"));
736 if (vnn->iface == NULL && have_ip) {
737 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
738 "but we have no interface assigned, has someone manually configured it? Ignore for now.\n",
739 ctdb_addr_to_str(&vnn->public_address)));
743 if (vnn->pnn != ctdb->pnn && have_ip && vnn->pnn != -1) {
744 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
745 "and we have it on iface[%s], but it was assigned to node %d"
746 "and we are node %d, banning ourself\n",
747 ctdb_addr_to_str(&vnn->public_address),
748 ctdb_vnn_iface_string(vnn), vnn->pnn, ctdb->pnn));
753 if (vnn->pnn == -1 && have_ip) {
754 vnn->pnn = ctdb->pnn;
755 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
756 "and we already have it on iface[%s], update local daemon\n",
757 ctdb_addr_to_str(&vnn->public_address),
758 ctdb_vnn_iface_string(vnn)));
763 if (vnn->iface != best_iface) {
764 if (!vnn->iface->link_up) {
766 } else if (vnn->iface->references > (best_iface->references + 1)) {
767 /* only move when the rebalance gains something */
775 ctdb_vnn_unassign_iface(ctdb, vnn);
782 ret = ctdb_do_takeip(ctdb, c, vnn);
786 } else if (do_updateip) {
787 ret = ctdb_do_updateip(ctdb, c, vnn);
793 * The interface is up and the kernel known the ip
796 DEBUG(DEBUG_INFO,("Redundant takeover of IP %s/%u on interface %s (ip already held)\n",
797 ctdb_addr_to_str(&pip->addr),
798 vnn->public_netmask_bits,
799 ctdb_vnn_iface_string(vnn)));
803 /* tell ctdb_control.c that we will be replying asynchronously */
810 takeover an ip address old v4 style
812 int32_t ctdb_control_takeover_ipv4(struct ctdb_context *ctdb,
813 struct ctdb_req_control *c,
819 data.dsize = sizeof(struct ctdb_public_ip);
820 data.dptr = (uint8_t *)talloc_zero(c, struct ctdb_public_ip);
821 CTDB_NO_MEMORY(ctdb, data.dptr);
823 memcpy(data.dptr, indata.dptr, indata.dsize);
824 return ctdb_control_takeover_ip(ctdb, c, data, async_reply);
828 kill any clients that are registered with a IP that is being released
830 static void release_kill_clients(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
832 struct ctdb_client_ip *ip;
834 DEBUG(DEBUG_INFO,("release_kill_clients for ip %s\n",
835 ctdb_addr_to_str(addr)));
837 for (ip=ctdb->client_ip_list; ip; ip=ip->next) {
838 ctdb_sock_addr tmp_addr;
841 DEBUG(DEBUG_INFO,("checking for client %u with IP %s\n",
843 ctdb_addr_to_str(&ip->addr)));
845 if (ctdb_same_ip(&tmp_addr, addr)) {
846 struct ctdb_client *client = ctdb_reqid_find(ctdb,
849 DEBUG(DEBUG_INFO,("matched client %u with IP %s and pid %u\n",
851 ctdb_addr_to_str(&ip->addr),
854 if (client->pid != 0) {
855 DEBUG(DEBUG_INFO,(__location__ " Killing client pid %u for IP %s on client_id %u\n",
856 (unsigned)client->pid,
857 ctdb_addr_to_str(addr),
859 ctdb_kill(ctdb, client->pid, SIGKILL);
866 called when releaseip event finishes
868 static void release_ip_callback(struct ctdb_context *ctdb, int status,
871 struct takeover_callback_state *state =
872 talloc_get_type(private_data, struct takeover_callback_state);
875 if (status == -ETIME) {
879 /* send a message to all clients of this node telling them
880 that the cluster has been reconfigured and they should
881 release any sockets on this IP */
882 data.dptr = (uint8_t *)talloc_strdup(state, ctdb_addr_to_str(state->addr));
883 CTDB_NO_MEMORY_VOID(ctdb, data.dptr);
884 data.dsize = strlen((char *)data.dptr)+1;
886 DEBUG(DEBUG_INFO,(__location__ " sending RELEASE_IP for '%s'\n", data.dptr));
888 ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
890 /* kill clients that have registered with this IP */
891 release_kill_clients(ctdb, state->addr);
893 ctdb_vnn_unassign_iface(ctdb, state->vnn);
895 /* the control succeeded */
896 ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
900 static int ctdb_releaseip_destructor(struct takeover_callback_state *state)
902 state->vnn->update_in_flight = false;
907 release an ip address
909 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
910 struct ctdb_req_control *c,
915 struct takeover_callback_state *state;
916 struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
917 struct ctdb_vnn *vnn;
920 /* update our vnn list */
921 vnn = find_public_ip_vnn(ctdb, &pip->addr);
923 DEBUG(DEBUG_INFO,("releaseip called for an ip '%s' that is not a public address\n",
924 ctdb_addr_to_str(&pip->addr)));
929 /* stop any previous arps */
930 talloc_free(vnn->takeover_ctx);
931 vnn->takeover_ctx = NULL;
933 /* Some ctdb tool commands (e.g. moveip, rebalanceip) send
934 * lazy multicast to drop an IP from any node that isn't the
935 * intended new node. The following causes makes ctdbd ignore
936 * a release for any address it doesn't host.
938 if (ctdb->do_checkpublicip) {
939 if (!ctdb_sys_have_ip(&pip->addr)) {
940 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
941 ctdb_addr_to_str(&pip->addr),
942 vnn->public_netmask_bits,
943 ctdb_vnn_iface_string(vnn)));
944 ctdb_vnn_unassign_iface(ctdb, vnn);
948 if (vnn->iface == NULL) {
949 DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
950 ctdb_addr_to_str(&pip->addr),
951 vnn->public_netmask_bits));
956 /* There is a potential race between take_ip and us because we
957 * update the VNN via a callback that run when the
958 * eventscripts have been run. Avoid the race by allowing one
959 * update to be in flight at a time.
961 if (vnn->update_in_flight) {
962 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
963 "update for this IP already in flight\n",
964 ctdb_addr_to_str(&vnn->public_address),
965 vnn->public_netmask_bits));
969 if (ctdb->do_checkpublicip) {
970 iface = ctdb_sys_find_ifname(&pip->addr);
972 DEBUG(DEBUG_ERR, ("Could not find which interface the ip address is hosted on. can not release it\n"));
976 iface = strdup(ctdb_vnn_iface_string(vnn));
979 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s node:%d\n",
980 ctdb_addr_to_str(&pip->addr),
981 vnn->public_netmask_bits,
985 state = talloc(ctdb, struct takeover_callback_state);
986 CTDB_NO_MEMORY(ctdb, state);
988 state->c = talloc_steal(state, c);
989 state->addr = talloc(state, ctdb_sock_addr);
990 CTDB_NO_MEMORY(ctdb, state->addr);
991 *state->addr = pip->addr;
994 vnn->update_in_flight = true;
995 talloc_set_destructor(state, ctdb_releaseip_destructor);
997 ret = ctdb_event_script_callback(ctdb,
998 state, release_ip_callback, state,
1000 CTDB_EVENT_RELEASE_IP,
1003 ctdb_addr_to_str(&pip->addr),
1004 vnn->public_netmask_bits);
1007 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
1008 ctdb_addr_to_str(&pip->addr),
1009 ctdb_vnn_iface_string(vnn)));
1014 /* tell the control that we will be reply asynchronously */
1015 *async_reply = true;
1020 release an ip address old v4 style
1022 int32_t ctdb_control_release_ipv4(struct ctdb_context *ctdb,
1023 struct ctdb_req_control *c,
1029 data.dsize = sizeof(struct ctdb_public_ip);
1030 data.dptr = (uint8_t *)talloc_zero(c, struct ctdb_public_ip);
1031 CTDB_NO_MEMORY(ctdb, data.dptr);
1033 memcpy(data.dptr, indata.dptr, indata.dsize);
1034 return ctdb_control_release_ip(ctdb, c, data, async_reply);
1038 static int ctdb_add_public_address(struct ctdb_context *ctdb,
1039 ctdb_sock_addr *addr,
1040 unsigned mask, const char *ifaces,
1043 struct ctdb_vnn *vnn;
1050 tmp = strdup(ifaces);
1051 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1052 if (!ctdb_sys_check_iface_exists(iface)) {
1053 DEBUG(DEBUG_CRIT,("Interface %s does not exist. Can not add public-address : %s\n", iface, ctdb_addr_to_str(addr)));
1060 /* Verify that we dont have an entry for this ip yet */
1061 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1062 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
1063 DEBUG(DEBUG_CRIT,("Same ip '%s' specified multiple times in the public address list \n",
1064 ctdb_addr_to_str(addr)));
1069 /* create a new vnn structure for this ip address */
1070 vnn = talloc_zero(ctdb, struct ctdb_vnn);
1071 CTDB_NO_MEMORY_FATAL(ctdb, vnn);
1072 vnn->ifaces = talloc_array(vnn, const char *, num + 2);
1073 tmp = talloc_strdup(vnn, ifaces);
1074 CTDB_NO_MEMORY_FATAL(ctdb, tmp);
1075 for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1076 vnn->ifaces = talloc_realloc(vnn, vnn->ifaces, const char *, num + 2);
1077 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces);
1078 vnn->ifaces[num] = talloc_strdup(vnn, iface);
1079 CTDB_NO_MEMORY_FATAL(ctdb, vnn->ifaces[num]);
1083 vnn->ifaces[num] = NULL;
1084 vnn->public_address = *addr;
1085 vnn->public_netmask_bits = mask;
1087 if (check_address) {
1088 if (ctdb_sys_have_ip(addr)) {
1089 DEBUG(DEBUG_ERR,("We are already hosting public address '%s'. setting PNN to ourself:%d\n", ctdb_addr_to_str(addr), ctdb->pnn));
1090 vnn->pnn = ctdb->pnn;
1094 for (i=0; vnn->ifaces[i]; i++) {
1095 ret = ctdb_add_local_iface(ctdb, vnn->ifaces[i]);
1097 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1098 "for public_address[%s]\n",
1099 vnn->ifaces[i], ctdb_addr_to_str(addr)));
1105 DLIST_ADD(ctdb->vnn, vnn);
1111 setup the event script directory
1113 int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir)
1115 ctdb->event_script_dir = talloc_strdup(ctdb, script_dir);
1116 CTDB_NO_MEMORY(ctdb, ctdb->event_script_dir);
1120 static void ctdb_check_interfaces_event(struct event_context *ev, struct timed_event *te,
1121 struct timeval t, void *private_data)
1123 struct ctdb_context *ctdb = talloc_get_type(private_data,
1124 struct ctdb_context);
1125 struct ctdb_vnn *vnn;
1127 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1130 for (i=0; vnn->ifaces[i] != NULL; i++) {
1131 if (!ctdb_sys_check_iface_exists(vnn->ifaces[i])) {
1132 DEBUG(DEBUG_CRIT,("Interface %s does not exist but is used by public ip %s\n",
1134 ctdb_addr_to_str(&vnn->public_address)));
1139 event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx,
1140 timeval_current_ofs(30, 0),
1141 ctdb_check_interfaces_event, ctdb);
1145 int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb)
1147 if (ctdb->check_public_ifaces_ctx != NULL) {
1148 talloc_free(ctdb->check_public_ifaces_ctx);
1149 ctdb->check_public_ifaces_ctx = NULL;
1152 ctdb->check_public_ifaces_ctx = talloc_new(ctdb);
1153 if (ctdb->check_public_ifaces_ctx == NULL) {
1154 ctdb_fatal(ctdb, "failed to allocate context for checking interfaces");
1157 event_add_timed(ctdb->ev, ctdb->check_public_ifaces_ctx,
1158 timeval_current_ofs(30, 0),
1159 ctdb_check_interfaces_event, ctdb);
1166 setup the public address lists from a file
1168 int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
1174 lines = file_lines_load(ctdb->public_addresses_file, &nlines, ctdb);
1175 if (lines == NULL) {
1176 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
1179 while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
1183 for (i=0;i<nlines;i++) {
1185 ctdb_sock_addr addr;
1186 const char *addrstr;
1191 while ((*line == ' ') || (*line == '\t')) {
1197 if (strcmp(line, "") == 0) {
1200 tok = strtok(line, " \t");
1202 tok = strtok(NULL, " \t");
1204 if (NULL == ctdb->default_public_interface) {
1205 DEBUG(DEBUG_CRIT,("No default public interface and no interface specified at line %u of public address list\n",
1210 ifaces = ctdb->default_public_interface;
1215 if (!addrstr || !parse_ip_mask(addrstr, ifaces, &addr, &mask)) {
1216 DEBUG(DEBUG_CRIT,("Badly formed line %u in public address list\n", i+1));
1220 if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
1221 DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
1232 int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
1236 struct ctdb_vnn *svnn;
1237 struct ctdb_iface *cur = NULL;
1241 svnn = talloc_zero(ctdb, struct ctdb_vnn);
1242 CTDB_NO_MEMORY(ctdb, svnn);
1244 svnn->ifaces = talloc_array(svnn, const char *, 2);
1245 CTDB_NO_MEMORY(ctdb, svnn->ifaces);
1246 svnn->ifaces[0] = talloc_strdup(svnn->ifaces, iface);
1247 CTDB_NO_MEMORY(ctdb, svnn->ifaces[0]);
1248 svnn->ifaces[1] = NULL;
1250 ok = parse_ip(ip, iface, 0, &svnn->public_address);
1256 ret = ctdb_add_local_iface(ctdb, svnn->ifaces[0]);
1258 DEBUG(DEBUG_CRIT, (__location__ " failed to add iface[%s] "
1259 "for single_ip[%s]\n",
1261 ctdb_addr_to_str(&svnn->public_address)));
1266 /* assume the single public ip interface is initially "good" */
1267 cur = ctdb_find_iface(ctdb, iface);
1269 DEBUG(DEBUG_CRIT,("Can not find public interface %s used by --single-public-ip", iface));
1272 cur->link_up = true;
1274 ret = ctdb_vnn_assign_iface(ctdb, svnn);
1280 ctdb->single_ip_vnn = svnn;
1284 /* Given a physical node, return the number of
1285 public addresses that is currently assigned to this node.
1287 static int node_ip_coverage(struct ctdb_context *ctdb,
1289 struct ctdb_public_ip_list *ips)
1293 for (;ips;ips=ips->next) {
1294 if (ips->pnn == pnn) {
1302 /* Check if this is a public ip known to the node, i.e. can that
1303 node takeover this ip ?
1305 static int can_node_serve_ip(struct ctdb_context *ctdb, int32_t pnn,
1306 struct ctdb_public_ip_list *ip)
1308 struct ctdb_all_public_ips *public_ips;
1311 public_ips = ctdb->nodes[pnn]->available_public_ips;
1313 if (public_ips == NULL) {
1317 for (i=0;i<public_ips->num;i++) {
1318 if (ctdb_same_ip(&ip->addr, &public_ips->ips[i].addr)) {
1319 /* yes, this node can serve this public ip */
1328 /* search the node lists list for a node to takeover this ip.
1329 pick the node that currently are serving the least number of ips
1330 so that the ips get spread out evenly.
1332 static int find_takeover_node(struct ctdb_context *ctdb,
1333 struct ctdb_node_map *nodemap, uint32_t mask,
1334 struct ctdb_public_ip_list *ip,
1335 struct ctdb_public_ip_list *all_ips)
1337 int pnn, min=0, num;
1341 for (i=0;i<nodemap->num;i++) {
1342 if (nodemap->nodes[i].flags & NODE_FLAGS_NOIPTAKEOVER) {
1343 /* This node is not allowed to takeover any addresses
1348 if (nodemap->nodes[i].flags & mask) {
1349 /* This node is not healty and can not be used to serve
1355 /* verify that this node can serve this ip */
1356 if (can_node_serve_ip(ctdb, i, ip)) {
1357 /* no it couldnt so skip to the next node */
1361 num = node_ip_coverage(ctdb, i, all_ips);
1362 /* was this the first node we checked ? */
1374 DEBUG(DEBUG_WARNING,(__location__ " Could not find node to take over public address '%s'\n",
1375 ctdb_addr_to_str(&ip->addr)));
1385 static uint32_t *ip_key(ctdb_sock_addr *ip)
1387 static uint32_t key[IP_KEYLEN];
1389 bzero(key, sizeof(key));
1391 switch (ip->sa.sa_family) {
1393 key[3] = htonl(ip->ip.sin_addr.s_addr);
1396 uint32_t *s6_a32 = (uint32_t *)&(ip->ip6.sin6_addr.s6_addr);
1397 key[0] = htonl(s6_a32[0]);
1398 key[1] = htonl(s6_a32[1]);
1399 key[2] = htonl(s6_a32[2]);
1400 key[3] = htonl(s6_a32[3]);
1404 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1411 static void *add_ip_callback(void *parm, void *data)
1413 struct ctdb_public_ip_list *this_ip = parm;
1414 struct ctdb_public_ip_list *prev_ip = data;
1416 if (prev_ip == NULL) {
1419 if (this_ip->pnn == -1) {
1420 this_ip->pnn = prev_ip->pnn;
1426 static int getips_count_callback(void *param, void *data)
1428 struct ctdb_public_ip_list **ip_list = (struct ctdb_public_ip_list **)param;
1429 struct ctdb_public_ip_list *new_ip = (struct ctdb_public_ip_list *)data;
1431 new_ip->next = *ip_list;
1436 static struct ctdb_public_ip_list *
1437 create_merged_ip_list(struct ctdb_context *ctdb)
1440 struct ctdb_public_ip_list *ip_list;
1441 struct ctdb_all_public_ips *public_ips;
1443 if (ctdb->ip_tree != NULL) {
1444 talloc_free(ctdb->ip_tree);
1445 ctdb->ip_tree = NULL;
1447 ctdb->ip_tree = trbt_create(ctdb, 0);
1449 for (i=0;i<ctdb->num_nodes;i++) {
1450 public_ips = ctdb->nodes[i]->known_public_ips;
1452 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
1456 /* there were no public ips for this node */
1457 if (public_ips == NULL) {
1461 for (j=0;j<public_ips->num;j++) {
1462 struct ctdb_public_ip_list *tmp_ip;
1464 tmp_ip = talloc_zero(ctdb->ip_tree, struct ctdb_public_ip_list);
1465 CTDB_NO_MEMORY_NULL(ctdb, tmp_ip);
1466 /* Do not use information about IP addresses hosted
1467 * on other nodes, it may not be accurate */
1468 if (public_ips->ips[j].pnn == ctdb->nodes[i]->pnn) {
1469 tmp_ip->pnn = public_ips->ips[j].pnn;
1473 tmp_ip->addr = public_ips->ips[j].addr;
1474 tmp_ip->next = NULL;
1476 trbt_insertarray32_callback(ctdb->ip_tree,
1477 IP_KEYLEN, ip_key(&public_ips->ips[j].addr),
1484 trbt_traversearray32(ctdb->ip_tree, IP_KEYLEN, getips_count_callback, &ip_list);
1490 * This is the length of the longtest common prefix between the IPs.
1491 * It is calculated by XOR-ing the 2 IPs together and counting the
1492 * number of leading zeroes. The implementation means that all
1493 * addresses end up being 128 bits long.
1495 * FIXME? Should we consider IPv4 and IPv6 separately given that the
1496 * 12 bytes of 0 prefix padding will hurt the algorithm if there are
1497 * lots of nodes and IP addresses?
1499 static uint32_t ip_distance(ctdb_sock_addr *ip1, ctdb_sock_addr *ip2)
1501 uint32_t ip1_k[IP_KEYLEN];
1506 uint32_t distance = 0;
1508 memcpy(ip1_k, ip_key(ip1), sizeof(ip1_k));
1510 for (i=0; i<IP_KEYLEN; i++) {
1511 x = ip1_k[i] ^ t[i];
1515 /* Count number of leading zeroes.
1516 * FIXME? This could be optimised...
1518 while ((x & (1 << 31)) == 0) {
1528 /* Calculate the IP distance for the given IP relative to IPs on the
1529 given node. The ips argument is generally the all_ips variable
1530 used in the main part of the algorithm.
1532 static uint32_t ip_distance_2_sum(ctdb_sock_addr *ip,
1533 struct ctdb_public_ip_list *ips,
1536 struct ctdb_public_ip_list *t;
1541 for (t=ips; t != NULL; t=t->next) {
1542 if (t->pnn != pnn) {
1546 /* Optimisation: We never calculate the distance
1547 * between an address and itself. This allows us to
1548 * calculate the effect of removing an address from a
1549 * node by simply calculating the distance between
1550 * that address and all of the exitsing addresses.
1551 * Moreover, we assume that we're only ever dealing
1552 * with addresses from all_ips so we can identify an
1553 * address via a pointer rather than doing a more
1554 * expensive address comparison. */
1555 if (&(t->addr) == ip) {
1559 d = ip_distance(ip, &(t->addr));
1560 sum += d * d; /* Cheaper than pulling in math.h :-) */
1566 /* Return the LCP2 imbalance metric for addresses currently assigned
1569 static uint32_t lcp2_imbalance(struct ctdb_public_ip_list * all_ips, int pnn)
1571 struct ctdb_public_ip_list *t;
1573 uint32_t imbalance = 0;
1575 for (t=all_ips; t!=NULL; t=t->next) {
1576 if (t->pnn != pnn) {
1579 /* Pass the rest of the IPs rather than the whole
1582 imbalance += ip_distance_2_sum(&(t->addr), t->next, pnn);
1588 /* Allocate any unassigned IPs just by looping through the IPs and
1589 * finding the best node for each.
1591 static void basic_allocate_unassigned(struct ctdb_context *ctdb,
1592 struct ctdb_node_map *nodemap,
1594 struct ctdb_public_ip_list *all_ips)
1596 struct ctdb_public_ip_list *tmp_ip;
1598 /* loop over all ip's and find a physical node to cover for
1601 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1602 if (tmp_ip->pnn == -1) {
1603 if (find_takeover_node(ctdb, nodemap, mask, tmp_ip, all_ips)) {
1604 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1605 ctdb_addr_to_str(&tmp_ip->addr)));
1611 /* Basic non-deterministic rebalancing algorithm.
1613 static void basic_failback(struct ctdb_context *ctdb,
1614 struct ctdb_node_map *nodemap,
1616 struct ctdb_public_ip_list *all_ips,
1620 int maxnode, maxnum, minnode, minnum, num, retries;
1621 struct ctdb_public_ip_list *tmp_ip;
1629 /* for each ip address, loop over all nodes that can serve
1630 this ip and make sure that the difference between the node
1631 serving the most and the node serving the least ip's are
1634 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1635 if (tmp_ip->pnn == -1) {
1639 /* Get the highest and lowest number of ips's served by any
1640 valid node which can serve this ip.
1644 for (i=0;i<nodemap->num;i++) {
1645 if (nodemap->nodes[i].flags & mask) {
1649 /* Only check nodes that are allowed to takeover an ip */
1650 if (nodemap->nodes[i].flags & NODE_FLAGS_NOIPTAKEOVER) {
1654 /* only check nodes that can actually serve this ip */
1655 if (can_node_serve_ip(ctdb, i, tmp_ip)) {
1656 /* no it couldnt so skip to the next node */
1660 num = node_ip_coverage(ctdb, i, all_ips);
1661 if (maxnode == -1) {
1670 if (minnode == -1) {
1680 if (maxnode == -1) {
1681 DEBUG(DEBUG_WARNING,(__location__ " Could not find maxnode. May not be able to serve ip '%s'\n",
1682 ctdb_addr_to_str(&tmp_ip->addr)));
1687 /* if the spread between the smallest and largest coverage by
1688 a node is >=2 we steal one of the ips from the node with
1689 most coverage to even things out a bit.
1690 try to do this a limited number of times since we dont
1691 want to spend too much time balancing the ip coverage.
1693 if ( (maxnum > minnum+1)
1694 && (retries < (num_ips + 5)) ){
1695 struct ctdb_public_ip_list *tmp;
1697 /* Reassign one of maxnode's VNNs */
1698 for (tmp=all_ips;tmp;tmp=tmp->next) {
1699 if (tmp->pnn == maxnode) {
1700 (void)find_takeover_node(ctdb, nodemap, mask, tmp, all_ips);
1709 struct ctdb_rebalancenodes {
1710 struct ctdb_rebalancenodes *next;
1713 static struct ctdb_rebalancenodes *force_rebalance_list = NULL;
1716 /* set this flag to force the node to be rebalanced even if it just didnt
1717 become healthy again.
1719 void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn)
1721 struct ctdb_rebalancenodes *rebalance;
1723 for (rebalance = force_rebalance_list; rebalance; rebalance = rebalance->next) {
1724 if (rebalance->pnn == pnn) {
1729 rebalance = talloc(ctdb, struct ctdb_rebalancenodes);
1730 rebalance->pnn = pnn;
1731 rebalance->next = force_rebalance_list;
1732 force_rebalance_list = rebalance;
1735 /* Do necessary LCP2 initialisation. Bury it in a function here so
1736 * that we can unit test it.
1738 static void lcp2_init(struct ctdb_context * tmp_ctx,
1739 struct ctdb_node_map * nodemap,
1741 struct ctdb_public_ip_list *all_ips,
1742 uint32_t **lcp2_imbalances,
1743 bool **newly_healthy)
1746 struct ctdb_public_ip_list *tmp_ip;
1748 *newly_healthy = talloc_array(tmp_ctx, bool, nodemap->num);
1749 CTDB_NO_MEMORY_FATAL(tmp_ctx, *newly_healthy);
1750 *lcp2_imbalances = talloc_array(tmp_ctx, uint32_t, nodemap->num);
1751 CTDB_NO_MEMORY_FATAL(tmp_ctx, *lcp2_imbalances);
1753 for (i=0;i<nodemap->num;i++) {
1754 (*lcp2_imbalances)[i] = lcp2_imbalance(all_ips, i);
1755 /* First step: is the node "healthy"? */
1756 (*newly_healthy)[i] = ! (bool)(nodemap->nodes[i].flags & mask);
1759 /* 2nd step: if a ndoe has IPs assigned then it must have been
1760 * healthy before, so we remove it from consideration... */
1761 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1762 if (tmp_ip->pnn != -1) {
1763 (*newly_healthy)[tmp_ip->pnn] = false;
1767 /* 3rd step: if a node is forced to re-balance then
1768 we allow failback onto the node */
1769 while (force_rebalance_list != NULL) {
1770 struct ctdb_rebalancenodes *next = force_rebalance_list->next;
1772 if (force_rebalance_list->pnn <= nodemap->num) {
1773 (*newly_healthy)[force_rebalance_list->pnn] = true;
1776 DEBUG(DEBUG_ERR,("During ipreallocation, forced rebalance of node %d\n", force_rebalance_list->pnn));
1777 talloc_free(force_rebalance_list);
1778 force_rebalance_list = next;
1782 /* Allocate any unassigned addresses using the LCP2 algorithm to find
1783 * the IP/node combination that will cost the least.
1785 static void lcp2_allocate_unassigned(struct ctdb_context *ctdb,
1786 struct ctdb_node_map *nodemap,
1788 struct ctdb_public_ip_list *all_ips,
1789 uint32_t *lcp2_imbalances)
1791 struct ctdb_public_ip_list *tmp_ip;
1795 uint32_t mindsum, dstdsum, dstimbl, minimbl;
1796 struct ctdb_public_ip_list *minip;
1798 bool should_loop = true;
1799 bool have_unassigned = true;
1801 while (have_unassigned && should_loop) {
1802 should_loop = false;
1804 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1805 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES (UNASSIGNED)\n"));
1811 /* loop over each unassigned ip. */
1812 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1813 if (tmp_ip->pnn != -1) {
1817 for (dstnode=0; dstnode < nodemap->num; dstnode++) {
1818 /* Only check nodes that are allowed to takeover an ip */
1819 if (nodemap->nodes[dstnode].flags & NODE_FLAGS_NOIPTAKEOVER) {
1823 /* only check nodes that can actually serve this ip */
1824 if (can_node_serve_ip(ctdb, dstnode, tmp_ip)) {
1825 /* no it couldnt so skip to the next node */
1828 if (nodemap->nodes[dstnode].flags & mask) {
1832 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1833 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1834 DEBUG(DEBUG_DEBUG,(" %s -> %d [+%d]\n",
1835 ctdb_addr_to_str(&(tmp_ip->addr)),
1837 dstimbl - lcp2_imbalances[dstnode]));
1840 if ((minnode == -1) || (dstdsum < mindsum)) {
1850 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1852 /* If we found one then assign it to the given node. */
1853 if (minnode != -1) {
1854 minip->pnn = minnode;
1855 lcp2_imbalances[minnode] = minimbl;
1856 DEBUG(DEBUG_INFO,(" %s -> %d [+%d]\n",
1857 ctdb_addr_to_str(&(minip->addr)),
1862 /* There might be a better way but at least this is clear. */
1863 have_unassigned = false;
1864 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1865 if (tmp_ip->pnn == -1) {
1866 have_unassigned = true;
1871 /* We know if we have an unassigned addresses so we might as
1874 if (have_unassigned) {
1875 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
1876 if (tmp_ip->pnn == -1) {
1877 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
1878 ctdb_addr_to_str(&tmp_ip->addr)));
1884 /* LCP2 algorithm for rebalancing the cluster. Given a candidate node
1885 * to move IPs from, determines the best IP/destination node
1886 * combination to move from the source node.
1888 static bool lcp2_failback_candidate(struct ctdb_context *ctdb,
1889 struct ctdb_node_map *nodemap,
1890 struct ctdb_public_ip_list *all_ips,
1893 uint32_t *lcp2_imbalances,
1894 bool *newly_healthy)
1896 int dstnode, mindstnode;
1897 uint32_t srcimbl, srcdsum, dstimbl, dstdsum;
1898 uint32_t minsrcimbl, mindstimbl;
1899 struct ctdb_public_ip_list *minip;
1900 struct ctdb_public_ip_list *tmp_ip;
1902 /* Find an IP and destination node that best reduces imbalance. */
1908 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1909 DEBUG(DEBUG_DEBUG,(" CONSIDERING MOVES FROM %d [%d]\n", srcnode, candimbl));
1911 for (tmp_ip=all_ips; tmp_ip; tmp_ip=tmp_ip->next) {
1912 /* Only consider addresses on srcnode. */
1913 if (tmp_ip->pnn != srcnode) {
1917 /* What is this IP address costing the source node? */
1918 srcdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, srcnode);
1919 srcimbl = candimbl - srcdsum;
1921 /* Consider this IP address would cost each potential
1922 * destination node. Destination nodes are limited to
1923 * those that are newly healthy, since we don't want
1924 * to do gratuitous failover of IPs just to make minor
1925 * balance improvements.
1927 for (dstnode=0; dstnode < nodemap->num; dstnode++) {
1928 if (! newly_healthy[dstnode]) {
1932 /* Only check nodes that are allowed to takeover an ip */
1933 if (nodemap->nodes[dstnode].flags & NODE_FLAGS_NOIPTAKEOVER) {
1937 /* only check nodes that can actually serve this ip */
1938 if (can_node_serve_ip(ctdb, dstnode, tmp_ip)) {
1939 /* no it couldnt so skip to the next node */
1943 dstdsum = ip_distance_2_sum(&(tmp_ip->addr), all_ips, dstnode);
1944 dstimbl = lcp2_imbalances[dstnode] + dstdsum;
1945 DEBUG(DEBUG_DEBUG,(" %d [%d] -> %s -> %d [+%d]\n",
1946 srcnode, srcimbl - lcp2_imbalances[srcnode],
1947 ctdb_addr_to_str(&(tmp_ip->addr)),
1948 dstnode, dstimbl - lcp2_imbalances[dstnode]));
1950 if ((dstimbl < candimbl) && (dstdsum < srcdsum) && \
1951 ((mindstnode == -1) || \
1952 ((srcimbl + dstimbl) < (minsrcimbl + mindstimbl)))) {
1955 minsrcimbl = srcimbl;
1956 mindstnode = dstnode;
1957 mindstimbl = dstimbl;
1961 DEBUG(DEBUG_DEBUG,(" ----------------------------------------\n"));
1963 if (mindstnode != -1) {
1964 /* We found a move that makes things better... */
1965 DEBUG(DEBUG_INFO,("%d [%d] -> %s -> %d [+%d]\n",
1966 srcnode, minsrcimbl - lcp2_imbalances[srcnode],
1967 ctdb_addr_to_str(&(minip->addr)),
1968 mindstnode, mindstimbl - lcp2_imbalances[mindstnode]));
1971 lcp2_imbalances[srcnode] = srcimbl;
1972 lcp2_imbalances[mindstnode] = mindstimbl;
1973 minip->pnn = mindstnode;
1982 struct lcp2_imbalance_pnn {
1987 static int lcp2_cmp_imbalance_pnn(const void * a, const void * b)
1989 const struct lcp2_imbalance_pnn * lipa = (const struct lcp2_imbalance_pnn *) a;
1990 const struct lcp2_imbalance_pnn * lipb = (const struct lcp2_imbalance_pnn *) b;
1992 if (lipa->imbalance > lipb->imbalance) {
1994 } else if (lipa->imbalance == lipb->imbalance) {
2001 /* LCP2 algorithm for rebalancing the cluster. This finds the source
2002 * node with the highest LCP2 imbalance, and then determines the best
2003 * IP/destination node combination to move from the source node.
2005 static void lcp2_failback(struct ctdb_context *ctdb,
2006 struct ctdb_node_map *nodemap,
2008 struct ctdb_public_ip_list *all_ips,
2009 uint32_t *lcp2_imbalances,
2010 bool *newly_healthy)
2012 int i, num_newly_healthy;
2013 struct lcp2_imbalance_pnn * lips;
2018 /* It is only worth continuing if we have suitable target
2019 * nodes to transfer IPs to. This check is much cheaper than
2022 num_newly_healthy = 0;
2023 for (i = 0; i < nodemap->num; i++) {
2024 if (newly_healthy[i]) {
2025 num_newly_healthy++;
2028 if (num_newly_healthy == 0) {
2032 /* Put the imbalances and nodes into an array, sort them and
2033 * iterate through candidates. Usually the 1st one will be
2034 * used, so this doesn't cost much...
2036 lips = talloc_array(ctdb, struct lcp2_imbalance_pnn, nodemap->num);
2037 for (i = 0; i < nodemap->num; i++) {
2038 lips[i].imbalance = lcp2_imbalances[i];
2041 qsort(lips, nodemap->num, sizeof(struct lcp2_imbalance_pnn),
2042 lcp2_cmp_imbalance_pnn);
2045 for (i = 0; i < nodemap->num; i++) {
2046 /* This means that all nodes had 0 or 1 addresses, so
2047 * can't be imbalanced.
2049 if (lips[i].imbalance == 0) {
2053 if (lcp2_failback_candidate(ctdb,
2071 static void unassign_unsuitable_ips(struct ctdb_context *ctdb,
2072 struct ctdb_node_map *nodemap,
2073 struct ctdb_public_ip_list *all_ips,
2076 struct ctdb_public_ip_list *tmp_ip;
2078 /* mark all public addresses with a masked node as being served by
2081 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2082 if (tmp_ip->pnn == -1) {
2085 if (nodemap->nodes[tmp_ip->pnn].flags & mask) {
2090 /* verify that the assigned nodes can serve that public ip
2091 and set it to -1 if not
2093 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2094 if (tmp_ip->pnn == -1) {
2097 if (can_node_serve_ip(ctdb, tmp_ip->pnn, tmp_ip) != 0) {
2098 /* this node can not serve this ip. */
2104 static void ip_alloc_deterministic_ips(struct ctdb_context *ctdb,
2105 struct ctdb_node_map *nodemap,
2106 struct ctdb_public_ip_list *all_ips,
2109 struct ctdb_public_ip_list *tmp_ip;
2112 DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
2113 /* Allocate IPs to nodes in a modulo fashion so that IPs will
2114 * always be allocated the same way for a specific set of
2115 * available/unavailable nodes.
2118 for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
2119 tmp_ip->pnn = i%nodemap->num;
2122 /* IP failback doesn't make sense with deterministic
2123 * IPs, since the modulo step above implicitly fails
2124 * back IPs to their "home" node.
2126 if (1 == ctdb->tunable.no_ip_failback) {
2127 DEBUG(DEBUG_WARNING, ("WARNING: 'NoIPFailback' set but ignored - incompatible with 'DeterministicIPs\n"));
2130 unassign_unsuitable_ips(ctdb, nodemap, all_ips, mask);
2132 basic_allocate_unassigned(ctdb, nodemap, mask, all_ips);
2134 /* No failback here! */
2137 static void ip_alloc_nondeterministic_ips(struct ctdb_context *ctdb,
2138 struct ctdb_node_map *nodemap,
2139 struct ctdb_public_ip_list *all_ips,
2142 /* This should be pushed down into basic_failback. */
2143 struct ctdb_public_ip_list *tmp_ip;
2145 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2149 unassign_unsuitable_ips(ctdb, nodemap, all_ips, mask);
2151 basic_allocate_unassigned(ctdb, nodemap, mask, all_ips);
2153 /* If we don't want IPs to fail back then don't rebalance IPs. */
2154 if (1 == ctdb->tunable.no_ip_failback) {
2158 /* Now, try to make sure the ip adresses are evenly distributed
2161 basic_failback(ctdb, nodemap, mask, all_ips, num_ips);
2164 static void ip_alloc_lcp2(struct ctdb_context *ctdb,
2165 struct ctdb_node_map *nodemap,
2166 struct ctdb_public_ip_list *all_ips,
2169 uint32_t *lcp2_imbalances;
2170 bool *newly_healthy;
2172 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2174 unassign_unsuitable_ips(ctdb, nodemap, all_ips, mask);
2176 lcp2_init(tmp_ctx, nodemap, mask, all_ips, &lcp2_imbalances, &newly_healthy);
2178 lcp2_allocate_unassigned(ctdb, nodemap, mask, all_ips, lcp2_imbalances);
2180 /* If we don't want IPs to fail back then don't rebalance IPs. */
2181 if (1 == ctdb->tunable.no_ip_failback) {
2185 /* Now, try to make sure the ip adresses are evenly distributed
2188 lcp2_failback(ctdb, nodemap, mask, all_ips, lcp2_imbalances, newly_healthy);
2191 talloc_free(tmp_ctx);
2194 /* The calculation part of the IP allocation algorithm. */
2195 static void ctdb_takeover_run_core(struct ctdb_context *ctdb,
2196 struct ctdb_node_map *nodemap,
2197 struct ctdb_public_ip_list **all_ips_p)
2202 /* Count how many completely healthy nodes we have */
2204 for (i=0;i<nodemap->num;i++) {
2205 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
2210 /* If we have healthy nodes then we will only consider them
2211 for serving public addresses
2213 mask = NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED;
2214 if ((num_healthy == 0) &&
2215 (ctdb->tunable.no_ip_takeover_on_disabled == 0)) {
2216 /* We didnt have any completely healthy nodes so
2217 use "disabled" nodes as a fallback
2219 mask = NODE_FLAGS_INACTIVE;
2222 /* since nodes only know about those public addresses that
2223 can be served by that particular node, no single node has
2224 a full list of all public addresses that exist in the cluster.
2225 Walk over all node structures and create a merged list of
2226 all public addresses that exist in the cluster.
2228 keep the tree of ips around as ctdb->ip_tree
2230 *all_ips_p = create_merged_ip_list(ctdb);
2232 if (1 == ctdb->tunable.lcp2_public_ip_assignment) {
2233 ip_alloc_lcp2(ctdb, nodemap, *all_ips_p, mask);
2234 } else if (1 == ctdb->tunable.deterministic_public_ips) {
2235 ip_alloc_deterministic_ips(ctdb, nodemap, *all_ips_p, mask);
2237 ip_alloc_nondeterministic_ips(ctdb, nodemap, *all_ips_p, mask);
2240 /* at this point ->pnn is the node which will own each IP
2241 or -1 if there is no node that can cover this ip
2247 static void noiptakeover_cb(struct ctdb_context *ctdb, uint32_t pnn, int32_t res, TDB_DATA outdata, void *callback)
2249 struct ctdb_node_map *nodemap = (struct ctdb_node_map *)callback;
2252 DEBUG(DEBUG_ERR,("Failure to read NoIPTakeover tunable from remote node %d\n", pnn));
2256 if (outdata.dsize != sizeof(uint32_t)) {
2257 DEBUG(DEBUG_ERR,("Wrong size of returned data when reading NoIPTakeover tunable from node %d. Expected %d bytes but received %d bytes\n", pnn, (int)sizeof(uint32_t), (int)outdata.dsize));
2261 if (pnn >= nodemap->num) {
2262 DEBUG(DEBUG_ERR,("Got NoIPTakeover reply from node %d but nodemap only has %d entries\n", pnn, nodemap->num));
2266 if (*(uint32_t *)outdata.dptr != 0) {
2267 nodemap->nodes[pnn].flags |= NODE_FLAGS_NOIPTAKEOVER;
2272 make any IP alias changes for public addresses that are necessary
2274 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
2275 client_async_callback fail_callback, void *callback_data)
2278 struct ctdb_public_ip ip;
2279 struct ctdb_public_ipv4 ipv4;
2280 struct ctdb_control_get_tunable *t;
2282 struct ctdb_public_ip_list *all_ips, *tmp_ip;
2284 struct timeval timeout;
2285 struct client_async_data *async_data;
2286 struct ctdb_client_control_state *state;
2287 TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2288 uint32_t disable_timeout;
2291 * ip failover is completely disabled, just send out the
2292 * ipreallocated event.
2294 if (ctdb->tunable.disable_ip_failover != 0) {
2299 /* assume all nodes do support failback */
2300 for (i=0;i<nodemap->num;i++) {
2301 nodemap->nodes[i].flags &= ~NODE_FLAGS_NOIPTAKEOVER;
2303 data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen("NoIPTakeover") + 1;
2304 data.dptr = talloc_size(tmp_ctx, data.dsize);
2305 t = (struct ctdb_control_get_tunable *)data.dptr;
2306 t->length = strlen("NoIPTakeover")+1;
2307 memcpy(t->name, "NoIPTakeover", t->length);
2308 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2309 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_TUNABLE,
2310 nodes, 0, TAKEOVER_TIMEOUT(),
2312 noiptakeover_cb, NULL,
2314 DEBUG(DEBUG_ERR, (__location__ " ctdb_control to get noiptakeover tunable failed\n"));
2317 talloc_free(data.dptr);
2322 /* Do the IP reassignment calculations */
2323 ctdb_takeover_run_core(ctdb, nodemap, &all_ips);
2325 /* The recovery daemon does regular sanity checks of the IPs.
2326 * However, sometimes it is overzealous and thinks changes are
2327 * required when they're already underway. This stops the
2328 * checks for a while before we start moving IPs.
2330 disable_timeout = ctdb->tunable.takeover_timeout;
2331 data.dptr = (uint8_t*)&disable_timeout;
2332 data.dsize = sizeof(disable_timeout);
2333 if (ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
2334 CTDB_SRVID_DISABLE_IP_CHECK, data) != 0) {
2335 DEBUG(DEBUG_INFO,("Failed to disable ip verification\n"));
2338 /* now tell all nodes to delete any alias that they should not
2339 have. This will be a NOOP on nodes that don't currently
2340 hold the given alias */
2341 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2342 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2344 async_data->fail_callback = fail_callback;
2345 async_data->callback_data = callback_data;
2347 for (i=0;i<nodemap->num;i++) {
2348 /* don't talk to unconnected nodes, but do talk to banned nodes */
2349 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
2353 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2354 if (tmp_ip->pnn == nodemap->nodes[i].pnn) {
2355 /* This node should be serving this
2356 vnn so dont tell it to release the ip
2360 if (tmp_ip->addr.sa.sa_family == AF_INET) {
2361 ipv4.pnn = tmp_ip->pnn;
2362 ipv4.sin = tmp_ip->addr.ip;
2364 timeout = TAKEOVER_TIMEOUT();
2365 data.dsize = sizeof(ipv4);
2366 data.dptr = (uint8_t *)&ipv4;
2367 state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
2368 0, CTDB_CONTROL_RELEASE_IPv4, 0,
2372 ip.pnn = tmp_ip->pnn;
2373 ip.addr = tmp_ip->addr;
2375 timeout = TAKEOVER_TIMEOUT();
2376 data.dsize = sizeof(ip);
2377 data.dptr = (uint8_t *)&ip;
2378 state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
2379 0, CTDB_CONTROL_RELEASE_IP, 0,
2384 if (state == NULL) {
2385 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_RELEASE_IP to node %u\n", nodemap->nodes[i].pnn));
2386 talloc_free(tmp_ctx);
2390 ctdb_client_async_add(async_data, state);
2393 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2394 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_RELEASE_IP failed\n"));
2395 talloc_free(tmp_ctx);
2398 talloc_free(async_data);
2401 /* tell all nodes to get their own IPs */
2402 async_data = talloc_zero(tmp_ctx, struct client_async_data);
2403 CTDB_NO_MEMORY_FATAL(ctdb, async_data);
2405 async_data->fail_callback = fail_callback;
2406 async_data->callback_data = callback_data;
2408 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
2409 if (tmp_ip->pnn == -1) {
2410 /* this IP won't be taken over */
2414 if (tmp_ip->addr.sa.sa_family == AF_INET) {
2415 ipv4.pnn = tmp_ip->pnn;
2416 ipv4.sin = tmp_ip->addr.ip;
2418 timeout = TAKEOVER_TIMEOUT();
2419 data.dsize = sizeof(ipv4);
2420 data.dptr = (uint8_t *)&ipv4;
2421 state = ctdb_control_send(ctdb, tmp_ip->pnn,
2422 0, CTDB_CONTROL_TAKEOVER_IPv4, 0,
2426 ip.pnn = tmp_ip->pnn;
2427 ip.addr = tmp_ip->addr;
2429 timeout = TAKEOVER_TIMEOUT();
2430 data.dsize = sizeof(ip);
2431 data.dptr = (uint8_t *)&ip;
2432 state = ctdb_control_send(ctdb, tmp_ip->pnn,
2433 0, CTDB_CONTROL_TAKEOVER_IP, 0,
2437 if (state == NULL) {
2438 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_TAKEOVER_IP to node %u\n", tmp_ip->pnn));
2439 talloc_free(tmp_ctx);
2443 ctdb_client_async_add(async_data, state);
2445 if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2446 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_TAKEOVER_IP failed\n"));
2447 talloc_free(tmp_ctx);
2453 * Tell all nodes to run eventscripts to process the
2454 * "ipreallocated" event. This can do a lot of things,
2455 * including restarting services to reconfigure them if public
2456 * IPs have moved. Once upon a time this event only used to
2459 nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
2460 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_IPREALLOCATED,
2461 nodes, 0, TAKEOVER_TIMEOUT(),
2463 NULL, fail_callback,
2464 callback_data) != 0) {
2465 DEBUG(DEBUG_ERR, (__location__ " failed to send control to run eventscripts with \"ipreallocated\"\n"));
2468 talloc_free(tmp_ctx);
2474 destroy a ctdb_client_ip structure
2476 static int ctdb_client_ip_destructor(struct ctdb_client_ip *ip)
2478 DEBUG(DEBUG_DEBUG,("destroying client tcp for %s:%u (client_id %u)\n",
2479 ctdb_addr_to_str(&ip->addr),
2480 ntohs(ip->addr.ip.sin_port),
2483 DLIST_REMOVE(ip->ctdb->client_ip_list, ip);
2488 called by a client to inform us of a TCP connection that it is managing
2489 that should tickled with an ACK when IP takeover is done
2490 we handle both the old ipv4 style of packets as well as the new ipv4/6
2493 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
2496 struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
2497 struct ctdb_control_tcp *old_addr = NULL;
2498 struct ctdb_control_tcp_addr new_addr;
2499 struct ctdb_control_tcp_addr *tcp_sock = NULL;
2500 struct ctdb_tcp_list *tcp;
2501 struct ctdb_tcp_connection t;
2504 struct ctdb_client_ip *ip;
2505 struct ctdb_vnn *vnn;
2506 ctdb_sock_addr addr;
2508 switch (indata.dsize) {
2509 case sizeof(struct ctdb_control_tcp):
2510 old_addr = (struct ctdb_control_tcp *)indata.dptr;
2511 ZERO_STRUCT(new_addr);
2512 tcp_sock = &new_addr;
2513 tcp_sock->src.ip = old_addr->src;
2514 tcp_sock->dest.ip = old_addr->dest;
2516 case sizeof(struct ctdb_control_tcp_addr):
2517 tcp_sock = (struct ctdb_control_tcp_addr *)indata.dptr;
2520 DEBUG(DEBUG_ERR,(__location__ " Invalid data structure passed "
2521 "to ctdb_control_tcp_client. size was %d but "
2522 "only allowed sizes are %lu and %lu\n",
2524 (long unsigned)sizeof(struct ctdb_control_tcp),
2525 (long unsigned)sizeof(struct ctdb_control_tcp_addr)));
2529 addr = tcp_sock->src;
2530 ctdb_canonicalize_ip(&addr, &tcp_sock->src);
2531 addr = tcp_sock->dest;
2532 ctdb_canonicalize_ip(&addr, &tcp_sock->dest);
2535 memcpy(&addr, &tcp_sock->dest, sizeof(addr));
2536 vnn = find_public_ip_vnn(ctdb, &addr);
2538 switch (addr.sa.sa_family) {
2540 if (ntohl(addr.ip.sin_addr.s_addr) != INADDR_LOOPBACK) {
2541 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public address.\n",
2542 ctdb_addr_to_str(&addr)));
2546 DEBUG(DEBUG_ERR,("Could not add client IP %s. This is not a public ipv6 address.\n",
2547 ctdb_addr_to_str(&addr)));
2550 DEBUG(DEBUG_ERR,(__location__ " Unknown family type %d\n", addr.sa.sa_family));
2556 if (vnn->pnn != ctdb->pnn) {
2557 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
2558 ctdb_addr_to_str(&addr),
2559 client_id, client->pid));
2560 /* failing this call will tell smbd to die */
2564 ip = talloc(client, struct ctdb_client_ip);
2565 CTDB_NO_MEMORY(ctdb, ip);
2569 ip->client_id = client_id;
2570 talloc_set_destructor(ip, ctdb_client_ip_destructor);
2571 DLIST_ADD(ctdb->client_ip_list, ip);
2573 tcp = talloc(client, struct ctdb_tcp_list);
2574 CTDB_NO_MEMORY(ctdb, tcp);
2576 tcp->connection.src_addr = tcp_sock->src;
2577 tcp->connection.dst_addr = tcp_sock->dest;
2579 DLIST_ADD(client->tcp_list, tcp);
2581 t.src_addr = tcp_sock->src;
2582 t.dst_addr = tcp_sock->dest;
2584 data.dptr = (uint8_t *)&t;
2585 data.dsize = sizeof(t);
2587 switch (addr.sa.sa_family) {
2589 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2590 (unsigned)ntohs(tcp_sock->dest.ip.sin_port),
2591 ctdb_addr_to_str(&tcp_sock->src),
2592 (unsigned)ntohs(tcp_sock->src.ip.sin_port), client_id, client->pid));
2595 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
2596 (unsigned)ntohs(tcp_sock->dest.ip6.sin6_port),
2597 ctdb_addr_to_str(&tcp_sock->src),
2598 (unsigned)ntohs(tcp_sock->src.ip6.sin6_port), client_id, client->pid));
2601 DEBUG(DEBUG_ERR,(__location__ " Unknown family %d\n", addr.sa.sa_family));
2605 /* tell all nodes about this tcp connection */
2606 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
2607 CTDB_CONTROL_TCP_ADD,
2608 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
2610 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
2618 find a tcp address on a list
2620 static struct ctdb_tcp_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
2621 struct ctdb_tcp_connection *tcp)
2625 if (array == NULL) {
2629 for (i=0;i<array->num;i++) {
2630 if (ctdb_same_sockaddr(&array->connections[i].src_addr, &tcp->src_addr) &&
2631 ctdb_same_sockaddr(&array->connections[i].dst_addr, &tcp->dst_addr)) {
2632 return &array->connections[i];
2641 called by a daemon to inform us of a TCP connection that one of its
2642 clients managing that should tickled with an ACK when IP takeover is
2645 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed)
2647 struct ctdb_tcp_connection *p = (struct ctdb_tcp_connection *)indata.dptr;
2648 struct ctdb_tcp_array *tcparray;
2649 struct ctdb_tcp_connection tcp;
2650 struct ctdb_vnn *vnn;
2652 vnn = find_public_ip_vnn(ctdb, &p->dst_addr);
2654 DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
2655 ctdb_addr_to_str(&p->dst_addr)));
2661 tcparray = vnn->tcp_array;
2663 /* If this is the first tickle */
2664 if (tcparray == NULL) {
2665 tcparray = talloc_size(ctdb->nodes,
2666 offsetof(struct ctdb_tcp_array, connections) +
2667 sizeof(struct ctdb_tcp_connection) * 1);
2668 CTDB_NO_MEMORY(ctdb, tcparray);
2669 vnn->tcp_array = tcparray;
2672 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_tcp_connection));
2673 CTDB_NO_MEMORY(ctdb, tcparray->connections);
2675 tcparray->connections[tcparray->num].src_addr = p->src_addr;
2676 tcparray->connections[tcparray->num].dst_addr = p->dst_addr;
2679 if (tcp_update_needed) {
2680 vnn->tcp_update_needed = true;
2686 /* Do we already have this tickle ?*/
2687 tcp.src_addr = p->src_addr;
2688 tcp.dst_addr = p->dst_addr;
2689 if (ctdb_tcp_find(vnn->tcp_array, &tcp) != NULL) {
2690 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
2691 ctdb_addr_to_str(&tcp.dst_addr),
2692 ntohs(tcp.dst_addr.ip.sin_port),
2697 /* A new tickle, we must add it to the array */
2698 tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
2699 struct ctdb_tcp_connection,
2701 CTDB_NO_MEMORY(ctdb, tcparray->connections);
2703 vnn->tcp_array = tcparray;
2704 tcparray->connections[tcparray->num].src_addr = p->src_addr;
2705 tcparray->connections[tcparray->num].dst_addr = p->dst_addr;
2708 DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
2709 ctdb_addr_to_str(&tcp.dst_addr),
2710 ntohs(tcp.dst_addr.ip.sin_port),
2713 if (tcp_update_needed) {
2714 vnn->tcp_update_needed = true;
2722 called by a daemon to inform us of a TCP connection that one of its
2723 clients managing that should tickled with an ACK when IP takeover is
2726 static void ctdb_remove_tcp_connection(struct ctdb_context *ctdb, struct ctdb_tcp_connection *conn)
2728 struct ctdb_tcp_connection *tcpp;
2729 struct ctdb_vnn *vnn = find_public_ip_vnn(ctdb, &conn->dst_addr);
2732 DEBUG(DEBUG_ERR,(__location__ " unable to find public address %s\n",
2733 ctdb_addr_to_str(&conn->dst_addr)));
2737 /* if the array is empty we cant remove it
2738 and we dont need to do anything
2740 if (vnn->tcp_array == NULL) {
2741 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist (array is empty) %s:%u\n",
2742 ctdb_addr_to_str(&conn->dst_addr),
2743 ntohs(conn->dst_addr.ip.sin_port)));
2748 /* See if we know this connection
2749 if we dont know this connection then we dont need to do anything
2751 tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
2753 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist %s:%u\n",
2754 ctdb_addr_to_str(&conn->dst_addr),
2755 ntohs(conn->dst_addr.ip.sin_port)));
2760 /* We need to remove this entry from the array.
2761 Instead of allocating a new array and copying data to it
2762 we cheat and just copy the last entry in the existing array
2763 to the entry that is to be removed and just shring the
2766 *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
2767 vnn->tcp_array->num--;
2769 /* If we deleted the last entry we also need to remove the entire array
2771 if (vnn->tcp_array->num == 0) {
2772 talloc_free(vnn->tcp_array);
2773 vnn->tcp_array = NULL;
2776 vnn->tcp_update_needed = true;
2778 DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
2779 ctdb_addr_to_str(&conn->src_addr),
2780 ntohs(conn->src_addr.ip.sin_port)));
2785 called by a daemon to inform us of a TCP connection that one of its
2786 clients used are no longer needed in the tickle database
2788 int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
2790 struct ctdb_tcp_connection *conn = (struct ctdb_tcp_connection *)indata.dptr;
2792 ctdb_remove_tcp_connection(ctdb, conn);
2799 called when a daemon restarts - send all tickes for all public addresses
2800 we are serving immediately to the new node.
2802 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn)
2804 /*XXX here we should send all tickes we are serving to the new node */
2810 called when a client structure goes away - hook to remove
2811 elements from the tcp_list in all daemons
2813 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
2815 while (client->tcp_list) {
2816 struct ctdb_tcp_list *tcp = client->tcp_list;
2817 DLIST_REMOVE(client->tcp_list, tcp);
2818 ctdb_remove_tcp_connection(client->ctdb, &tcp->connection);
2824 release all IPs on shutdown
2826 void ctdb_release_all_ips(struct ctdb_context *ctdb)
2828 struct ctdb_vnn *vnn;
2830 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2831 if (!ctdb_sys_have_ip(&vnn->public_address)) {
2832 ctdb_vnn_unassign_iface(ctdb, vnn);
2838 ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
2839 ctdb_vnn_iface_string(vnn),
2840 ctdb_addr_to_str(&vnn->public_address),
2841 vnn->public_netmask_bits);
2842 release_kill_clients(ctdb, &vnn->public_address);
2843 ctdb_vnn_unassign_iface(ctdb, vnn);
2849 get list of public IPs
2851 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
2852 struct ctdb_req_control *c, TDB_DATA *outdata)
2855 struct ctdb_all_public_ips *ips;
2856 struct ctdb_vnn *vnn;
2857 bool only_available = false;
2859 if (c->flags & CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE) {
2860 only_available = true;
2863 /* count how many public ip structures we have */
2865 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2869 len = offsetof(struct ctdb_all_public_ips, ips) +
2870 num*sizeof(struct ctdb_public_ip);
2871 ips = talloc_zero_size(outdata, len);
2872 CTDB_NO_MEMORY(ctdb, ips);
2875 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2876 if (only_available && !ctdb_vnn_available(ctdb, vnn)) {
2879 ips->ips[i].pnn = vnn->pnn;
2880 ips->ips[i].addr = vnn->public_address;
2884 len = offsetof(struct ctdb_all_public_ips, ips) +
2885 i*sizeof(struct ctdb_public_ip);
2887 outdata->dsize = len;
2888 outdata->dptr = (uint8_t *)ips;
2895 get list of public IPs, old ipv4 style. only returns ipv4 addresses
2897 int32_t ctdb_control_get_public_ipsv4(struct ctdb_context *ctdb,
2898 struct ctdb_req_control *c, TDB_DATA *outdata)
2901 struct ctdb_all_public_ipsv4 *ips;
2902 struct ctdb_vnn *vnn;
2904 /* count how many public ip structures we have */
2906 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2907 if (vnn->public_address.sa.sa_family != AF_INET) {
2913 len = offsetof(struct ctdb_all_public_ipsv4, ips) +
2914 num*sizeof(struct ctdb_public_ipv4);
2915 ips = talloc_zero_size(outdata, len);
2916 CTDB_NO_MEMORY(ctdb, ips);
2918 outdata->dsize = len;
2919 outdata->dptr = (uint8_t *)ips;
2923 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2924 if (vnn->public_address.sa.sa_family != AF_INET) {
2927 ips->ips[i].pnn = vnn->pnn;
2928 ips->ips[i].sin = vnn->public_address.ip;
2935 int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
2936 struct ctdb_req_control *c,
2941 ctdb_sock_addr *addr;
2942 struct ctdb_control_public_ip_info *info;
2943 struct ctdb_vnn *vnn;
2945 addr = (ctdb_sock_addr *)indata.dptr;
2947 vnn = find_public_ip_vnn(ctdb, addr);
2949 /* if it is not a public ip it could be our 'single ip' */
2950 if (ctdb->single_ip_vnn) {
2951 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, addr)) {
2952 vnn = ctdb->single_ip_vnn;
2957 DEBUG(DEBUG_ERR,(__location__ " Could not get public ip info, "
2958 "'%s'not a public address\n",
2959 ctdb_addr_to_str(addr)));
2963 /* count how many public ip structures we have */
2965 for (;vnn->ifaces[num];) {
2969 len = offsetof(struct ctdb_control_public_ip_info, ifaces) +
2970 num*sizeof(struct ctdb_control_iface_info);
2971 info = talloc_zero_size(outdata, len);
2972 CTDB_NO_MEMORY(ctdb, info);
2974 info->ip.addr = vnn->public_address;
2975 info->ip.pnn = vnn->pnn;
2976 info->active_idx = 0xFFFFFFFF;
2978 for (i=0; vnn->ifaces[i]; i++) {
2979 struct ctdb_iface *cur;
2981 cur = ctdb_find_iface(ctdb, vnn->ifaces[i]);
2983 DEBUG(DEBUG_CRIT, (__location__ " internal error iface[%s] unknown\n",
2987 if (vnn->iface == cur) {
2988 info->active_idx = i;
2990 strcpy(info->ifaces[i].name, cur->name);
2991 info->ifaces[i].link_state = cur->link_up;
2992 info->ifaces[i].references = cur->references;
2995 len = offsetof(struct ctdb_control_public_ip_info, ifaces) +
2996 i*sizeof(struct ctdb_control_iface_info);
2998 outdata->dsize = len;
2999 outdata->dptr = (uint8_t *)info;
3004 int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
3005 struct ctdb_req_control *c,
3009 struct ctdb_control_get_ifaces *ifaces;
3010 struct ctdb_iface *cur;
3012 /* count how many public ip structures we have */
3014 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3018 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3019 num*sizeof(struct ctdb_control_iface_info);
3020 ifaces = talloc_zero_size(outdata, len);
3021 CTDB_NO_MEMORY(ctdb, ifaces);
3024 for (cur=ctdb->ifaces;cur;cur=cur->next) {
3025 strcpy(ifaces->ifaces[i].name, cur->name);
3026 ifaces->ifaces[i].link_state = cur->link_up;
3027 ifaces->ifaces[i].references = cur->references;
3031 len = offsetof(struct ctdb_control_get_ifaces, ifaces) +
3032 i*sizeof(struct ctdb_control_iface_info);
3034 outdata->dsize = len;
3035 outdata->dptr = (uint8_t *)ifaces;
3040 int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
3041 struct ctdb_req_control *c,
3044 struct ctdb_control_iface_info *info;
3045 struct ctdb_iface *iface;
3046 bool link_up = false;
3048 info = (struct ctdb_control_iface_info *)indata.dptr;
3050 if (info->name[CTDB_IFACE_SIZE] != '\0') {
3051 int len = strnlen(info->name, CTDB_IFACE_SIZE);
3052 DEBUG(DEBUG_ERR, (__location__ " name[%*.*s] not terminated\n",
3053 len, len, info->name));
3057 switch (info->link_state) {
3065 DEBUG(DEBUG_ERR, (__location__ " link_state[%u] invalid\n",
3066 (unsigned int)info->link_state));
3070 if (info->references != 0) {
3071 DEBUG(DEBUG_ERR, (__location__ " references[%u] should be 0\n",
3072 (unsigned int)info->references));
3076 iface = ctdb_find_iface(ctdb, info->name);
3077 if (iface == NULL) {
3081 if (link_up == iface->link_up) {
3085 DEBUG(iface->link_up?DEBUG_ERR:DEBUG_NOTICE,
3086 ("iface[%s] has changed it's link status %s => %s\n",
3088 iface->link_up?"up":"down",
3089 link_up?"up":"down"));
3091 iface->link_up = link_up;
3097 structure containing the listening socket and the list of tcp connections
3098 that the ctdb daemon is to kill
3100 struct ctdb_kill_tcp {
3101 struct ctdb_vnn *vnn;
3102 struct ctdb_context *ctdb;
3104 struct fd_event *fde;
3105 trbt_tree_t *connections;
3110 a tcp connection that is to be killed
3112 struct ctdb_killtcp_con {
3113 ctdb_sock_addr src_addr;
3114 ctdb_sock_addr dst_addr;
3116 struct ctdb_kill_tcp *killtcp;
3119 /* this function is used to create a key to represent this socketpair
3120 in the killtcp tree.
3121 this key is used to insert and lookup matching socketpairs that are
3122 to be tickled and RST
3124 #define KILLTCP_KEYLEN 10
3125 static uint32_t *killtcp_key(ctdb_sock_addr *src, ctdb_sock_addr *dst)
3127 static uint32_t key[KILLTCP_KEYLEN];
3129 bzero(key, sizeof(key));
3131 if (src->sa.sa_family != dst->sa.sa_family) {
3132 DEBUG(DEBUG_ERR, (__location__ " ERROR, different families passed :%u vs %u\n", src->sa.sa_family, dst->sa.sa_family));
3136 switch (src->sa.sa_family) {
3138 key[0] = dst->ip.sin_addr.s_addr;
3139 key[1] = src->ip.sin_addr.s_addr;
3140 key[2] = dst->ip.sin_port;
3141 key[3] = src->ip.sin_port;
3144 uint32_t *dst6_addr32 =
3145 (uint32_t *)&(dst->ip6.sin6_addr.s6_addr);
3146 uint32_t *src6_addr32 =
3147 (uint32_t *)&(src->ip6.sin6_addr.s6_addr);
3148 key[0] = dst6_addr32[3];
3149 key[1] = src6_addr32[3];
3150 key[2] = dst6_addr32[2];
3151 key[3] = src6_addr32[2];
3152 key[4] = dst6_addr32[1];
3153 key[5] = src6_addr32[1];
3154 key[6] = dst6_addr32[0];
3155 key[7] = src6_addr32[0];
3156 key[8] = dst->ip6.sin6_port;
3157 key[9] = src->ip6.sin6_port;
3161 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", src->sa.sa_family));
3169 called when we get a read event on the raw socket
3171 static void capture_tcp_handler(struct event_context *ev, struct fd_event *fde,
3172 uint16_t flags, void *private_data)
3174 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3175 struct ctdb_killtcp_con *con;
3176 ctdb_sock_addr src, dst;
3177 uint32_t ack_seq, seq;
3179 if (!(flags & EVENT_FD_READ)) {
3183 if (ctdb_sys_read_tcp_packet(killtcp->capture_fd,
3184 killtcp->private_data,
3186 &ack_seq, &seq) != 0) {
3187 /* probably a non-tcp ACK packet */
3191 /* check if we have this guy in our list of connections
3194 con = trbt_lookuparray32(killtcp->connections,
3195 KILLTCP_KEYLEN, killtcp_key(&src, &dst));
3197 /* no this was some other packet we can just ignore */
3201 /* This one has been tickled !
3202 now reset him and remove him from the list.
3204 DEBUG(DEBUG_INFO, ("sending a tcp reset to kill connection :%d -> %s:%d\n",
3205 ntohs(con->dst_addr.ip.sin_port),
3206 ctdb_addr_to_str(&con->src_addr),
3207 ntohs(con->src_addr.ip.sin_port)));
3209 ctdb_sys_send_tcp(&con->dst_addr, &con->src_addr, ack_seq, seq, 1);
3214 /* when traversing the list of all tcp connections to send tickle acks to
3215 (so that we can capture the ack coming back and kill the connection
3217 this callback is called for each connection we are currently trying to kill
3219 static int tickle_connection_traverse(void *param, void *data)
3221 struct ctdb_killtcp_con *con = talloc_get_type(data, struct ctdb_killtcp_con);
3223 /* have tried too many times, just give up */
3224 if (con->count >= 5) {
3225 /* can't delete in traverse: reparent to delete_cons */
3226 talloc_steal(param, con);
3230 /* othervise, try tickling it again */
3233 (ctdb_sock_addr *)&con->dst_addr,
3234 (ctdb_sock_addr *)&con->src_addr,
3241 called every second until all sentenced connections have been reset
3243 static void ctdb_tickle_sentenced_connections(struct event_context *ev, struct timed_event *te,
3244 struct timeval t, void *private_data)
3246 struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
3247 void *delete_cons = talloc_new(NULL);
3249 /* loop over all connections sending tickle ACKs */
3250 trbt_traversearray32(killtcp->connections, KILLTCP_KEYLEN, tickle_connection_traverse, delete_cons);
3252 /* now we've finished traverse, it's safe to do deletion. */
3253 talloc_free(delete_cons);
3255 /* If there are no more connections to kill we can remove the
3256 entire killtcp structure
3258 if ( (killtcp->connections == NULL) ||
3259 (killtcp->connections->root == NULL) ) {
3260 talloc_free(killtcp);
3264 /* try tickling them again in a seconds time
3266 event_add_timed(killtcp->ctdb->ev, killtcp, timeval_current_ofs(1, 0),
3267 ctdb_tickle_sentenced_connections, killtcp);
3271 destroy the killtcp structure
3273 static int ctdb_killtcp_destructor(struct ctdb_kill_tcp *killtcp)
3275 struct ctdb_vnn *tmpvnn;
3277 /* verify that this vnn is still active */
3278 for (tmpvnn = killtcp->ctdb->vnn; tmpvnn; tmpvnn = tmpvnn->next) {
3279 if (tmpvnn == killtcp->vnn) {
3284 if (tmpvnn == NULL) {
3288 if (killtcp->vnn->killtcp != killtcp) {
3292 killtcp->vnn->killtcp = NULL;
3298 /* nothing fancy here, just unconditionally replace any existing
3299 connection structure with the new one.
3301 dont even free the old one if it did exist, that one is talloc_stolen
3302 by the same node in the tree anyway and will be deleted when the new data
3305 static void *add_killtcp_callback(void *parm, void *data)
3311 add a tcp socket to the list of connections we want to RST
3313 static int ctdb_killtcp_add_connection(struct ctdb_context *ctdb,
3317 ctdb_sock_addr src, dst;
3318 struct ctdb_kill_tcp *killtcp;
3319 struct ctdb_killtcp_con *con;
3320 struct ctdb_vnn *vnn;
3322 ctdb_canonicalize_ip(s, &src);
3323 ctdb_canonicalize_ip(d, &dst);
3325 vnn = find_public_ip_vnn(ctdb, &dst);
3327 vnn = find_public_ip_vnn(ctdb, &src);
3330 /* if it is not a public ip it could be our 'single ip' */
3331 if (ctdb->single_ip_vnn) {
3332 if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, &dst)) {
3333 vnn = ctdb->single_ip_vnn;
3338 DEBUG(DEBUG_ERR,(__location__ " Could not killtcp, not a public address\n"));
3342 killtcp = vnn->killtcp;
3344 /* If this is the first connection to kill we must allocate
3347 if (killtcp == NULL) {
3348 killtcp = talloc_zero(vnn, struct ctdb_kill_tcp);
3349 CTDB_NO_MEMORY(ctdb, killtcp);
3352 killtcp->ctdb = ctdb;
3353 killtcp->capture_fd = -1;
3354 killtcp->connections = trbt_create(killtcp, 0);
3356 vnn->killtcp = killtcp;
3357 talloc_set_destructor(killtcp, ctdb_killtcp_destructor);
3362 /* create a structure that describes this connection we want to
3363 RST and store it in killtcp->connections
3365 con = talloc(killtcp, struct ctdb_killtcp_con);
3366 CTDB_NO_MEMORY(ctdb, con);
3367 con->src_addr = src;
3368 con->dst_addr = dst;
3370 con->killtcp = killtcp;
3373 trbt_insertarray32_callback(killtcp->connections,
3374 KILLTCP_KEYLEN, killtcp_key(&con->dst_addr, &con->src_addr),
3375 add_killtcp_callback, con);
3378 If we dont have a socket to listen on yet we must create it
3380 if (killtcp->capture_fd == -1) {
3381 const char *iface = ctdb_vnn_iface_string(vnn);
3382 killtcp->capture_fd = ctdb_sys_open_capture_socket(iface, &killtcp->private_data);
3383 if (killtcp->capture_fd == -1) {
3384 DEBUG(DEBUG_CRIT,(__location__ " Failed to open capturing "
3385 "socket on iface '%s' for killtcp (%s)\n",
3386 iface, strerror(errno)));
3392 if (killtcp->fde == NULL) {
3393 killtcp->fde = event_add_fd(ctdb->ev, killtcp, killtcp->capture_fd,
3395 capture_tcp_handler, killtcp);
3396 tevent_fd_set_auto_close(killtcp->fde);
3398 /* We also need to set up some events to tickle all these connections
3399 until they are all reset
3401 event_add_timed(ctdb->ev, killtcp, timeval_current_ofs(1, 0),
3402 ctdb_tickle_sentenced_connections, killtcp);
3405 /* tickle him once now */
3414 talloc_free(vnn->killtcp);
3415 vnn->killtcp = NULL;
3420 kill a TCP connection.
3422 int32_t ctdb_control_kill_tcp(struct ctdb_context *ctdb, TDB_DATA indata)
3424 struct ctdb_control_killtcp *killtcp = (struct ctdb_control_killtcp *)indata.dptr;
3426 return ctdb_killtcp_add_connection(ctdb, &killtcp->src_addr, &killtcp->dst_addr);
3430 called by a daemon to inform us of the entire list of TCP tickles for
3431 a particular public address.
3432 this control should only be sent by the node that is currently serving
3433 that public address.
3435 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
3437 struct ctdb_control_tcp_tickle_list *list = (struct ctdb_control_tcp_tickle_list *)indata.dptr;
3438 struct ctdb_tcp_array *tcparray;
3439 struct ctdb_vnn *vnn;
3441 /* We must at least have tickles.num or else we cant verify the size
3442 of the received data blob
3444 if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
3445 tickles.connections)) {
3446 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list. Not enough data for the tickle.num field\n"));
3450 /* verify that the size of data matches what we expect */
3451 if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
3452 tickles.connections)
3453 + sizeof(struct ctdb_tcp_connection)
3454 * list->tickles.num) {
3455 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list\n"));
3459 vnn = find_public_ip_vnn(ctdb, &list->addr);
3461 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
3462 ctdb_addr_to_str(&list->addr)));
3467 /* remove any old ticklelist we might have */
3468 talloc_free(vnn->tcp_array);
3469 vnn->tcp_array = NULL;
3471 tcparray = talloc(ctdb->nodes, struct ctdb_tcp_array);
3472 CTDB_NO_MEMORY(ctdb, tcparray);
3474 tcparray->num = list->tickles.num;
3476 tcparray->connections = talloc_array(tcparray, struct ctdb_tcp_connection, tcparray->num);
3477 CTDB_NO_MEMORY(ctdb, tcparray->connections);
3479 memcpy(tcparray->connections, &list->tickles.connections[0],
3480 sizeof(struct ctdb_tcp_connection)*tcparray->num);
3482 /* We now have a new fresh tickle list array for this vnn */
3483 vnn->tcp_array = talloc_steal(vnn, tcparray);
3489 called to return the full list of tickles for the puclic address associated
3490 with the provided vnn
3492 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
3494 ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
3495 struct ctdb_control_tcp_tickle_list *list;
3496 struct ctdb_tcp_array *tcparray;
3498 struct ctdb_vnn *vnn;
3500 vnn = find_public_ip_vnn(ctdb, addr);
3502 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
3503 ctdb_addr_to_str(addr)));
3508 tcparray = vnn->tcp_array;
3510 num = tcparray->num;
3515 outdata->dsize = offsetof(struct ctdb_control_tcp_tickle_list,
3516 tickles.connections)
3517 + sizeof(struct ctdb_tcp_connection) * num;
3519 outdata->dptr = talloc_size(outdata, outdata->dsize);
3520 CTDB_NO_MEMORY(ctdb, outdata->dptr);
3521 list = (struct ctdb_control_tcp_tickle_list *)outdata->dptr;
3524 list->tickles.num = num;
3526 memcpy(&list->tickles.connections[0], tcparray->connections,
3527 sizeof(struct ctdb_tcp_connection) * num);
3535 set the list of all tcp tickles for a public address
3537 static int ctdb_ctrl_set_tcp_tickles(struct ctdb_context *ctdb,
3538 struct timeval timeout, uint32_t destnode,
3539 ctdb_sock_addr *addr,
3540 struct ctdb_tcp_array *tcparray)
3544 struct ctdb_control_tcp_tickle_list *list;
3547 num = tcparray->num;
3552 data.dsize = offsetof(struct ctdb_control_tcp_tickle_list,
3553 tickles.connections) +
3554 sizeof(struct ctdb_tcp_connection) * num;
3555 data.dptr = talloc_size(ctdb, data.dsize);
3556 CTDB_NO_MEMORY(ctdb, data.dptr);
3558 list = (struct ctdb_control_tcp_tickle_list *)data.dptr;
3560 list->tickles.num = num;
3562 memcpy(&list->tickles.connections[0], tcparray->connections, sizeof(struct ctdb_tcp_connection) * num);
3565 ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
3566 CTDB_CONTROL_SET_TCP_TICKLE_LIST,
3567 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
3569 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
3573 talloc_free(data.dptr);
3580 perform tickle updates if required
3582 static void ctdb_update_tcp_tickles(struct event_context *ev,
3583 struct timed_event *te,
3584 struct timeval t, void *private_data)
3586 struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
3588 struct ctdb_vnn *vnn;
3590 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3591 /* we only send out updates for public addresses that
3594 if (ctdb->pnn != vnn->pnn) {
3597 /* We only send out the updates if we need to */
3598 if (!vnn->tcp_update_needed) {
3601 ret = ctdb_ctrl_set_tcp_tickles(ctdb,
3603 CTDB_BROADCAST_CONNECTED,
3604 &vnn->public_address,
3607 DEBUG(DEBUG_ERR,("Failed to send the tickle update for public address %s\n",
3608 ctdb_addr_to_str(&vnn->public_address)));
3612 event_add_timed(ctdb->ev, ctdb->tickle_update_context,
3613 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
3614 ctdb_update_tcp_tickles, ctdb);
3619 start periodic update of tcp tickles
3621 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
3623 ctdb->tickle_update_context = talloc_new(ctdb);
3625 event_add_timed(ctdb->ev, ctdb->tickle_update_context,
3626 timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
3627 ctdb_update_tcp_tickles, ctdb);
3633 struct control_gratious_arp {
3634 struct ctdb_context *ctdb;
3635 ctdb_sock_addr addr;
3641 send a control_gratuitous arp
3643 static void send_gratious_arp(struct event_context *ev, struct timed_event *te,
3644 struct timeval t, void *private_data)
3647 struct control_gratious_arp *arp = talloc_get_type(private_data,
3648 struct control_gratious_arp);
3650 ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
3652 DEBUG(DEBUG_ERR,(__location__ " sending of gratious arp on iface '%s' failed (%s)\n",
3653 arp->iface, strerror(errno)));
3658 if (arp->count == CTDB_ARP_REPEAT) {
3663 event_add_timed(arp->ctdb->ev, arp,
3664 timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
3665 send_gratious_arp, arp);
3672 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
3674 struct ctdb_control_gratious_arp *gratious_arp = (struct ctdb_control_gratious_arp *)indata.dptr;
3675 struct control_gratious_arp *arp;
3677 /* verify the size of indata */
3678 if (indata.dsize < offsetof(struct ctdb_control_gratious_arp, iface)) {
3679 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
3680 (unsigned)indata.dsize,
3681 (unsigned)offsetof(struct ctdb_control_gratious_arp, iface)));
3685 ( offsetof(struct ctdb_control_gratious_arp, iface)
3686 + gratious_arp->len ) ){
3688 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
3689 "but should be %u bytes\n",
3690 (unsigned)indata.dsize,
3691 (unsigned)(offsetof(struct ctdb_control_gratious_arp, iface)+gratious_arp->len)));
3696 arp = talloc(ctdb, struct control_gratious_arp);
3697 CTDB_NO_MEMORY(ctdb, arp);
3700 arp->addr = gratious_arp->addr;
3701 arp->iface = talloc_strdup(arp, gratious_arp->iface);
3702 CTDB_NO_MEMORY(ctdb, arp->iface);
3705 event_add_timed(arp->ctdb->ev, arp,
3706 timeval_zero(), send_gratious_arp, arp);
3711 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
3713 struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
3716 /* verify the size of indata */
3717 if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
3718 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
3722 ( offsetof(struct ctdb_control_ip_iface, iface)
3725 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
3726 "but should be %u bytes\n",
3727 (unsigned)indata.dsize,
3728 (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
3732 ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
3735 DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
3743 called when releaseip event finishes for del_public_address
3745 static void delete_ip_callback(struct ctdb_context *ctdb, int status,
3748 talloc_free(private_data);
3751 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
3753 struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
3754 struct ctdb_vnn *vnn;
3757 /* verify the size of indata */
3758 if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
3759 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
3763 ( offsetof(struct ctdb_control_ip_iface, iface)
3766 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
3767 "but should be %u bytes\n",
3768 (unsigned)indata.dsize,
3769 (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
3773 /* walk over all public addresses until we find a match */
3774 for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
3775 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
3776 TALLOC_CTX *mem_ctx = talloc_new(ctdb);
3778 DLIST_REMOVE(ctdb->vnn, vnn);
3779 talloc_steal(mem_ctx, vnn);
3780 ctdb_remove_orphaned_ifaces(ctdb, vnn, mem_ctx);
3781 if (vnn->pnn != ctdb->pnn) {
3782 if (vnn->iface != NULL) {
3783 ctdb_vnn_unassign_iface(ctdb, vnn);
3785 talloc_free(mem_ctx);
3790 ret = ctdb_event_script_callback(ctdb,
3791 mem_ctx, delete_ip_callback, mem_ctx,
3793 CTDB_EVENT_RELEASE_IP,
3795 ctdb_vnn_iface_string(vnn),
3796 ctdb_addr_to_str(&vnn->public_address),
3797 vnn->public_netmask_bits);
3798 if (vnn->iface != NULL) {
3799 ctdb_vnn_unassign_iface(ctdb, vnn);
3812 struct ipreallocated_callback_state {
3813 struct ctdb_req_control *c;
3816 static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
3817 int status, void *p)
3819 struct ipreallocated_callback_state *state =
3820 talloc_get_type(p, struct ipreallocated_callback_state);
3824 (" \"ipreallocated\" event script failed (status %d)\n",
3826 if (status == -ETIME) {
3827 ctdb_ban_self(ctdb);
3831 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
3835 /* A control to run the ipreallocated event */
3836 int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
3837 struct ctdb_req_control *c,
3841 struct ipreallocated_callback_state *state;
3843 state = talloc(ctdb, struct ipreallocated_callback_state);
3844 CTDB_NO_MEMORY(ctdb, state);
3846 DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
3848 ret = ctdb_event_script_callback(ctdb, state,
3849 ctdb_ipreallocated_callback, state,
3850 false, CTDB_EVENT_IPREALLOCATED,
3854 DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
3859 /* tell the control that we will be reply asynchronously */
3860 state->c = talloc_steal(state, c);
3861 *async_reply = true;
3867 /* This function is called from the recovery daemon to verify that a remote
3868 node has the expected ip allocation.
3869 This is verified against ctdb->ip_tree
3871 int verify_remote_ip_allocation(struct ctdb_context *ctdb, struct ctdb_all_public_ips *ips)
3873 struct ctdb_public_ip_list *tmp_ip;
3876 if (ctdb->ip_tree == NULL) {
3877 /* dont know the expected allocation yet, assume remote node
3886 for (i=0; i<ips->num; i++) {
3887 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ips->ips[i].addr));
3888 if (tmp_ip == NULL) {
3889 DEBUG(DEBUG_ERR,(__location__ " Could not find host for address %s, reassign ips\n", ctdb_addr_to_str(&ips->ips[i].addr)));
3893 if (tmp_ip->pnn == -1 || ips->ips[i].pnn == -1) {
3897 if (tmp_ip->pnn != ips->ips[i].pnn) {
3898 DEBUG(DEBUG_ERR,("Inconsistent ip allocation. Trigger reallocation. Thinks %s is held by node %u while it is held by node %u\n", ctdb_addr_to_str(&ips->ips[i].addr), ips->ips[i].pnn, tmp_ip->pnn));
3906 int update_ip_assignment_tree(struct ctdb_context *ctdb, struct ctdb_public_ip *ip)
3908 struct ctdb_public_ip_list *tmp_ip;
3910 if (ctdb->ip_tree == NULL) {
3911 DEBUG(DEBUG_ERR,("No ctdb->ip_tree yet. Failed to update ip assignment\n"));
3915 tmp_ip = trbt_lookuparray32(ctdb->ip_tree, IP_KEYLEN, ip_key(&ip->addr));
3916 if (tmp_ip == NULL) {
3917 DEBUG(DEBUG_ERR,(__location__ " Could not find record for address %s, update ip\n", ctdb_addr_to_str(&ip->addr)));
3921 DEBUG(DEBUG_NOTICE,("Updated ip assignment tree for ip : %s from node %u to node %u\n", ctdb_addr_to_str(&ip->addr), tmp_ip->pnn, ip->pnn));
3922 tmp_ip->pnn = ip->pnn;
3928 struct ctdb_reloadips_handle {
3929 struct ctdb_context *ctdb;
3930 struct ctdb_req_control *c;
3934 struct fd_event *fde;
3937 static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
3939 if (h == h->ctdb->reload_ips) {
3940 h->ctdb->reload_ips = NULL;
3943 ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
3946 ctdb_kill(h->ctdb, h->child, SIGKILL);
3950 static void ctdb_reloadips_timeout_event(struct event_context *ev,
3951 struct timed_event *te,
3952 struct timeval t, void *private_data)
3954 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
3959 static void ctdb_reloadips_child_handler(struct event_context *ev, struct fd_event *fde,
3960 uint16_t flags, void *private_data)
3962 struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
3967 ret = read(h->fd[0], &res, 1);
3968 if (ret < 1 || res != 0) {
3969 DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
3977 static int ctdb_reloadips_child(struct ctdb_context *ctdb)
3979 TALLOC_CTX *mem_ctx = talloc_new(NULL);
3980 struct ctdb_all_public_ips *ips;
3981 struct ctdb_vnn *vnn;
3984 /* read the ip allocation from the local node */
3985 ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(), CTDB_CURRENT_NODE, mem_ctx, &ips);
3987 DEBUG(DEBUG_ERR, ("Unable to get public ips from local node\n"));
3988 talloc_free(mem_ctx);
3992 /* re-read the public ips file */
3994 if (ctdb_set_public_addresses(ctdb, false) != 0) {
3995 DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
3996 talloc_free(mem_ctx);
4001 /* check the previous list of ips and scan for ips that have been
4004 for (i = 0; i < ips->num; i++) {
4005 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4006 if (ctdb_same_ip(&vnn->public_address, &ips->ips[i].addr)) {
4011 /* we need to delete this ip, no longer available on this node */
4013 struct ctdb_control_ip_iface pub;
4015 DEBUG(DEBUG_NOTICE,("RELOADIPS: IP%s is no longer available on this node. Deleting it.\n", ctdb_addr_to_str(&ips->ips[i].addr)));
4016 pub.addr = ips->ips[i].addr;
4020 ret = ctdb_ctrl_del_public_ip(ctdb, TAKEOVER_TIMEOUT(), CTDB_CURRENT_NODE, &pub);
4022 DEBUG(DEBUG_ERR, ("RELOADIPS: Unable to del public ip:%s from local node\n", ctdb_addr_to_str(&ips->ips[i].addr)));
4029 /* loop over all new ones and check the ones we need to add */
4030 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
4031 for (i = 0; i < ips->num; i++) {
4032 if (ctdb_same_ip(&vnn->public_address, &ips->ips[i].addr)) {
4036 if (i == ips->num) {
4037 struct ctdb_control_ip_iface pub;
4038 const char *ifaces = NULL;
4041 DEBUG(DEBUG_NOTICE,("RELOADIPS: New ip:%s found, adding it.\n", ctdb_addr_to_str(&vnn->public_address)));
4043 pub.addr = vnn->public_address;
4044 pub.mask = vnn->public_netmask_bits;
4047 ifaces = vnn->ifaces[0];
4049 while (vnn->ifaces[iface] != NULL) {
4050 ifaces = talloc_asprintf(vnn, "%s,%s", ifaces, vnn->ifaces[iface]);
4053 pub.len = strlen(ifaces)+1;
4054 memcpy(&pub.iface[0], ifaces, strlen(ifaces)+1);
4056 ret = ctdb_ctrl_add_public_ip(ctdb, TAKEOVER_TIMEOUT(), CTDB_CURRENT_NODE, &pub);
4058 DEBUG(DEBUG_ERR, ("RELOADIPS: Unable to add public ip:%s to local node\n", ctdb_addr_to_str(&vnn->public_address)));
4067 /* This control is sent to force the node to re-read the public addresses file
4068 and drop any addresses we should nnot longer host, and add new addresses
4069 that we are now able to host
4071 int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
4073 struct ctdb_reloadips_handle *h;
4074 pid_t parent = getpid();
4076 if (ctdb->reload_ips != NULL) {
4077 talloc_free(ctdb->reload_ips);
4078 ctdb->reload_ips = NULL;
4081 h = talloc(ctdb, struct ctdb_reloadips_handle);
4082 CTDB_NO_MEMORY(ctdb, h);
4087 if (pipe(h->fd) == -1) {
4088 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
4093 h->child = ctdb_fork(ctdb);
4094 if (h->child == (pid_t)-1) {
4095 DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
4103 if (h->child == 0) {
4104 signed char res = 0;
4107 debug_extra = talloc_asprintf(NULL, "reloadips:");
4109 if (switch_from_server_to_client(ctdb, "reloadips-child") != 0) {
4110 DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
4113 res = ctdb_reloadips_child(ctdb);
4115 DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
4119 write(h->fd[1], &res, 1);
4120 /* make sure we die when our parent dies */
4121 while (ctdb_kill(ctdb, parent, 0) == 0 || errno != ESRCH) {
4127 h->c = talloc_steal(h, c);
4130 set_close_on_exec(h->fd[0]);
4132 talloc_set_destructor(h, ctdb_reloadips_destructor);
4135 h->fde = event_add_fd(ctdb->ev, h, h->fd[0],
4136 EVENT_FD_READ, ctdb_reloadips_child_handler,
4138 tevent_fd_set_auto_close(h->fde);
4140 event_add_timed(ctdb->ev, h,
4141 timeval_current_ofs(120, 0),
4142 ctdb_reloadips_timeout_event, h);
4144 /* we reply later */
4145 *async_reply = true;