server/ctdb_takeover.c

   1 /*
   2    ctdb ip takeover code
   3
   4    Copyright (C) Ronnie Sahlberg  2007
   5    Copyright (C) Andrew Tridgell  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20 #include "includes.h"
  21 #include "lib/events/events.h"
  22 #include "lib/tdb/include/tdb.h"
  23 #include "lib/util/dlinklist.h"
  24 #include "system/network.h"
  25 #include "system/filesys.h"
  26 #include "system/wait.h"
  27 #include "../include/ctdb_private.h"
  28 #include "../common/rb_tree.h"
  29
  30
  31 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
  32
  33 #define CTDB_ARP_INTERVAL 1
  34 #define CTDB_ARP_REPEAT   3
  35
  36 struct ctdb_takeover_arp {
  37         struct ctdb_context *ctdb;
  38         uint32_t count;
  39         ctdb_sock_addr addr;
  40         struct ctdb_tcp_array *tcparray;
  41         struct ctdb_vnn *vnn;
  42 };
  43
  44
  45 /*
  46   lists of tcp endpoints
  47  */
  48 struct ctdb_tcp_list {
  49         struct ctdb_tcp_list *prev, *next;
  50         struct ctdb_tcp_connection connection;
  51 };
  52
  53 /*
  54   list of clients to kill on IP release
  55  */
  56 struct ctdb_client_ip {
  57         struct ctdb_client_ip *prev, *next;
  58         struct ctdb_context *ctdb;
  59         ctdb_sock_addr addr;
  60         uint32_t client_id;
  61 };
  62
  63
  64 /*
  65   send a gratuitous arp
  66  */
  67 static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *te,
  68                                   struct timeval t, void *private_data)
  69 {
  70         struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
  71                                                         struct ctdb_takeover_arp);
  72         int i, ret;
  73         struct ctdb_tcp_array *tcparray;
  74
  75         ret = ctdb_sys_send_arp(&arp->addr, arp->vnn->iface);
  76         if (ret != 0) {
  77                 DEBUG(DEBUG_CRIT,(__location__ " sending of arp failed (%s)\n", strerror(errno)));
  78         }
  79
  80         tcparray = arp->tcparray;
  81         if (tcparray) {
  82                 for (i=0;i<tcparray->num;i++) {
  83                         struct ctdb_tcp_connection *tcon;
  84
  85                         tcon = &tcparray->connections[i];
  86                         DEBUG(DEBUG_INFO,("sending tcp tickle ack for %u->%s:%u\n",
  87                                 (unsigned)ntohs(tcon->dst_addr.ip.sin_port),
  88                                 ctdb_addr_to_str(&tcon->src_addr),
  89                                 (unsigned)ntohs(tcon->src_addr.ip.sin_port)));
  90                         ret = ctdb_sys_send_tcp(
  91                                 &tcon->src_addr,
  92                                 &tcon->dst_addr,
  93                                 0, 0, 0);
  94                         if (ret != 0) {
  95                                 DEBUG(DEBUG_CRIT,(__location__ " Failed to send tcp tickle ack for %s\n",
  96                                         ctdb_addr_to_str(&tcon->src_addr)));
  97                         }
  98                 }
  99         }
 100
 101         arp->count++;
 102
 103         if (arp->count == CTDB_ARP_REPEAT) {
 104                 talloc_free(arp);
 105                 return;
 106         }
 107
 108         event_add_timed(arp->ctdb->ev, arp->vnn->takeover_ctx,
 109                         timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
 110                         ctdb_control_send_arp, arp);
 111 }
 112
 113 struct takeover_callback_state {
 114         struct ctdb_req_control *c;
 115         ctdb_sock_addr *addr;
 116         struct ctdb_vnn *vnn;
 117 };
 118
 119 /*
 120   called when takeip event finishes
 121  */
 122 static void takeover_ip_callback(struct ctdb_context *ctdb, int status,
 123                                  void *private_data)
 124 {
 125         struct takeover_callback_state *state =
 126                 talloc_get_type(private_data, struct takeover_callback_state);
 127         struct ctdb_takeover_arp *arp;
 128         struct ctdb_tcp_array *tcparray;
 129
 130         if (status != 0) {
 131                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 132                         ctdb_addr_to_str(state->addr),
 133                         state->vnn->iface));
 134                 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
 135                 talloc_free(state);
 136                 return;
 137         }
 138
 139         if (!state->vnn->takeover_ctx) {
 140                 state->vnn->takeover_ctx = talloc_new(ctdb);
 141                 if (!state->vnn->takeover_ctx) {
 142                         goto failed;
 143                 }
 144         }
 145
 146         arp = talloc_zero(state->vnn->takeover_ctx, struct ctdb_takeover_arp);
 147         if (!arp) goto failed;
 148
 149         arp->ctdb = ctdb;
 150         arp->addr = *state->addr;
 151         arp->vnn  = state->vnn;
 152
 153         tcparray = state->vnn->tcp_array;
 154         if (tcparray) {
 155                 /* add all of the known tcp connections for this IP to the
 156                    list of tcp connections to send tickle acks for */
 157                 arp->tcparray = talloc_steal(arp, tcparray);
 158
 159                 state->vnn->tcp_array = NULL;
 160                 state->vnn->tcp_update_needed = true;
 161         }
 162
 163         event_add_timed(arp->ctdb->ev, state->vnn->takeover_ctx,
 164                         timeval_zero(), ctdb_control_send_arp, arp);
 165
 166         /* the control succeeded */
 167         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 168         talloc_free(state);
 169         return;
 170
 171 failed:
 172         ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
 173         talloc_free(state);
 174         return;
 175 }
 176
 177 /*
 178   Find the vnn of the node that has a public ip address
 179   returns -1 if the address is not known as a public address
 180  */
 181 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
 182 {
 183         struct ctdb_vnn *vnn;
 184
 185         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
 186                 if (ctdb_same_ip(&vnn->public_address, addr)) {
 187                         return vnn;
 188                 }
 189         }
 190
 191         return NULL;
 192 }
 193
 194
 195 /*
 196   take over an ip address
 197  */
 198 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
 199                                  struct ctdb_req_control *c,
 200                                  TDB_DATA indata,
 201                                  bool *async_reply)
 202 {
 203         int ret;
 204         struct takeover_callback_state *state;
 205         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 206         struct ctdb_vnn *vnn;
 207
 208         /* update out vnn list */
 209         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 210         if (vnn == NULL) {
 211                 DEBUG(DEBUG_ERR,("takeoverip called for an ip '%s' that is not a public address\n",
 212                         ctdb_addr_to_str(&pip->addr)));
 213                 return 0;
 214         }
 215         vnn->pnn = pip->pnn;
 216
 217         /* if our kernel already has this IP, do nothing */
 218         if (ctdb_sys_have_ip(&pip->addr)) {
 219                 return 0;
 220         }
 221
 222         state = talloc(ctdb, struct takeover_callback_state);
 223         CTDB_NO_MEMORY(ctdb, state);
 224
 225         state->c = talloc_steal(ctdb, c);
 226         state->addr = talloc(ctdb, ctdb_sock_addr);
 227         CTDB_NO_MEMORY(ctdb, state->addr);
 228
 229         *state->addr = pip->addr;
 230         state->vnn   = vnn;
 231
 232         DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
 233                 ctdb_addr_to_str(&pip->addr),
 234                 vnn->public_netmask_bits,
 235                 vnn->iface));
 236
 237         ret = ctdb_event_script_callback(ctdb,
 238                                          timeval_current_ofs(ctdb->tunable.script_timeout, 0),
 239                                          state, takeover_ip_callback, state,
 240                                          "takeip %s %s %u",
 241                                          vnn->iface,
 242                                          talloc_strdup(state, ctdb_addr_to_str(&pip->addr)),
 243                                          vnn->public_netmask_bits);
 244
 245         if (ret != 0) {
 246                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 247                         ctdb_addr_to_str(&pip->addr),
 248                         vnn->iface));
 249                 talloc_free(state);
 250                 return -1;
 251         }
 252
 253         /* tell ctdb_control.c that we will be replying asynchronously */
 254         *async_reply = true;
 255
 256         return 0;
 257 }
 258
 259 /*
 260   kill any clients that are registered with a IP that is being released
 261  */
 262 static void release_kill_clients(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
 263 {
 264         struct ctdb_client_ip *ip;
 265
 266         DEBUG(DEBUG_INFO,("release_kill_clients for ip %s\n",
 267                 ctdb_addr_to_str(addr)));
 268
 269         for (ip=ctdb->client_ip_list; ip; ip=ip->next) {
 270                 ctdb_sock_addr tmp_addr;
 271
 272                 tmp_addr = ip->addr;
 273                 DEBUG(DEBUG_INFO,("checking for client %u with IP %s\n",
 274                         ip->client_id,
 275                         ctdb_addr_to_str(&ip->addr)));
 276
 277                 if (ctdb_same_ip(&tmp_addr, addr)) {
 278                         struct ctdb_client *client = ctdb_reqid_find(ctdb,
 279                                                                      ip->client_id,
 280                                                                      struct ctdb_client);
 281                         DEBUG(DEBUG_INFO,("matched client %u with IP %s and pid %u\n",
 282                                 ip->client_id,
 283                                 ctdb_addr_to_str(&ip->addr),
 284                                 client->pid));
 285
 286                         if (client->pid != 0) {
 287                                 DEBUG(DEBUG_INFO,(__location__ " Killing client pid %u for IP %s on client_id %u\n",
 288                                         (unsigned)client->pid,
 289                                         ctdb_addr_to_str(addr),
 290                                         ip->client_id));
 291                                 kill(client->pid, SIGKILL);
 292                         }
 293                 }
 294         }
 295 }
 296
 297 /*
 298   called when releaseip event finishes
 299  */
 300 static void release_ip_callback(struct ctdb_context *ctdb, int status,
 301                                 void *private_data)
 302 {
 303         struct takeover_callback_state *state =
 304                 talloc_get_type(private_data, struct takeover_callback_state);
 305         TDB_DATA data;
 306
 307         /* send a message to all clients of this node telling them
 308            that the cluster has been reconfigured and they should
 309            release any sockets on this IP */
 310         data.dptr = (uint8_t *)talloc_strdup(state, ctdb_addr_to_str(state->addr));
 311         data.dsize = strlen((char *)data.dptr)+1;
 312
 313         DEBUG(DEBUG_INFO,(__location__ " sending RELEASE_IP for '%s'\n", data.dptr));
 314
 315         ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
 316
 317         /* kill clients that have registered with this IP */
 318         release_kill_clients(ctdb, state->addr);
 319
 320         /* the control succeeded */
 321         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 322         talloc_free(state);
 323 }
 324
 325 /*
 326   release an ip address
 327  */
 328 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
 329                                 struct ctdb_req_control *c,
 330                                 TDB_DATA indata,
 331                                 bool *async_reply)
 332 {
 333         int ret;
 334         struct takeover_callback_state *state;
 335         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 336         struct ctdb_vnn *vnn;
 337
 338         /* update our vnn list */
 339         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 340         if (vnn == NULL) {
 341                 DEBUG(DEBUG_ERR,("takeoverip called for an ip '%s' that is not a public address\n",
 342                         ctdb_addr_to_str(&pip->addr)));
 343                 return 0;
 344         }
 345         vnn->pnn = pip->pnn;
 346
 347         /* stop any previous arps */
 348         talloc_free(vnn->takeover_ctx);
 349         vnn->takeover_ctx = NULL;
 350
 351         if (!ctdb_sys_have_ip(&pip->addr)) {
 352                 DEBUG(DEBUG_INFO,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
 353                         ctdb_addr_to_str(&pip->addr),
 354                         vnn->public_netmask_bits,
 355                         vnn->iface));
 356                 return 0;
 357         }
 358
 359         DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s\n",
 360                 ctdb_addr_to_str(&pip->addr),
 361                 vnn->public_netmask_bits,
 362                 vnn->iface));
 363
 364         state = talloc(ctdb, struct takeover_callback_state);
 365         CTDB_NO_MEMORY(ctdb, state);
 366
 367         state->c = talloc_steal(state, c);
 368         state->addr = talloc(state, ctdb_sock_addr);
 369         CTDB_NO_MEMORY(ctdb, state->addr);
 370         *state->addr = pip->addr;
 371         state->vnn   = vnn;
 372
 373         ret = ctdb_event_script_callback(ctdb,
 374                                          timeval_current_ofs(ctdb->tunable.script_timeout, 0),
 375                                          state, release_ip_callback, state,
 376                                          "releaseip %s %s %u",
 377                                          vnn->iface,
 378                                          talloc_strdup(state, ctdb_addr_to_str(&pip->addr)),
 379                                          vnn->public_netmask_bits);
 380         if (ret != 0) {
 381                 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
 382                         ctdb_addr_to_str(&pip->addr),
 383                         vnn->iface));
 384                 talloc_free(state);
 385                 return -1;
 386         }
 387
 388         /* tell the control that we will be reply asynchronously */
 389         *async_reply = true;
 390         return 0;
 391 }
 392
 393
 394
 395 static int ctdb_add_public_address(struct ctdb_context *ctdb, ctdb_sock_addr *addr, unsigned mask, const char *iface)
 396 {
 397         struct ctdb_vnn      *vnn;
 398
 399         /* Verify that we dont have an entry for this ip yet */
 400         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
 401                 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
 402                         DEBUG(DEBUG_CRIT,("Same ip '%s' specified multiple times in the public address list \n",
 403                                 ctdb_addr_to_str(addr)));
 404                         return -1;
 405                 }
 406         }
 407
 408         /* create a new vnn structure for this ip address */
 409         vnn = talloc_zero(ctdb, struct ctdb_vnn);
 410         CTDB_NO_MEMORY_FATAL(ctdb, vnn);
 411         vnn->iface = talloc_strdup(vnn, iface);
 412         vnn->public_address      = *addr;
 413         vnn->public_netmask_bits = mask;
 414         vnn->pnn                 = -1;
 415
 416         DLIST_ADD(ctdb->vnn, vnn);
 417
 418         return 0;
 419 }
 420
 421
 422 /*
 423   setup the event script directory
 424 */
 425 int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir)
 426 {
 427         ctdb->event_script_dir = talloc_strdup(ctdb, script_dir);
 428         CTDB_NO_MEMORY(ctdb, ctdb->event_script_dir);
 429         return 0;
 430 }
 431
 432 /*
 433   setup the public address lists from a file
 434 */
 435 int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist)
 436 {
 437         char **lines;
 438         int nlines;
 439         int i;
 440
 441         lines = file_lines_load(alist, &nlines, ctdb);
 442         if (lines == NULL) {
 443                 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", alist);
 444                 return -1;
 445         }
 446         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 447                 nlines--;
 448         }
 449
 450         for (i=0;i<nlines;i++) {
 451                 unsigned mask;
 452                 ctdb_sock_addr addr;
 453                 const char *iface;
 454                 char *tok;
 455
 456                 tok = strtok(lines[i], " \t");
 457                 if (!tok || !parse_ip_mask(tok, &addr, &mask)) {
 458                         DEBUG(DEBUG_CRIT,("Badly formed line %u in public address list\n", i+1));
 459                         talloc_free(lines);
 460                         return -1;
 461                 }
 462                 tok = strtok(NULL, " \t");
 463                 if (tok == NULL) {
 464                         if (NULL == ctdb->default_public_interface) {
 465                                 DEBUG(DEBUG_CRIT,("No default public interface and no interface specified at line %u of public address list\n",
 466                                          i+1));
 467                                 talloc_free(lines);
 468                                 return -1;
 469                         }
 470                         iface = ctdb->default_public_interface;
 471                 } else {
 472                         iface = tok;
 473                 }
 474
 475                 if (ctdb_add_public_address(ctdb, &addr, mask, iface)) {
 476                         DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
 477                         talloc_free(lines);
 478                         return -1;
 479                 }
 480         }
 481
 482         talloc_free(lines);
 483         return 0;
 484 }
 485
 486
 487
 488
 489 struct ctdb_public_ip_list {
 490         struct ctdb_public_ip_list *next;
 491         uint32_t pnn;
 492         ctdb_sock_addr addr;
 493 };
 494
 495
 496 /* Given a physical node, return the number of
 497    public addresses that is currently assigned to this node.
 498 */
 499 static int node_ip_coverage(struct ctdb_context *ctdb,
 500         int32_t pnn,
 501         struct ctdb_public_ip_list *ips)
 502 {
 503         int num=0;
 504
 505         for (;ips;ips=ips->next) {
 506                 if (ips->pnn == pnn) {
 507                         num++;
 508                 }
 509         }
 510         return num;
 511 }
 512
 513
 514 /* Check if this is a public ip known to the node, i.e. can that
 515    node takeover this ip ?
 516 */
 517 static int can_node_serve_ip(struct ctdb_context *ctdb, int32_t pnn,
 518                 struct ctdb_public_ip_list *ip)
 519 {
 520         struct ctdb_all_public_ips *public_ips;
 521         int i;
 522
 523         public_ips = ctdb->nodes[pnn]->public_ips;
 524
 525         if (public_ips == NULL) {
 526                 return -1;
 527         }
 528
 529         for (i=0;i<public_ips->num;i++) {
 530                 if (ctdb_same_ip(&ip->addr, &public_ips->ips[i].addr)) {
 531                         /* yes, this node can serve this public ip */
 532                         return 0;
 533                 }
 534         }
 535
 536         return -1;
 537 }
 538
 539
 540 /* search the node lists list for a node to takeover this ip.
 541    pick the node that currently are serving the least number of ips
 542    so that the ips get spread out evenly.
 543 */
 544 static int find_takeover_node(struct ctdb_context *ctdb,
 545                 struct ctdb_node_map *nodemap, uint32_t mask,
 546                 struct ctdb_public_ip_list *ip,
 547                 struct ctdb_public_ip_list *all_ips)
 548 {
 549         int pnn, min=0, num;
 550         int i;
 551
 552         pnn    = -1;
 553         for (i=0;i<nodemap->num;i++) {
 554                 if (nodemap->nodes[i].flags & mask) {
 555                         /* This node is not healty and can not be used to serve
 556                            a public address
 557                         */
 558                         continue;
 559                 }
 560
 561                 /* verify that this node can serve this ip */
 562                 if (can_node_serve_ip(ctdb, i, ip)) {
 563                         /* no it couldnt   so skip to the next node */
 564                         continue;
 565                 }
 566
 567                 num = node_ip_coverage(ctdb, i, all_ips);
 568                 /* was this the first node we checked ? */
 569                 if (pnn == -1) {
 570                         pnn = i;
 571                         min  = num;
 572                 } else {
 573                         if (num < min) {
 574                                 pnn = i;
 575                                 min  = num;
 576                         }
 577                 }
 578         }
 579         if (pnn == -1) {
 580                 DEBUG(DEBUG_WARNING,(__location__ " Could not find node to take over public address '%s'\n",
 581                         ctdb_addr_to_str(&ip->addr)));
 582
 583                 return -1;
 584         }
 585
 586         ip->pnn = pnn;
 587         return 0;
 588 }
 589
 590 struct ctdb_public_ip_list *
 591 add_ip_to_merged_list(struct ctdb_context *ctdb,
 592                         TALLOC_CTX *tmp_ctx,
 593                         struct ctdb_public_ip_list *ip_list,
 594                         struct ctdb_public_ip *ip)
 595 {
 596         struct ctdb_public_ip_list *tmp_ip;
 597
 598         /* do we already have this ip in our merged list ?*/
 599         for (tmp_ip=ip_list;tmp_ip;tmp_ip=tmp_ip->next) {
 600
 601                 /* we already have this public ip in the list */
 602                 if (ctdb_same_ip(&tmp_ip->addr, &ip->addr)) {
 603                         return ip_list;
 604                 }
 605         }
 606
 607         /* this is a new public ip, we must add it to the list */
 608         tmp_ip = talloc_zero(tmp_ctx, struct ctdb_public_ip_list);
 609         CTDB_NO_MEMORY_NULL(ctdb, tmp_ip);
 610         tmp_ip->pnn  = ip->pnn;
 611         tmp_ip->addr = ip->addr;
 612         tmp_ip->next = ip_list;
 613
 614         return tmp_ip;
 615 }
 616
 617 struct ctdb_public_ip_list *
 618 create_merged_ip_list(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx)
 619 {
 620         int i, j;
 621         struct ctdb_public_ip_list *ip_list = NULL;
 622         struct ctdb_all_public_ips *public_ips;
 623
 624         for (i=0;i<ctdb->num_nodes;i++) {
 625                 public_ips = ctdb->nodes[i]->public_ips;
 626
 627                 /* there were no public ips for this node */
 628                 if (public_ips == NULL) {
 629                         continue;
 630                 }
 631
 632                 for (j=0;j<public_ips->num;j++) {
 633                         ip_list = add_ip_to_merged_list(ctdb, tmp_ctx,
 634                                         ip_list, &public_ips->ips[j]);
 635                 }
 636         }
 637
 638         return ip_list;
 639 }
 640
 641 /*
 642   make any IP alias changes for public addresses that are necessary
 643  */
 644 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
 645 {
 646         int i, num_healthy, retries;
 647         struct ctdb_public_ip ip;
 648         uint32_t mask;
 649         struct ctdb_public_ip_list *all_ips, *tmp_ip;
 650         int maxnode, maxnum=0, minnode, minnum=0, num;
 651         TDB_DATA data;
 652         struct timeval timeout;
 653         struct client_async_data *async_data;
 654         struct ctdb_client_control_state *state;
 655         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
 656
 657
 658         ZERO_STRUCT(ip);
 659
 660         /* Count how many completely healthy nodes we have */
 661         num_healthy = 0;
 662         for (i=0;i<nodemap->num;i++) {
 663                 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
 664                         num_healthy++;
 665                 }
 666         }
 667
 668         if (num_healthy > 0) {
 669                 /* We have healthy nodes, so only consider them for
 670                    serving public addresses
 671                 */
 672                 mask = NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED;
 673         } else {
 674                 /* We didnt have any completely healthy nodes so
 675                    use "disabled" nodes as a fallback
 676                 */
 677                 mask = NODE_FLAGS_INACTIVE;
 678         }
 679
 680         /* since nodes only know about those public addresses that
 681            can be served by that particular node, no single node has
 682            a full list of all public addresses that exist in the cluster.
 683            Walk over all node structures and create a merged list of
 684            all public addresses that exist in the cluster.
 685         */
 686         all_ips = create_merged_ip_list(ctdb, tmp_ctx);
 687
 688         /* If we want deterministic ip allocations, i.e. that the ip addresses
 689            will always be allocated the same way for a specific set of
 690            available/unavailable nodes.
 691         */
 692         if (1 == ctdb->tunable.deterministic_public_ips) {
 693                 DEBUG(DEBUG_NOTICE,("Deterministic IPs enabled. Resetting all ip allocations\n"));
 694                 for (i=0,tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next,i++) {
 695                         tmp_ip->pnn = i%nodemap->num;
 696                 }
 697         }
 698
 699
 700         /* mark all public addresses with a masked node as being served by
 701            node -1
 702         */
 703         for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 704                 if (tmp_ip->pnn == -1) {
 705                         continue;
 706                 }
 707                 if (nodemap->nodes[tmp_ip->pnn].flags & mask) {
 708                         tmp_ip->pnn = -1;
 709                 }
 710         }
 711
 712         /* verify that the assigned nodes can serve that public ip
 713            and set it to -1 if not
 714         */
 715         for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 716                 if (tmp_ip->pnn == -1) {
 717                         continue;
 718                 }
 719                 if (can_node_serve_ip(ctdb, tmp_ip->pnn, tmp_ip) != 0) {
 720                         /* this node can not serve this ip. */
 721                         tmp_ip->pnn = -1;
 722                 }
 723         }
 724
 725
 726         /* now we must redistribute all public addresses with takeover node
 727            -1 among the nodes available
 728         */
 729         retries = 0;
 730 try_again:
 731         /* loop over all ip's and find a physical node to cover for
 732            each unassigned ip.
 733         */
 734         for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 735                 if (tmp_ip->pnn == -1) {
 736                         if (find_takeover_node(ctdb, nodemap, mask, tmp_ip, all_ips)) {
 737                                 DEBUG(DEBUG_WARNING,("Failed to find node to cover ip %s\n",
 738                                         ctdb_addr_to_str(&tmp_ip->addr)));
 739                         }
 740                 }
 741         }
 742
 743         /* If we dont want ips to fail back after a node becomes healthy
 744            again, we wont even try to reallocat the ip addresses so that
 745            they are evenly spread out.
 746            This can NOT be used at the same time as DeterministicIPs !
 747         */
 748         if (1 == ctdb->tunable.no_ip_failback) {
 749                 if (1 == ctdb->tunable.deterministic_public_ips) {
 750                         DEBUG(DEBUG_ERR, ("ERROR: You can not use 'DeterministicIPs' and 'NoIPFailback' at the same time\n"));
 751                 }
 752                 goto finished;
 753         }
 754
 755
 756         /* now, try to make sure the ip adresses are evenly distributed
 757            across the node.
 758            for each ip address, loop over all nodes that can serve this
 759            ip and make sure that the difference between the node
 760            serving the most and the node serving the least ip's are not greater
 761            than 1.
 762         */
 763         for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 764                 if (tmp_ip->pnn == -1) {
 765                         continue;
 766                 }
 767
 768                 /* Get the highest and lowest number of ips's served by any
 769                    valid node which can serve this ip.
 770                 */
 771                 maxnode = -1;
 772                 minnode = -1;
 773                 for (i=0;i<nodemap->num;i++) {
 774                         if (nodemap->nodes[i].flags & mask) {
 775                                 continue;
 776                         }
 777
 778                         /* only check nodes that can actually serve this ip */
 779                         if (can_node_serve_ip(ctdb, i, tmp_ip)) {
 780                                 /* no it couldnt   so skip to the next node */
 781                                 continue;
 782                         }
 783
 784                         num = node_ip_coverage(ctdb, i, all_ips);
 785                         if (maxnode == -1) {
 786                                 maxnode = i;
 787                                 maxnum  = num;
 788                         } else {
 789                                 if (num > maxnum) {
 790                                         maxnode = i;
 791                                         maxnum  = num;
 792                                 }
 793                         }
 794                         if (minnode == -1) {
 795                                 minnode = i;
 796                                 minnum  = num;
 797                         } else {
 798                                 if (num < minnum) {
 799                                         minnode = i;
 800                                         minnum  = num;
 801                                 }
 802                         }
 803                 }
 804                 if (maxnode == -1) {
 805                         DEBUG(DEBUG_WARNING,(__location__ " Could not find maxnode. May not be able to serve ip '%s'\n",
 806                                 ctdb_addr_to_str(&tmp_ip->addr)));
 807
 808                         continue;
 809                 }
 810
 811                 /* If we want deterministic IPs then dont try to reallocate
 812                    them to spread out the load.
 813                 */
 814                 if (1 == ctdb->tunable.deterministic_public_ips) {
 815                         continue;
 816                 }
 817
 818                 /* if the spread between the smallest and largest coverage by
 819                    a node is >=2 we steal one of the ips from the node with
 820                    most coverage to even things out a bit.
 821                    try to do this at most 5 times  since we dont want to spend
 822                    too much time balancing the ip coverage.
 823                 */
 824                 if ( (maxnum > minnum+1)
 825                   && (retries < 5) ){
 826                         struct ctdb_public_ip_list *tmp;
 827
 828                         /* mark one of maxnode's vnn's as unassigned and try
 829                            again
 830                         */
 831                         for (tmp=all_ips;tmp;tmp=tmp->next) {
 832                                 if (tmp->pnn == maxnode) {
 833                                         tmp->pnn = -1;
 834                                         retries++;
 835                                         goto try_again;
 836                                 }
 837                         }
 838                 }
 839         }
 840
 841
 842         /* finished distributing the public addresses, now just send the
 843            info out to the nodes
 844         */
 845 finished:
 846
 847         /* at this point ->pnn is the node which will own each IP
 848            or -1 if there is no node that can cover this ip
 849         */
 850
 851         /* now tell all nodes to delete any alias that they should not
 852            have.  This will be a NOOP on nodes that don't currently
 853            hold the given alias */
 854         async_data = talloc_zero(tmp_ctx, struct client_async_data);
 855         CTDB_NO_MEMORY_FATAL(ctdb, async_data);
 856
 857         for (i=0;i<nodemap->num;i++) {
 858                 /* don't talk to unconnected nodes, but do talk to banned nodes */
 859                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
 860                         continue;
 861                 }
 862
 863                 for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 864                         if (tmp_ip->pnn == nodemap->nodes[i].pnn) {
 865                                 /* This node should be serving this
 866                                    vnn so dont tell it to release the ip
 867                                 */
 868                                 continue;
 869                         }
 870                         ip.pnn  = tmp_ip->pnn;
 871                         ip.addr = tmp_ip->addr;
 872
 873                         timeout = TAKEOVER_TIMEOUT();
 874                         data.dsize = sizeof(ip);
 875                         data.dptr  = (uint8_t *)&ip;
 876                         state = ctdb_control_send(ctdb, nodemap->nodes[i].pnn,
 877                                         0, CTDB_CONTROL_RELEASE_IP, 0,
 878                                         data, async_data,
 879                                         &timeout, NULL);
 880                         if (state == NULL) {
 881                                 DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_RELEASE_IP to node %u\n", nodemap->nodes[i].pnn));
 882                                 talloc_free(tmp_ctx);
 883                                 return -1;
 884                         }
 885
 886                         ctdb_client_async_add(async_data, state);
 887                 }
 888         }
 889         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
 890                 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_RELEASE_IP failed\n"));
 891                 talloc_free(tmp_ctx);
 892                 return -1;
 893         }
 894         talloc_free(async_data);
 895
 896
 897         /* tell all nodes to get their own IPs */
 898         async_data = talloc_zero(tmp_ctx, struct client_async_data);
 899         CTDB_NO_MEMORY_FATAL(ctdb, async_data);
 900         for (tmp_ip=all_ips;tmp_ip;tmp_ip=tmp_ip->next) {
 901                 if (tmp_ip->pnn == -1) {
 902                         /* this IP won't be taken over */
 903                         continue;
 904                 }
 905                 ip.pnn  = tmp_ip->pnn;
 906                 ip.addr = tmp_ip->addr;
 907
 908                 timeout = TAKEOVER_TIMEOUT();
 909                 data.dsize = sizeof(ip);
 910                 data.dptr  = (uint8_t *)&ip;
 911                 state = ctdb_control_send(ctdb, tmp_ip->pnn,
 912                                 0, CTDB_CONTROL_TAKEOVER_IP, 0,
 913                                 data, async_data,
 914                                 &timeout, NULL);
 915                 if (state == NULL) {
 916                         DEBUG(DEBUG_ERR,(__location__ " Failed to call async control CTDB_CONTROL_TAKEOVER_IP to node %u\n", tmp_ip->pnn));
 917                         talloc_free(tmp_ctx);
 918                         return -1;
 919                 }
 920
 921                 ctdb_client_async_add(async_data, state);
 922         }
 923         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
 924                 DEBUG(DEBUG_ERR,(__location__ " Async control CTDB_CONTROL_TAKEOVER_IP failed\n"));
 925                 talloc_free(tmp_ctx);
 926                 return -1;
 927         }
 928
 929         talloc_free(tmp_ctx);
 930         return 0;
 931 }
 932
 933
 934 /*
 935   destroy a ctdb_client_ip structure
 936  */
 937 static int ctdb_client_ip_destructor(struct ctdb_client_ip *ip)
 938 {
 939         DEBUG(DEBUG_DEBUG,("destroying client tcp for %s:%u (client_id %u)\n",
 940                 ctdb_addr_to_str(&ip->addr),
 941                 ntohs(ip->addr.ip.sin_port),
 942                 ip->client_id));
 943
 944         DLIST_REMOVE(ip->ctdb->client_ip_list, ip);
 945         return 0;
 946 }
 947
 948 /*
 949   called by a client to inform us of a TCP connection that it is managing
 950   that should tickled with an ACK when IP takeover is done
 951  */
 952 //qqq we need a new version of this control that takes ctdb_sock_addr
 953 //and have samba move to that instead.
 954 // This is IPV4 ONLY
 955 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
 956                                 TDB_DATA indata)
 957 {
 958         struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
 959         struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
 960         struct ctdb_tcp_list *tcp;
 961         struct ctdb_control_tcp_vnn t;
 962         int ret;
 963         TDB_DATA data;
 964         struct ctdb_client_ip *ip;
 965         struct ctdb_vnn *vnn;
 966         ctdb_sock_addr addr;
 967
 968         ZERO_STRUCT(addr);
 969         addr.ip = p->dest;
 970         vnn = find_public_ip_vnn(ctdb, &addr);
 971         if (vnn == NULL) {
 972                 if (ntohl(p->dest.sin_addr.s_addr) != INADDR_LOOPBACK) {
 973                         DEBUG(DEBUG_INFO,("Could not add client IP %s. This is not a public address.\n",
 974                                 ctdb_addr_to_str((ctdb_sock_addr *)&p->dest)));
 975                 }
 976                 return 0;
 977         }
 978
 979         if (vnn->pnn != ctdb->pnn) {
 980                 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
 981                         ctdb_addr_to_str((ctdb_sock_addr *)&p->dest),
 982                         client_id, client->pid));
 983                 /* failing this call will tell smbd to die */
 984                 return -1;
 985         }
 986
 987         ip = talloc(client, struct ctdb_client_ip);
 988         CTDB_NO_MEMORY(ctdb, ip);
 989
 990         ip->ctdb      = ctdb;
 991         ip->addr.ip   = p->dest;
 992         ip->client_id = client_id;
 993         talloc_set_destructor(ip, ctdb_client_ip_destructor);
 994         DLIST_ADD(ctdb->client_ip_list, ip);
 995
 996         tcp = talloc(client, struct ctdb_tcp_list);
 997         CTDB_NO_MEMORY(ctdb, tcp);
 998
 999         tcp->connection.src_addr.ip = p->src;
1000         tcp->connection.dst_addr.ip = p->dest;
1001
1002         DLIST_ADD(client->tcp_list, tcp);
1003
1004         t.src.ip  = p->src;
1005         t.dest.ip = p->dest;
1006
1007         data.dptr = (uint8_t *)&t;
1008         data.dsize = sizeof(t);
1009
1010         DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
1011                 (unsigned)ntohs(p->dest.sin_port),
1012                 ctdb_addr_to_str((ctdb_sock_addr *)&p->src),
1013                 (unsigned)ntohs(p->src.sin_port), client_id, client->pid));
1014
1015         /* tell all nodes about this tcp connection */
1016         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
1017                                        CTDB_CONTROL_TCP_ADD,
1018                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
1019         if (ret != 0) {
1020                 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
1021                 return -1;
1022         }
1023
1024         return 0;
1025 }
1026
1027 /*
1028   find a tcp address on a list
1029  */
1030 static struct ctdb_tcp_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
1031                                            struct ctdb_tcp_connection *tcp)
1032 {
1033         int i;
1034
1035         if (array == NULL) {
1036                 return NULL;
1037         }
1038
1039         for (i=0;i<array->num;i++) {
1040                 if (ctdb_same_sockaddr(&array->connections[i].src_addr, &tcp->src_addr) &&
1041                     ctdb_same_sockaddr(&array->connections[i].dst_addr, &tcp->dst_addr)) {
1042                         return &array->connections[i];
1043                 }
1044         }
1045         return NULL;
1046 }
1047
1048 /*
1049   called by a daemon to inform us of a TCP connection that one of its
1050   clients managing that should tickled with an ACK when IP takeover is
1051   done
1052  */
1053 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata)
1054 {
1055         struct ctdb_control_tcp_vnn *p = (struct ctdb_control_tcp_vnn *)indata.dptr;
1056         struct ctdb_tcp_array *tcparray;
1057         struct ctdb_tcp_connection tcp;
1058         struct ctdb_vnn *vnn;
1059
1060         vnn = find_public_ip_vnn(ctdb, &p->dest);
1061         if (vnn == NULL) {
1062                 DEBUG(DEBUG_ERR,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
1063                         ctdb_addr_to_str(&p->dest)));
1064
1065                 return -1;
1066         }
1067
1068
1069         tcparray = vnn->tcp_array;
1070
1071         /* If this is the first tickle */
1072         if (tcparray == NULL) {
1073                 tcparray = talloc_size(ctdb->nodes,
1074                         offsetof(struct ctdb_tcp_array, connections) +
1075                         sizeof(struct ctdb_tcp_connection) * 1);
1076                 CTDB_NO_MEMORY(ctdb, tcparray);
1077                 vnn->tcp_array = tcparray;
1078
1079                 tcparray->num = 0;
1080                 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_tcp_connection));
1081                 CTDB_NO_MEMORY(ctdb, tcparray->connections);
1082
1083                 tcparray->connections[tcparray->num].src_addr = p->src;
1084                 tcparray->connections[tcparray->num].dst_addr = p->dest;
1085                 tcparray->num++;
1086                 return 0;
1087         }
1088
1089
1090         /* Do we already have this tickle ?*/
1091         tcp.src_addr = p->src;
1092         tcp.dst_addr = p->dest;
1093         if (ctdb_tcp_find(vnn->tcp_array, &tcp) != NULL) {
1094                 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
1095                         ctdb_addr_to_str(&tcp.dst_addr),
1096                         ntohs(tcp.dst_addr.ip.sin_port),
1097                         vnn->pnn));
1098                 return 0;
1099         }
1100
1101         /* A new tickle, we must add it to the array */
1102         tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
1103                                         struct ctdb_tcp_connection,
1104                                         tcparray->num+1);
1105         CTDB_NO_MEMORY(ctdb, tcparray->connections);
1106
1107         vnn->tcp_array = tcparray;
1108         tcparray->connections[tcparray->num].src_addr = p->src;
1109         tcparray->connections[tcparray->num].dst_addr = p->dest;
1110         tcparray->num++;
1111
1112         DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
1113                 ctdb_addr_to_str(&tcp.dst_addr),
1114                 ntohs(tcp.dst_addr.ip.sin_port),
1115                 vnn->pnn));
1116
1117         return 0;
1118 }
1119
1120
1121 /*
1122   called by a daemon to inform us of a TCP connection that one of its
1123   clients managing that should tickled with an ACK when IP takeover is
1124   done
1125  */
1126 static void ctdb_remove_tcp_connection(struct ctdb_context *ctdb, struct ctdb_tcp_connection *conn)
1127 {
1128         struct ctdb_tcp_connection *tcpp;
1129         struct ctdb_vnn *vnn = find_public_ip_vnn(ctdb, &conn->dst_addr);
1130
1131         if (vnn == NULL) {
1132                 DEBUG(DEBUG_ERR,(__location__ " unable to find public address %s\n",
1133                         ctdb_addr_to_str(&conn->dst_addr)));
1134                 return;
1135         }
1136
1137         /* if the array is empty we cant remove it
1138            and we dont need to do anything
1139          */
1140         if (vnn->tcp_array == NULL) {
1141                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist (array is empty) %s:%u\n",
1142                         ctdb_addr_to_str(&conn->dst_addr),
1143                         ntohs(conn->dst_addr.ip.sin_port)));
1144                 return;
1145         }
1146
1147
1148         /* See if we know this connection
1149            if we dont know this connection  then we dont need to do anything
1150          */
1151         tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
1152         if (tcpp == NULL) {
1153                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesnt exist %s:%u\n",
1154                         ctdb_addr_to_str(&conn->dst_addr),
1155                         ntohs(conn->dst_addr.ip.sin_port)));
1156                 return;
1157         }
1158
1159
1160         /* We need to remove this entry from the array.
1161            Instead of allocating a new array and copying data to it
1162            we cheat and just copy the last entry in the existing array
1163            to the entry that is to be removed and just shring the
1164            ->num field
1165          */
1166         *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
1167         vnn->tcp_array->num--;
1168
1169         /* If we deleted the last entry we also need to remove the entire array
1170          */
1171         if (vnn->tcp_array->num == 0) {
1172                 talloc_free(vnn->tcp_array);
1173                 vnn->tcp_array = NULL;
1174         }
1175
1176         vnn->tcp_update_needed = true;
1177
1178         DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
1179                 ctdb_addr_to_str(&conn->src_addr),
1180                 ntohs(conn->src_addr.ip.sin_port)));
1181 }
1182
1183
1184 /*
1185   called when a daemon restarts - send all tickes for all public addresses
1186   we are serving immediately to the new node.
1187  */
1188 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn)
1189 {
1190 /*XXX here we should send all tickes we are serving to the new node */
1191         return 0;
1192 }
1193
1194
1195 /*
1196   called when a client structure goes away - hook to remove
1197   elements from the tcp_list in all daemons
1198  */
1199 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
1200 {
1201         while (client->tcp_list) {
1202                 struct ctdb_tcp_list *tcp = client->tcp_list;
1203                 DLIST_REMOVE(client->tcp_list, tcp);
1204                 ctdb_remove_tcp_connection(client->ctdb, &tcp->connection);
1205         }
1206 }
1207
1208
1209 /*
1210   release all IPs on shutdown
1211  */
1212 void ctdb_release_all_ips(struct ctdb_context *ctdb)
1213 {
1214         struct ctdb_vnn *vnn;
1215
1216         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1217                 if (!ctdb_sys_have_ip(&vnn->public_address)) {
1218                         continue;
1219                 }
1220                 ctdb_event_script(ctdb, "releaseip %s %s %u",
1221                                   vnn->iface,
1222                                   talloc_strdup(ctdb, ctdb_addr_to_str(&vnn->public_address)),
1223                                   vnn->public_netmask_bits);
1224                 release_kill_clients(ctdb, &vnn->public_address);
1225         }
1226 }
1227
1228
1229 /*
1230   get list of public IPs
1231  */
1232 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
1233                                     struct ctdb_req_control *c, TDB_DATA *outdata)
1234 {
1235         int i, num, len;
1236         struct ctdb_all_public_ips *ips;
1237         struct ctdb_vnn *vnn;
1238
1239         /* count how many public ip structures we have */
1240         num = 0;
1241         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1242                 num++;
1243         }
1244
1245         len = offsetof(struct ctdb_all_public_ips, ips) +
1246                 num*sizeof(struct ctdb_public_ip);
1247         ips = talloc_zero_size(outdata, len);
1248         CTDB_NO_MEMORY(ctdb, ips);
1249
1250         outdata->dsize = len;
1251         outdata->dptr  = (uint8_t *)ips;
1252
1253         ips->num = num;
1254         i = 0;
1255         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1256                 ips->ips[i].pnn  = vnn->pnn;
1257                 ips->ips[i].addr = vnn->public_address;
1258                 i++;
1259         }
1260
1261         return 0;
1262 }
1263
1264
1265
1266 /*
1267    structure containing the listening socket and the list of tcp connections
1268    that the ctdb daemon is to kill
1269 */
1270 struct ctdb_kill_tcp {
1271         struct ctdb_vnn *vnn;
1272         struct ctdb_context *ctdb;
1273         int capture_fd;
1274         struct fd_event *fde;
1275         trbt_tree_t *connections;
1276         void *private_data;
1277 };
1278
1279 /*
1280   a tcp connection that is to be killed
1281  */
1282 struct ctdb_killtcp_con {
1283         ctdb_sock_addr src_addr;
1284         ctdb_sock_addr dst_addr;
1285         int count;
1286         struct ctdb_kill_tcp *killtcp;
1287 };
1288
1289 /* this function is used to create a key to represent this socketpair
1290    in the killtcp tree.
1291    this key is used to insert and lookup matching socketpairs that are
1292    to be tickled and RST
1293 */
1294 #define KILLTCP_KEYLEN  10
1295 static uint32_t *killtcp_key(ctdb_sock_addr *src, ctdb_sock_addr *dst)
1296 {
1297         static uint32_t key[KILLTCP_KEYLEN];
1298
1299         bzero(key, sizeof(key));
1300
1301         if (src->sa.sa_family != dst->sa.sa_family) {
1302                 DEBUG(DEBUG_ERR, (__location__ " ERROR, different families passed :%u vs %u\n", src->sa.sa_family, dst->sa.sa_family));
1303                 return key;
1304         }
1305
1306         switch (src->sa.sa_family) {
1307         case AF_INET:
1308                 key[0]  = dst->ip.sin_addr.s_addr;
1309                 key[1]  = src->ip.sin_addr.s_addr;
1310                 key[2]  = dst->ip.sin_port;
1311                 key[3]  = src->ip.sin_port;
1312                 break;
1313         case AF_INET6:
1314                 key[0]  = dst->ip6.sin6_addr.s6_addr32[3];
1315                 key[1]  = src->ip6.sin6_addr.s6_addr32[3];
1316                 key[2]  = dst->ip6.sin6_addr.s6_addr32[2];
1317                 key[3]  = src->ip6.sin6_addr.s6_addr32[2];
1318                 key[4]  = dst->ip6.sin6_addr.s6_addr32[1];
1319                 key[5]  = src->ip6.sin6_addr.s6_addr32[1];
1320                 key[6]  = dst->ip6.sin6_addr.s6_addr32[0];
1321                 key[7]  = src->ip6.sin6_addr.s6_addr32[0];
1322                 key[8]  = dst->ip6.sin6_port;
1323                 key[9]  = src->ip6.sin6_port;
1324                 break;
1325         default:
1326                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", src->sa.sa_family));
1327                 return key;
1328         }
1329
1330         return key;
1331 }
1332
1333 /*
1334   called when we get a read event on the raw socket
1335  */
1336 static void capture_tcp_handler(struct event_context *ev, struct fd_event *fde,
1337                                 uint16_t flags, void *private_data)
1338 {
1339         struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
1340         struct ctdb_killtcp_con *con;
1341         ctdb_sock_addr src, dst;
1342         uint32_t ack_seq, seq;
1343
1344         if (!(flags & EVENT_FD_READ)) {
1345                 return;
1346         }
1347
1348         if (ctdb_sys_read_tcp_packet(killtcp->capture_fd,
1349                                 killtcp->private_data,
1350                                 &src, &dst,
1351                                 &ack_seq, &seq) != 0) {
1352                 /* probably a non-tcp ACK packet */
1353                 return;
1354         }
1355
1356         /* check if we have this guy in our list of connections
1357            to kill
1358         */
1359         con = trbt_lookuparray32(killtcp->connections,
1360                         KILLTCP_KEYLEN, killtcp_key(&src, &dst));
1361         if (con == NULL) {
1362                 /* no this was some other packet we can just ignore */
1363                 return;
1364         }
1365
1366         /* This one has been tickled !
1367            now reset him and remove him from the list.
1368          */
1369         DEBUG(DEBUG_INFO, ("sending a tcp reset to kill connection :%d -> %s:%d\n",
1370                 ntohs(con->dst_addr.ip.sin_port),
1371                 ctdb_addr_to_str(&con->src_addr),
1372                 ntohs(con->src_addr.ip.sin_port)));
1373
1374         ctdb_sys_send_tcp(&con->dst_addr, &con->src_addr, ack_seq, seq, 1);
1375         talloc_free(con);
1376 }
1377
1378
1379 /* when traversing the list of all tcp connections to send tickle acks to
1380    (so that we can capture the ack coming back and kill the connection
1381     by a RST)
1382    this callback is called for each connection we are currently trying to kill
1383 */
1384 static void tickle_connection_traverse(void *param, void *data)
1385 {
1386         struct ctdb_killtcp_con *con = talloc_get_type(data, struct ctdb_killtcp_con);
1387
1388         /* have tried too many times, just give up */
1389         if (con->count >= 5) {
1390                 talloc_free(con);
1391                 return;
1392         }
1393
1394         /* othervise, try tickling it again */
1395         con->count++;
1396         ctdb_sys_send_tcp(
1397                 (ctdb_sock_addr *)&con->dst_addr,
1398                 (ctdb_sock_addr *)&con->src_addr,
1399                 0, 0, 0);
1400 }
1401
1402
1403 /*
1404    called every second until all sentenced connections have been reset
1405  */
1406 static void ctdb_tickle_sentenced_connections(struct event_context *ev, struct timed_event *te,
1407                                               struct timeval t, void *private_data)
1408 {
1409         struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
1410
1411
1412         /* loop over all connections sending tickle ACKs */
1413         trbt_traversearray32(killtcp->connections, KILLTCP_KEYLEN, tickle_connection_traverse, NULL);
1414
1415
1416         /* If there are no more connections to kill we can remove the
1417            entire killtcp structure
1418          */
1419         if ( (killtcp->connections == NULL) ||
1420              (killtcp->connections->root == NULL) ) {
1421                 talloc_free(killtcp);
1422                 return;
1423         }
1424
1425         /* try tickling them again in a seconds time
1426          */
1427         event_add_timed(killtcp->ctdb->ev, killtcp, timeval_current_ofs(1, 0),
1428                         ctdb_tickle_sentenced_connections, killtcp);
1429 }
1430
1431 /*
1432   destroy the killtcp structure
1433  */
1434 static int ctdb_killtcp_destructor(struct ctdb_kill_tcp *killtcp)
1435 {
1436         killtcp->vnn->killtcp = NULL;
1437         return 0;
1438 }
1439
1440
1441 /* nothing fancy here, just unconditionally replace any existing
1442    connection structure with the new one.
1443
1444    dont even free the old one if it did exist, that one is talloc_stolen
1445    by the same node in the tree anyway and will be deleted when the new data
1446    is deleted
1447 */
1448 static void *add_killtcp_callback(void *parm, void *data)
1449 {
1450         return parm;
1451 }
1452
1453 /*
1454   add a tcp socket to the list of connections we want to RST
1455  */
1456 static int ctdb_killtcp_add_connection(struct ctdb_context *ctdb,
1457                                        ctdb_sock_addr *s,
1458                                        ctdb_sock_addr *d)
1459 {
1460         ctdb_sock_addr src, dst;
1461         struct ctdb_kill_tcp *killtcp;
1462         struct ctdb_killtcp_con *con;
1463         struct ctdb_vnn *vnn;
1464
1465         ctdb_canonicalize_ip(s, &src);
1466         ctdb_canonicalize_ip(d, &dst);
1467
1468         vnn = find_public_ip_vnn(ctdb, &dst);
1469         if (vnn == NULL) {
1470                 vnn = find_public_ip_vnn(ctdb, &src);
1471         }
1472         if (vnn == NULL) {
1473                 /* if it is not a public ip   it could be our 'single ip' */
1474                 if (ctdb->single_ip_vnn) {
1475                         if (ctdb_same_ip(&ctdb->single_ip_vnn->public_address, &dst)) {
1476                                 vnn = ctdb->single_ip_vnn;
1477                         }
1478                 }
1479         }
1480         if (vnn == NULL) {
1481                 DEBUG(DEBUG_ERR,(__location__ " Could not killtcp, not a public address\n"));
1482                 return -1;
1483         }
1484
1485         killtcp = vnn->killtcp;
1486
1487         /* If this is the first connection to kill we must allocate
1488            a new structure
1489          */
1490         if (killtcp == NULL) {
1491                 killtcp = talloc_zero(ctdb, struct ctdb_kill_tcp);
1492                 CTDB_NO_MEMORY(ctdb, killtcp);
1493
1494                 killtcp->vnn         = vnn;
1495                 killtcp->ctdb        = ctdb;
1496                 killtcp->capture_fd  = -1;
1497                 killtcp->connections = trbt_create(killtcp, 0);
1498
1499                 vnn->killtcp         = killtcp;
1500                 talloc_set_destructor(killtcp, ctdb_killtcp_destructor);
1501         }
1502
1503
1504
1505         /* create a structure that describes this connection we want to
1506            RST and store it in killtcp->connections
1507         */
1508         con = talloc(killtcp, struct ctdb_killtcp_con);
1509         CTDB_NO_MEMORY(ctdb, con);
1510         con->src_addr = src;
1511         con->dst_addr = dst;
1512         con->count    = 0;
1513         con->killtcp  = killtcp;
1514
1515
1516         trbt_insertarray32_callback(killtcp->connections,
1517                         KILLTCP_KEYLEN, killtcp_key(&con->dst_addr, &con->src_addr),
1518                         add_killtcp_callback, con);
1519
1520         /*
1521            If we dont have a socket to listen on yet we must create it
1522          */
1523         if (killtcp->capture_fd == -1) {
1524                 killtcp->capture_fd = ctdb_sys_open_capture_socket(vnn->iface, &killtcp->private_data);
1525                 if (killtcp->capture_fd == -1) {
1526                         DEBUG(DEBUG_CRIT,(__location__ " Failed to open capturing socket for killtcp\n"));
1527                         goto failed;
1528                 }
1529         }
1530
1531
1532         if (killtcp->fde == NULL) {
1533                 killtcp->fde = event_add_fd(ctdb->ev, killtcp, killtcp->capture_fd,
1534                                             EVENT_FD_READ | EVENT_FD_AUTOCLOSE,
1535                                             capture_tcp_handler, killtcp);
1536
1537                 /* We also need to set up some events to tickle all these connections
1538                    until they are all reset
1539                 */
1540                 event_add_timed(ctdb->ev, killtcp, timeval_current_ofs(1, 0),
1541                                 ctdb_tickle_sentenced_connections, killtcp);
1542         }
1543
1544         /* tickle him once now */
1545         ctdb_sys_send_tcp(
1546                 &con->dst_addr,
1547                 &con->src_addr,
1548                 0, 0, 0);
1549
1550         return 0;
1551
1552 failed:
1553         talloc_free(vnn->killtcp);
1554         vnn->killtcp = NULL;
1555         return -1;
1556 }
1557
1558 /*
1559   kill a TCP connection.
1560  */
1561 int32_t ctdb_control_kill_tcp(struct ctdb_context *ctdb, TDB_DATA indata)
1562 {
1563         struct ctdb_control_killtcp *killtcp = (struct ctdb_control_killtcp *)indata.dptr;
1564
1565         return ctdb_killtcp_add_connection(ctdb, &killtcp->src_addr, &killtcp->dst_addr);
1566 }
1567
1568 /*
1569   called by a daemon to inform us of the entire list of TCP tickles for
1570   a particular public address.
1571   this control should only be sent by the node that is currently serving
1572   that public address.
1573  */
1574 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
1575 {
1576         struct ctdb_control_tcp_tickle_list *list = (struct ctdb_control_tcp_tickle_list *)indata.dptr;
1577         struct ctdb_tcp_array *tcparray;
1578         struct ctdb_vnn *vnn;
1579
1580         /* We must at least have tickles.num or else we cant verify the size
1581            of the received data blob
1582          */
1583         if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
1584                                         tickles.connections)) {
1585                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list. Not enough data for the tickle.num field\n"));
1586                 return -1;
1587         }
1588
1589         /* verify that the size of data matches what we expect */
1590         if (indata.dsize < offsetof(struct ctdb_control_tcp_tickle_list,
1591                                 tickles.connections)
1592                          + sizeof(struct ctdb_tcp_connection)
1593                                  * list->tickles.num) {
1594                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_control_set_tcp_tickle_list\n"));
1595                 return -1;
1596         }
1597
1598         vnn = find_public_ip_vnn(ctdb, &list->addr);
1599         if (vnn == NULL) {
1600                 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
1601                         ctdb_addr_to_str(&list->addr)));
1602
1603                 return 1;
1604         }
1605
1606         /* remove any old ticklelist we might have */
1607         talloc_free(vnn->tcp_array);
1608         vnn->tcp_array = NULL;
1609
1610         tcparray = talloc(ctdb->nodes, struct ctdb_tcp_array);
1611         CTDB_NO_MEMORY(ctdb, tcparray);
1612
1613         tcparray->num = list->tickles.num;
1614
1615         tcparray->connections = talloc_array(tcparray, struct ctdb_tcp_connection, tcparray->num);
1616         CTDB_NO_MEMORY(ctdb, tcparray->connections);
1617
1618         memcpy(tcparray->connections, &list->tickles.connections[0],
1619                sizeof(struct ctdb_tcp_connection)*tcparray->num);
1620
1621         /* We now have a new fresh tickle list array for this vnn */
1622         vnn->tcp_array = talloc_steal(vnn, tcparray);
1623
1624         return 0;
1625 }
1626
1627 /*
1628   called to return the full list of tickles for the puclic address associated
1629   with the provided vnn
1630  */
1631 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
1632 {
1633         ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
1634         struct ctdb_control_tcp_tickle_list *list;
1635         struct ctdb_tcp_array *tcparray;
1636         int num;
1637         struct ctdb_vnn *vnn;
1638
1639         vnn = find_public_ip_vnn(ctdb, addr);
1640         if (vnn == NULL) {
1641                 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
1642                         ctdb_addr_to_str(addr)));
1643
1644                 return 1;
1645         }
1646
1647         tcparray = vnn->tcp_array;
1648         if (tcparray) {
1649                 num = tcparray->num;
1650         } else {
1651                 num = 0;
1652         }
1653
1654         outdata->dsize = offsetof(struct ctdb_control_tcp_tickle_list,
1655                                 tickles.connections)
1656                         + sizeof(struct ctdb_tcp_connection) * num;
1657
1658         outdata->dptr  = talloc_size(outdata, outdata->dsize);
1659         CTDB_NO_MEMORY(ctdb, outdata->dptr);
1660         list = (struct ctdb_control_tcp_tickle_list *)outdata->dptr;
1661
1662         list->addr = *addr;
1663         list->tickles.num = num;
1664         if (num) {
1665                 memcpy(&list->tickles.connections[0], tcparray->connections,
1666                         sizeof(struct ctdb_tcp_connection) * num);
1667         }
1668
1669         return 0;
1670 }
1671
1672
1673 /*
1674   set the list of all tcp tickles for a public address
1675  */
1676 static int ctdb_ctrl_set_tcp_tickles(struct ctdb_context *ctdb,
1677                               struct timeval timeout, uint32_t destnode,
1678                               ctdb_sock_addr *addr,
1679                               struct ctdb_tcp_array *tcparray)
1680 {
1681         int ret, num;
1682         TDB_DATA data;
1683         struct ctdb_control_tcp_tickle_list *list;
1684
1685         if (tcparray) {
1686                 num = tcparray->num;
1687         } else {
1688                 num = 0;
1689         }
1690
1691         data.dsize = offsetof(struct ctdb_control_tcp_tickle_list,
1692                                 tickles.connections) +
1693                         sizeof(struct ctdb_tcp_connection) * num;
1694         data.dptr = talloc_size(ctdb, data.dsize);
1695         CTDB_NO_MEMORY(ctdb, data.dptr);
1696
1697         list = (struct ctdb_control_tcp_tickle_list *)data.dptr;
1698         list->addr = *addr;
1699         list->tickles.num = num;
1700         if (tcparray) {
1701                 memcpy(&list->tickles.connections[0], tcparray->connections, sizeof(struct ctdb_tcp_connection) * num);
1702         }
1703
1704         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
1705                                        CTDB_CONTROL_SET_TCP_TICKLE_LIST,
1706                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
1707         if (ret != 0) {
1708                 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
1709                 return -1;
1710         }
1711
1712         talloc_free(data.dptr);
1713
1714         return ret;
1715 }
1716
1717
1718 /*
1719   perform tickle updates if required
1720  */
1721 static void ctdb_update_tcp_tickles(struct event_context *ev,
1722                                 struct timed_event *te,
1723                                 struct timeval t, void *private_data)
1724 {
1725         struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
1726         int ret;
1727         struct ctdb_vnn *vnn;
1728
1729         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1730                 /* we only send out updates for public addresses that
1731                    we have taken over
1732                  */
1733                 if (ctdb->pnn != vnn->pnn) {
1734                         continue;
1735                 }
1736                 /* We only send out the updates if we need to */
1737                 if (!vnn->tcp_update_needed) {
1738                         continue;
1739                 }
1740                 ret = ctdb_ctrl_set_tcp_tickles(ctdb,
1741                                 TAKEOVER_TIMEOUT(),
1742                                 CTDB_BROADCAST_CONNECTED,
1743                                 &vnn->public_address,
1744                                 vnn->tcp_array);
1745                 if (ret != 0) {
1746                         DEBUG(DEBUG_ERR,("Failed to send the tickle update for public address %s\n",
1747                                 ctdb_addr_to_str(&vnn->public_address)));
1748                 }
1749         }
1750
1751         event_add_timed(ctdb->ev, ctdb->tickle_update_context,
1752                              timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
1753                              ctdb_update_tcp_tickles, ctdb);
1754 }
1755
1756
1757 /*
1758   start periodic update of tcp tickles
1759  */
1760 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
1761 {
1762         ctdb->tickle_update_context = talloc_new(ctdb);
1763
1764         event_add_timed(ctdb->ev, ctdb->tickle_update_context,
1765                              timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
1766                              ctdb_update_tcp_tickles, ctdb);
1767 }
1768
1769
1770
1771
1772 struct control_gratious_arp {
1773         struct ctdb_context *ctdb;
1774         ctdb_sock_addr addr;
1775         const char *iface;
1776         int count;
1777 };
1778
1779 /*
1780   send a control_gratuitous arp
1781  */
1782 static void send_gratious_arp(struct event_context *ev, struct timed_event *te,
1783                                   struct timeval t, void *private_data)
1784 {
1785         int ret;
1786         struct control_gratious_arp *arp = talloc_get_type(private_data,
1787                                                         struct control_gratious_arp);
1788
1789         ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
1790         if (ret != 0) {
1791                 DEBUG(DEBUG_ERR,(__location__ " sending of gratious arp failed (%s)\n", strerror(errno)));
1792         }
1793
1794
1795         arp->count++;
1796         if (arp->count == CTDB_ARP_REPEAT) {
1797                 talloc_free(arp);
1798                 return;
1799         }
1800
1801         event_add_timed(arp->ctdb->ev, arp,
1802                         timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
1803                         send_gratious_arp, arp);
1804 }
1805
1806
1807 /*
1808   send a gratious arp
1809  */
1810 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
1811 {
1812         struct ctdb_control_gratious_arp *gratious_arp = (struct ctdb_control_gratious_arp *)indata.dptr;
1813         struct control_gratious_arp *arp;
1814
1815         /* verify the size of indata */
1816         if (indata.dsize < offsetof(struct ctdb_control_gratious_arp, iface)) {
1817                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
1818                                  (unsigned)indata.dsize,
1819                                  (unsigned)offsetof(struct ctdb_control_gratious_arp, iface)));
1820                 return -1;
1821         }
1822         if (indata.dsize !=
1823                 ( offsetof(struct ctdb_control_gratious_arp, iface)
1824                 + gratious_arp->len ) ){
1825
1826                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
1827                         "but should be %u bytes\n",
1828                          (unsigned)indata.dsize,
1829                          (unsigned)(offsetof(struct ctdb_control_gratious_arp, iface)+gratious_arp->len)));
1830                 return -1;
1831         }
1832
1833
1834         arp = talloc(ctdb, struct control_gratious_arp);
1835         CTDB_NO_MEMORY(ctdb, arp);
1836
1837         arp->ctdb  = ctdb;
1838         arp->addr   = gratious_arp->addr;
1839         arp->iface = talloc_strdup(arp, gratious_arp->iface);
1840         CTDB_NO_MEMORY(ctdb, arp->iface);
1841         arp->count = 0;
1842
1843         event_add_timed(arp->ctdb->ev, arp,
1844                         timeval_zero(), send_gratious_arp, arp);
1845
1846         return 0;
1847 }
1848
1849 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
1850 {
1851         struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
1852
1853
1854         /* verify the size of indata */
1855         if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
1856                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
1857                 return -1;
1858         }
1859         if (indata.dsize !=
1860                 ( offsetof(struct ctdb_control_ip_iface, iface)
1861                 + pub->len ) ){
1862
1863                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
1864                         "but should be %u bytes\n",
1865                          (unsigned)indata.dsize,
1866                          (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
1867                 return -1;
1868         }
1869
1870         return ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0]);
1871 }
1872
1873 /*
1874   called when releaseip event finishes for del_public_address
1875  */
1876 static void delete_ip_callback(struct ctdb_context *ctdb, int status,
1877                                 void *private_data)
1878 {
1879         talloc_free(private_data);
1880 }
1881
1882 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
1883 {
1884         struct ctdb_control_ip_iface *pub = (struct ctdb_control_ip_iface *)indata.dptr;
1885         struct ctdb_vnn *vnn;
1886         int ret;
1887
1888         /* verify the size of indata */
1889         if (indata.dsize < offsetof(struct ctdb_control_ip_iface, iface)) {
1890                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_ip_iface structure\n"));
1891                 return -1;
1892         }
1893         if (indata.dsize !=
1894                 ( offsetof(struct ctdb_control_ip_iface, iface)
1895                 + pub->len ) ){
1896
1897                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
1898                         "but should be %u bytes\n",
1899                          (unsigned)indata.dsize,
1900                          (unsigned)(offsetof(struct ctdb_control_ip_iface, iface)+pub->len)));
1901                 return -1;
1902         }
1903
1904         /* walk over all public addresses until we find a match */
1905         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1906                 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
1907                         TALLOC_CTX *mem_ctx = talloc_new(ctdb);
1908
1909                         DLIST_REMOVE(ctdb->vnn, vnn);
1910
1911                         ret = ctdb_event_script_callback(ctdb,
1912                                          timeval_current_ofs(ctdb->tunable.script_timeout, 0),
1913                                          mem_ctx, delete_ip_callback, mem_ctx,
1914                                          "releaseip %s %s %u",
1915                                          vnn->iface,
1916                                          talloc_strdup(mem_ctx, ctdb_addr_to_str(&vnn->public_address)),
1917                                          vnn->public_netmask_bits);
1918                         talloc_free(vnn);
1919                         if (ret != 0) {
1920                                 return -1;
1921                         }
1922                         return 0;
1923                 }
1924         }
1925
1926         return -1;
1927 }
1928