ctdb/server/ctdb_takeover.c

   1 /*
   2    ctdb ip takeover code
   3
   4    Copyright (C) Ronnie Sahlberg  2007
   5    Copyright (C) Andrew Tridgell  2007
   6    Copyright (C) Martin Schwenke  2011
   7
   8    This program is free software; you can redistribute it and/or modify
   9    it under the terms of the GNU General Public License as published by
  10    the Free Software Foundation; either version 3 of the License, or
  11    (at your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful,
  14    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16    GNU General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, see <http://www.gnu.org/licenses/>.
  20 */
  21 #include "replace.h"
  22 #include "system/network.h"
  23 #include "system/filesys.h"
  24 #include "system/time.h"
  25 #include "system/wait.h"
  26
  27 #include <talloc.h>
  28 #include <tevent.h>
  29
  30 #include "lib/util/dlinklist.h"
  31 #include "lib/util/debug.h"
  32 #include "lib/util/samba_util.h"
  33 #include "lib/util/sys_rw.h"
  34 #include "lib/util/util_process.h"
  35
  36 #include "protocol/protocol_util.h"
  37
  38 #include "ctdb_private.h"
  39 #include "ctdb_client.h"
  40
  41 #include "common/reqid.h"
  42 #include "common/system.h"
  43 #include "common/system_socket.h"
  44 #include "common/common.h"
  45 #include "common/logging.h"
  46
  47 #include "server/ctdb_config.h"
  48
  49 #include "server/ipalloc.h"
  50
  51 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
  52
  53 #define CTDB_ARP_INTERVAL 1
  54 #define CTDB_ARP_REPEAT   3
  55
  56 struct ctdb_interface {
  57         struct ctdb_interface *prev, *next;
  58         const char *name;
  59         bool link_up;
  60         uint32_t references;
  61 };
  62
  63 struct vnn_interface {
  64         struct vnn_interface *prev, *next;
  65         struct ctdb_interface *iface;
  66 };
  67
  68 /* state associated with a public ip address */
  69 struct ctdb_vnn {
  70         struct ctdb_vnn *prev, *next;
  71
  72         struct ctdb_interface *iface;
  73         struct vnn_interface *ifaces;
  74         ctdb_sock_addr public_address;
  75         uint8_t public_netmask_bits;
  76
  77         /*
  78          * The node number that is serving this public address - set
  79          * to CTDB_UNKNOWN_PNN if node is serving it
  80          */
  81         uint32_t pnn;
  82
  83         /* List of clients to tickle for this public address */
  84         struct ctdb_tcp_array *tcp_array;
  85
  86         /* whether we need to update the other nodes with changes to our list
  87            of connected clients */
  88         bool tcp_update_needed;
  89
  90         /* a context to hang sending gratious arp events off */
  91         TALLOC_CTX *takeover_ctx;
  92
  93         /* Set to true any time an update to this VNN is in flight.
  94            This helps to avoid races. */
  95         bool update_in_flight;
  96
  97         /* If CTDB_CONTROL_DEL_PUBLIC_IP is received for this IP
  98          * address then this flag is set.  It will be deleted in the
  99          * release IP callback. */
 100         bool delete_pending;
 101 };
 102
 103 static const char *iface_string(const struct ctdb_interface *iface)
 104 {
 105         return (iface != NULL ? iface->name : "__none__");
 106 }
 107
 108 static const char *ctdb_vnn_iface_string(const struct ctdb_vnn *vnn)
 109 {
 110         return iface_string(vnn->iface);
 111 }
 112
 113 static struct ctdb_interface *ctdb_find_iface(struct ctdb_context *ctdb,
 114                                               const char *iface);
 115
 116 static struct ctdb_interface *
 117 ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
 118 {
 119         struct ctdb_interface *i;
 120
 121         if (strlen(iface) > CTDB_IFACE_SIZE) {
 122                 DEBUG(DEBUG_ERR, ("Interface name too long \"%s\"\n", iface));
 123                 return NULL;
 124         }
 125
 126         /* Verify that we don't have an entry for this ip yet */
 127         i = ctdb_find_iface(ctdb, iface);
 128         if (i != NULL) {
 129                 return i;
 130         }
 131
 132         /* create a new structure for this interface */
 133         i = talloc_zero(ctdb, struct ctdb_interface);
 134         if (i == NULL) {
 135                 DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
 136                 return NULL;
 137         }
 138         i->name = talloc_strdup(i, iface);
 139         if (i->name == NULL) {
 140                 DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
 141                 talloc_free(i);
 142                 return NULL;
 143         }
 144
 145         i->link_up = true;
 146
 147         DLIST_ADD(ctdb->ifaces, i);
 148
 149         return i;
 150 }
 151
 152 static bool vnn_has_interface(struct ctdb_vnn *vnn,
 153                               const struct ctdb_interface *iface)
 154 {
 155         struct vnn_interface *i;
 156
 157         for (i = vnn->ifaces; i != NULL; i = i->next) {
 158                 if (iface == i->iface) {
 159                         return true;
 160                 }
 161         }
 162
 163         return false;
 164 }
 165
 166 /* If any interfaces now have no possible IPs then delete them.  This
 167  * implementation is naive (i.e. simple) rather than clever
 168  * (i.e. complex).  Given that this is run on delip and that operation
 169  * is rare, this doesn't need to be efficient - it needs to be
 170  * foolproof.  One alternative is reference counting, where the logic
 171  * is distributed and can, therefore, be broken in multiple places.
 172  * Another alternative is to build a red-black tree of interfaces that
 173  * can have addresses (by walking ctdb->vnn once) and then walking
 174  * ctdb->ifaces once and deleting those not in the tree.  Let's go to
 175  * one of those if the naive implementation causes problems...  :-)
 176  */
 177 static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
 178                                         struct ctdb_vnn *vnn)
 179 {
 180         struct ctdb_interface *i, *next;
 181
 182         /* For each interface, check if there's an IP using it. */
 183         for (i = ctdb->ifaces; i != NULL; i = next) {
 184                 struct ctdb_vnn *tv;
 185                 bool found;
 186                 next = i->next;
 187
 188                 /* Only consider interfaces named in the given VNN. */
 189                 if (!vnn_has_interface(vnn, i)) {
 190                         continue;
 191                 }
 192
 193                 /* Search for a vnn with this interface. */
 194                 found = false;
 195                 for (tv=ctdb->vnn; tv; tv=tv->next) {
 196                         if (vnn_has_interface(tv, i)) {
 197                                 found = true;
 198                                 break;
 199                         }
 200                 }
 201
 202                 if (!found) {
 203                         /* None of the VNNs are using this interface. */
 204                         DLIST_REMOVE(ctdb->ifaces, i);
 205                         talloc_free(i);
 206                 }
 207         }
 208 }
 209
 210
 211 static struct ctdb_interface *ctdb_find_iface(struct ctdb_context *ctdb,
 212                                               const char *iface)
 213 {
 214         struct ctdb_interface *i;
 215
 216         for (i=ctdb->ifaces;i;i=i->next) {
 217                 if (strcmp(i->name, iface) == 0) {
 218                         return i;
 219                 }
 220         }
 221
 222         return NULL;
 223 }
 224
 225 static struct ctdb_interface *ctdb_vnn_best_iface(struct ctdb_context *ctdb,
 226                                                   struct ctdb_vnn *vnn)
 227 {
 228         struct vnn_interface *i;
 229         struct ctdb_interface *cur = NULL;
 230         struct ctdb_interface *best = NULL;
 231
 232         for (i = vnn->ifaces; i != NULL; i = i->next) {
 233
 234                 cur = i->iface;
 235
 236                 if (!cur->link_up) {
 237                         continue;
 238                 }
 239
 240                 if (best == NULL) {
 241                         best = cur;
 242                         continue;
 243                 }
 244
 245                 if (cur->references < best->references) {
 246                         best = cur;
 247                         continue;
 248                 }
 249         }
 250
 251         return best;
 252 }
 253
 254 static int32_t ctdb_vnn_assign_iface(struct ctdb_context *ctdb,
 255                                      struct ctdb_vnn *vnn)
 256 {
 257         struct ctdb_interface *best = NULL;
 258
 259         if (vnn->iface) {
 260                 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 261                                    "still assigned to iface '%s'\n",
 262                                    ctdb_addr_to_str(&vnn->public_address),
 263                                    ctdb_vnn_iface_string(vnn)));
 264                 return 0;
 265         }
 266
 267         best = ctdb_vnn_best_iface(ctdb, vnn);
 268         if (best == NULL) {
 269                 DEBUG(DEBUG_ERR, (__location__ " public address '%s' "
 270                                   "cannot assign to iface any iface\n",
 271                                   ctdb_addr_to_str(&vnn->public_address)));
 272                 return -1;
 273         }
 274
 275         vnn->iface = best;
 276         best->references++;
 277         vnn->pnn = ctdb->pnn;
 278
 279         DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 280                            "now assigned to iface '%s' refs[%d]\n",
 281                            ctdb_addr_to_str(&vnn->public_address),
 282                            ctdb_vnn_iface_string(vnn),
 283                            best->references));
 284         return 0;
 285 }
 286
 287 static void ctdb_vnn_unassign_iface(struct ctdb_context *ctdb,
 288                                     struct ctdb_vnn *vnn)
 289 {
 290         DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 291                            "now unassigned (old iface '%s' refs[%d])\n",
 292                            ctdb_addr_to_str(&vnn->public_address),
 293                            ctdb_vnn_iface_string(vnn),
 294                            vnn->iface?vnn->iface->references:0));
 295         if (vnn->iface) {
 296                 vnn->iface->references--;
 297         }
 298         vnn->iface = NULL;
 299         if (vnn->pnn == ctdb->pnn) {
 300                 vnn->pnn = CTDB_UNKNOWN_PNN;
 301         }
 302 }
 303
 304 static bool ctdb_vnn_available(struct ctdb_context *ctdb,
 305                                struct ctdb_vnn *vnn)
 306 {
 307         uint32_t flags;
 308         struct vnn_interface *i;
 309
 310         /* Nodes that are not RUNNING can not host IPs */
 311         if (ctdb->runstate != CTDB_RUNSTATE_RUNNING) {
 312                 return false;
 313         }
 314
 315         flags = ctdb->nodes[ctdb->pnn]->flags;
 316         if ((flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED)) != 0) {
 317                 return false;
 318         }
 319
 320         if (vnn->delete_pending) {
 321                 return false;
 322         }
 323
 324         if (vnn->iface && vnn->iface->link_up) {
 325                 return true;
 326         }
 327
 328         for (i = vnn->ifaces; i != NULL; i = i->next) {
 329                 if (i->iface->link_up) {
 330                         return true;
 331                 }
 332         }
 333
 334         return false;
 335 }
 336
 337 struct ctdb_takeover_arp {
 338         struct ctdb_context *ctdb;
 339         uint32_t count;
 340         ctdb_sock_addr addr;
 341         struct ctdb_tcp_array *tcparray;
 342         struct ctdb_vnn *vnn;
 343 };
 344
 345
 346 /*
 347   lists of tcp endpoints
 348  */
 349 struct ctdb_tcp_list {
 350         struct ctdb_tcp_list *prev, *next;
 351         struct ctdb_client *client;
 352         struct ctdb_connection connection;
 353 };
 354
 355 /*
 356   send a gratuitous arp
 357  */
 358 static void ctdb_control_send_arp(struct tevent_context *ev,
 359                                   struct tevent_timer *te,
 360                                   struct timeval t, void *private_data)
 361 {
 362         struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
 363                                                         struct ctdb_takeover_arp);
 364         int ret;
 365         struct ctdb_tcp_array *tcparray;
 366         const char *iface;
 367
 368         /* IP address might have been released between sends */
 369         if (arp->vnn->iface == NULL) {
 370                 DBG_INFO("Cancelling ARP send for released IP %s\n",
 371                          ctdb_addr_to_str(&arp->vnn->public_address));
 372                 talloc_free(arp);
 373                 return;
 374         }
 375
 376         iface = ctdb_vnn_iface_string(arp->vnn);
 377         ret = ctdb_sys_send_arp(&arp->addr, iface);
 378         if (ret != 0) {
 379                 DBG_ERR("Failed to send ARP on interface %s: %s\n",
 380                         iface, strerror(ret));
 381         }
 382
 383         tcparray = arp->tcparray;
 384         if (tcparray) {
 385                 unsigned int i;
 386
 387                 for (i=0;i<tcparray->num;i++) {
 388                         struct ctdb_connection *tcon;
 389                         char buf[128];
 390
 391                         tcon = &tcparray->connections[i];
 392                         ret = ctdb_connection_to_buf(buf,
 393                                                      sizeof(buf),
 394                                                      tcon,
 395                                                      false,
 396                                                      " -> ");
 397                         if (ret != 0) {
 398                                 strlcpy(buf, "UNKNOWN", sizeof(buf));
 399                         }
 400                         D_INFO("Send TCP tickle ACK: %s\n", buf);
 401                         ret = ctdb_sys_send_tcp(
 402                                 &tcon->src,
 403                                 &tcon->dst,
 404                                 0, 0, 0);
 405                         if (ret != 0) {
 406                                 DBG_ERR("Failed to send TCP tickle ACK: %s\n",
 407                                         buf);
 408                         }
 409                 }
 410         }
 411
 412         arp->count++;
 413
 414         if (arp->count == CTDB_ARP_REPEAT) {
 415                 talloc_free(arp);
 416                 return;
 417         }
 418
 419         tevent_add_timer(arp->ctdb->ev, arp->vnn->takeover_ctx,
 420                          timeval_current_ofs(CTDB_ARP_INTERVAL, 100000),
 421                          ctdb_control_send_arp, arp);
 422 }
 423
 424 static int32_t ctdb_announce_vnn_iface(struct ctdb_context *ctdb,
 425                                        struct ctdb_vnn *vnn)
 426 {
 427         struct ctdb_takeover_arp *arp;
 428         struct ctdb_tcp_array *tcparray;
 429
 430         if (!vnn->takeover_ctx) {
 431                 vnn->takeover_ctx = talloc_new(vnn);
 432                 if (!vnn->takeover_ctx) {
 433                         return -1;
 434                 }
 435         }
 436
 437         arp = talloc_zero(vnn->takeover_ctx, struct ctdb_takeover_arp);
 438         if (!arp) {
 439                 return -1;
 440         }
 441
 442         arp->ctdb = ctdb;
 443         arp->addr = vnn->public_address;
 444         arp->vnn  = vnn;
 445
 446         tcparray = vnn->tcp_array;
 447         if (tcparray) {
 448                 /* add all of the known tcp connections for this IP to the
 449                    list of tcp connections to send tickle acks for */
 450                 arp->tcparray = talloc_steal(arp, tcparray);
 451
 452                 vnn->tcp_array = NULL;
 453                 vnn->tcp_update_needed = true;
 454         }
 455
 456         tevent_add_timer(arp->ctdb->ev, vnn->takeover_ctx,
 457                          timeval_zero(), ctdb_control_send_arp, arp);
 458
 459         return 0;
 460 }
 461
 462 struct ctdb_do_takeip_state {
 463         struct ctdb_req_control_old *c;
 464         struct ctdb_vnn *vnn;
 465 };
 466
 467 /*
 468   called when takeip event finishes
 469  */
 470 static void ctdb_do_takeip_callback(struct ctdb_context *ctdb, int status,
 471                                     void *private_data)
 472 {
 473         struct ctdb_do_takeip_state *state =
 474                 talloc_get_type(private_data, struct ctdb_do_takeip_state);
 475         int32_t ret;
 476         TDB_DATA data;
 477
 478         if (status != 0) {
 479                 if (status == -ETIMEDOUT) {
 480                         ctdb_ban_self(ctdb);
 481                 }
 482                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 483                                  ctdb_addr_to_str(&state->vnn->public_address),
 484                                  ctdb_vnn_iface_string(state->vnn)));
 485                 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
 486
 487                 talloc_free(state);
 488                 return;
 489         }
 490
 491         if (ctdb->do_checkpublicip) {
 492
 493         ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
 494         if (ret != 0) {
 495                 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
 496                 talloc_free(state);
 497                 return;
 498         }
 499
 500         }
 501
 502         data.dptr  = (uint8_t *)ctdb_addr_to_str(&state->vnn->public_address);
 503         data.dsize = strlen((char *)data.dptr) + 1;
 504         DEBUG(DEBUG_INFO,(__location__ " sending TAKE_IP for '%s'\n", data.dptr));
 505
 506         ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_TAKE_IP, data);
 507
 508
 509         /* the control succeeded */
 510         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 511         talloc_free(state);
 512         return;
 513 }
 514
 515 static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
 516 {
 517         state->vnn->update_in_flight = false;
 518         return 0;
 519 }
 520
 521 /*
 522   take over an ip address
 523  */
 524 static int32_t ctdb_do_takeip(struct ctdb_context *ctdb,
 525                               struct ctdb_req_control_old *c,
 526                               struct ctdb_vnn *vnn)
 527 {
 528         int ret;
 529         struct ctdb_do_takeip_state *state;
 530
 531         if (vnn->update_in_flight) {
 532                 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
 533                                     "update for this IP already in flight\n",
 534                                     ctdb_addr_to_str(&vnn->public_address),
 535                                     vnn->public_netmask_bits));
 536                 return -1;
 537         }
 538
 539         ret = ctdb_vnn_assign_iface(ctdb, vnn);
 540         if (ret != 0) {
 541                 DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
 542                                  "assign a usable interface\n",
 543                                  ctdb_addr_to_str(&vnn->public_address),
 544                                  vnn->public_netmask_bits));
 545                 return -1;
 546         }
 547
 548         state = talloc(vnn, struct ctdb_do_takeip_state);
 549         CTDB_NO_MEMORY(ctdb, state);
 550
 551         state->c = NULL;
 552         state->vnn   = vnn;
 553
 554         vnn->update_in_flight = true;
 555         talloc_set_destructor(state, ctdb_takeip_destructor);
 556
 557         DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
 558                             ctdb_addr_to_str(&vnn->public_address),
 559                             vnn->public_netmask_bits,
 560                             ctdb_vnn_iface_string(vnn)));
 561
 562         ret = ctdb_event_script_callback(ctdb,
 563                                          state,
 564                                          ctdb_do_takeip_callback,
 565                                          state,
 566                                          CTDB_EVENT_TAKE_IP,
 567                                          "%s %s %u",
 568                                          ctdb_vnn_iface_string(vnn),
 569                                          ctdb_addr_to_str(&vnn->public_address),
 570                                          vnn->public_netmask_bits);
 571
 572         if (ret != 0) {
 573                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 574                         ctdb_addr_to_str(&vnn->public_address),
 575                         ctdb_vnn_iface_string(vnn)));
 576                 talloc_free(state);
 577                 return -1;
 578         }
 579
 580         state->c = talloc_steal(ctdb, c);
 581         return 0;
 582 }
 583
 584 struct ctdb_do_updateip_state {
 585         struct ctdb_req_control_old *c;
 586         struct ctdb_interface *old;
 587         struct ctdb_vnn *vnn;
 588 };
 589
 590 /*
 591   called when updateip event finishes
 592  */
 593 static void ctdb_do_updateip_callback(struct ctdb_context *ctdb, int status,
 594                                       void *private_data)
 595 {
 596         struct ctdb_do_updateip_state *state =
 597                 talloc_get_type(private_data, struct ctdb_do_updateip_state);
 598
 599         if (status != 0) {
 600                 if (status == -ETIMEDOUT) {
 601                         ctdb_ban_self(ctdb);
 602                 }
 603                 DEBUG(DEBUG_ERR,
 604                       ("Failed update of IP %s from interface %s to %s\n",
 605                        ctdb_addr_to_str(&state->vnn->public_address),
 606                        iface_string(state->old),
 607                        ctdb_vnn_iface_string(state->vnn)));
 608
 609                 /*
 610                  * All we can do is reset the old interface
 611                  * and let the next run fix it
 612                  */
 613                 ctdb_vnn_unassign_iface(ctdb, state->vnn);
 614                 state->vnn->iface = state->old;
 615                 state->vnn->iface->references++;
 616
 617                 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
 618                 talloc_free(state);
 619                 return;
 620         }
 621
 622         /* the control succeeded */
 623         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 624         talloc_free(state);
 625         return;
 626 }
 627
 628 static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
 629 {
 630         state->vnn->update_in_flight = false;
 631         return 0;
 632 }
 633
 634 /*
 635   update (move) an ip address
 636  */
 637 static int32_t ctdb_do_updateip(struct ctdb_context *ctdb,
 638                                 struct ctdb_req_control_old *c,
 639                                 struct ctdb_vnn *vnn)
 640 {
 641         int ret;
 642         struct ctdb_do_updateip_state *state;
 643         struct ctdb_interface *old = vnn->iface;
 644         const char *old_name = iface_string(old);
 645         const char *new_name;
 646
 647         if (vnn->update_in_flight) {
 648                 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
 649                                     "update for this IP already in flight\n",
 650                                     ctdb_addr_to_str(&vnn->public_address),
 651                                     vnn->public_netmask_bits));
 652                 return -1;
 653         }
 654
 655         ctdb_vnn_unassign_iface(ctdb, vnn);
 656         ret = ctdb_vnn_assign_iface(ctdb, vnn);
 657         if (ret != 0) {
 658                 DEBUG(DEBUG_ERR,("Update of IP %s/%u failed to "
 659                                  "assign a usable interface (old iface '%s')\n",
 660                                  ctdb_addr_to_str(&vnn->public_address),
 661                                  vnn->public_netmask_bits,
 662                                  old_name));
 663                 return -1;
 664         }
 665
 666         if (old == vnn->iface) {
 667                 /* A benign update from one interface onto itself.
 668                  * no need to run the eventscripts in this case, just return
 669                  * success.
 670                  */
 671                 ctdb_request_control_reply(ctdb, c, NULL, 0, NULL);
 672                 return 0;
 673         }
 674
 675         state = talloc(vnn, struct ctdb_do_updateip_state);
 676         CTDB_NO_MEMORY(ctdb, state);
 677
 678         state->c = NULL;
 679         state->old = old;
 680         state->vnn = vnn;
 681
 682         vnn->update_in_flight = true;
 683         talloc_set_destructor(state, ctdb_updateip_destructor);
 684
 685         new_name = ctdb_vnn_iface_string(vnn);
 686         DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
 687                             "interface %s to %s\n",
 688                             ctdb_addr_to_str(&vnn->public_address),
 689                             vnn->public_netmask_bits,
 690                             old_name,
 691                             new_name));
 692
 693         ret = ctdb_event_script_callback(ctdb,
 694                                          state,
 695                                          ctdb_do_updateip_callback,
 696                                          state,
 697                                          CTDB_EVENT_UPDATE_IP,
 698                                          "%s %s %s %u",
 699                                          old_name,
 700                                          new_name,
 701                                          ctdb_addr_to_str(&vnn->public_address),
 702                                          vnn->public_netmask_bits);
 703         if (ret != 0) {
 704                 DEBUG(DEBUG_ERR,
 705                       ("Failed update IP %s from interface %s to %s\n",
 706                        ctdb_addr_to_str(&vnn->public_address),
 707                        old_name, new_name));
 708                 talloc_free(state);
 709                 return -1;
 710         }
 711
 712         state->c = talloc_steal(ctdb, c);
 713         return 0;
 714 }
 715
 716 /*
 717   Find the vnn of the node that has a public ip address
 718   returns -1 if the address is not known as a public address
 719  */
 720 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
 721 {
 722         struct ctdb_vnn *vnn;
 723
 724         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
 725                 if (ctdb_same_ip(&vnn->public_address, addr)) {
 726                         return vnn;
 727                 }
 728         }
 729
 730         return NULL;
 731 }
 732
 733 /*
 734   take over an ip address
 735  */
 736 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
 737                                  struct ctdb_req_control_old *c,
 738                                  TDB_DATA indata,
 739                                  bool *async_reply)
 740 {
 741         int ret;
 742         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 743         struct ctdb_vnn *vnn;
 744         bool have_ip = false;
 745         bool do_updateip = false;
 746         bool do_takeip = false;
 747         struct ctdb_interface *best_iface = NULL;
 748
 749         if (pip->pnn != ctdb->pnn) {
 750                 DEBUG(DEBUG_ERR,(__location__" takeoverip called for an ip '%s' "
 751                                  "with pnn %d, but we're node %d\n",
 752                                  ctdb_addr_to_str(&pip->addr),
 753                                  pip->pnn, ctdb->pnn));
 754                 return -1;
 755         }
 756
 757         /* update out vnn list */
 758         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 759         if (vnn == NULL) {
 760                 DEBUG(DEBUG_INFO,("takeoverip called for an ip '%s' that is not a public address\n",
 761                         ctdb_addr_to_str(&pip->addr)));
 762                 return 0;
 763         }
 764
 765         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 766                 have_ip = ctdb_sys_have_ip(&pip->addr);
 767         }
 768         best_iface = ctdb_vnn_best_iface(ctdb, vnn);
 769         if (best_iface == NULL) {
 770                 DEBUG(DEBUG_ERR,("takeoverip of IP %s/%u failed to find"
 771                                  "a usable interface (old %s, have_ip %d)\n",
 772                                  ctdb_addr_to_str(&vnn->public_address),
 773                                  vnn->public_netmask_bits,
 774                                  ctdb_vnn_iface_string(vnn),
 775                                  have_ip));
 776                 return -1;
 777         }
 778
 779         if (vnn->pnn != ctdb->pnn && have_ip && vnn->pnn != CTDB_UNKNOWN_PNN) {
 780                 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
 781                                   "and we have it on iface[%s], but it was assigned to node %d"
 782                                   "and we are node %d, banning ourself\n",
 783                                  ctdb_addr_to_str(&vnn->public_address),
 784                                  ctdb_vnn_iface_string(vnn), vnn->pnn, ctdb->pnn));
 785                 ctdb_ban_self(ctdb);
 786                 return -1;
 787         }
 788
 789         if (vnn->pnn == CTDB_UNKNOWN_PNN && have_ip) {
 790                 /* This will cause connections to be reset and
 791                  * reestablished.  However, this is a very unusual
 792                  * situation and doing this will completely repair the
 793                  * inconsistency in the VNN.
 794                  */
 795                 DEBUG(DEBUG_WARNING,
 796                       (__location__
 797                        " Doing updateip for IP %s already on an interface\n",
 798                        ctdb_addr_to_str(&vnn->public_address)));
 799                 do_updateip = true;
 800         }
 801
 802         if (vnn->iface) {
 803                 if (vnn->iface != best_iface) {
 804                         if (!vnn->iface->link_up) {
 805                                 do_updateip = true;
 806                         } else if (vnn->iface->references > (best_iface->references + 1)) {
 807                                 /* only move when the rebalance gains something */
 808                                         do_updateip = true;
 809                         }
 810                 }
 811         }
 812
 813         if (!have_ip) {
 814                 if (do_updateip) {
 815                         ctdb_vnn_unassign_iface(ctdb, vnn);
 816                         do_updateip = false;
 817                 }
 818                 do_takeip = true;
 819         }
 820
 821         if (do_takeip) {
 822                 ret = ctdb_do_takeip(ctdb, c, vnn);
 823                 if (ret != 0) {
 824                         return -1;
 825                 }
 826         } else if (do_updateip) {
 827                 ret = ctdb_do_updateip(ctdb, c, vnn);
 828                 if (ret != 0) {
 829                         return -1;
 830                 }
 831         } else {
 832                 /*
 833                  * The interface is up and the kernel known the ip
 834                  * => do nothing
 835                  */
 836                 DEBUG(DEBUG_INFO,("Redundant takeover of IP %s/%u on interface %s (ip already held)\n",
 837                         ctdb_addr_to_str(&pip->addr),
 838                         vnn->public_netmask_bits,
 839                         ctdb_vnn_iface_string(vnn)));
 840                 return 0;
 841         }
 842
 843         /* tell ctdb_control.c that we will be replying asynchronously */
 844         *async_reply = true;
 845
 846         return 0;
 847 }
 848
 849 static void do_delete_ip(struct ctdb_context *ctdb, struct ctdb_vnn *vnn)
 850 {
 851         DLIST_REMOVE(ctdb->vnn, vnn);
 852         ctdb_vnn_unassign_iface(ctdb, vnn);
 853         ctdb_remove_orphaned_ifaces(ctdb, vnn);
 854         talloc_free(vnn);
 855 }
 856
 857 static struct ctdb_vnn *release_ip_post(struct ctdb_context *ctdb,
 858                                         struct ctdb_vnn *vnn,
 859                                         ctdb_sock_addr *addr)
 860 {
 861         TDB_DATA data;
 862
 863         /* Send a message to all clients of this node telling them
 864          * that the cluster has been reconfigured and they should
 865          * close any connections on this IP address
 866          */
 867         data.dptr = (uint8_t *)ctdb_addr_to_str(addr);
 868         data.dsize = strlen((char *)data.dptr)+1;
 869         DEBUG(DEBUG_INFO, ("Sending RELEASE_IP message for %s\n", data.dptr));
 870         ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
 871
 872         ctdb_vnn_unassign_iface(ctdb, vnn);
 873
 874         /* Process the IP if it has been marked for deletion */
 875         if (vnn->delete_pending) {
 876                 do_delete_ip(ctdb, vnn);
 877                 return NULL;
 878         }
 879
 880         return vnn;
 881 }
 882
 883 struct release_ip_callback_state {
 884         struct ctdb_req_control_old *c;
 885         ctdb_sock_addr *addr;
 886         struct ctdb_vnn *vnn;
 887         uint32_t target_pnn;
 888 };
 889
 890 /*
 891   called when releaseip event finishes
 892  */
 893 static void release_ip_callback(struct ctdb_context *ctdb, int status,
 894                                 void *private_data)
 895 {
 896         struct release_ip_callback_state *state =
 897                 talloc_get_type(private_data, struct release_ip_callback_state);
 898
 899         if (status == -ETIMEDOUT) {
 900                 ctdb_ban_self(ctdb);
 901         }
 902
 903         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 904                 if  (ctdb_sys_have_ip(state->addr)) {
 905                         DEBUG(DEBUG_ERR,
 906                               ("IP %s still hosted during release IP callback, failing\n",
 907                                ctdb_addr_to_str(state->addr)));
 908                         ctdb_request_control_reply(ctdb, state->c,
 909                                                    NULL, -1, NULL);
 910                         talloc_free(state);
 911                         return;
 912                 }
 913         }
 914
 915         state->vnn->pnn = state->target_pnn;
 916         state->vnn = release_ip_post(ctdb, state->vnn, state->addr);
 917
 918         /* the control succeeded */
 919         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 920         talloc_free(state);
 921 }
 922
 923 static int ctdb_releaseip_destructor(struct release_ip_callback_state *state)
 924 {
 925         if (state->vnn != NULL) {
 926                 state->vnn->update_in_flight = false;
 927         }
 928         return 0;
 929 }
 930
 931 /*
 932   release an ip address
 933  */
 934 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
 935                                 struct ctdb_req_control_old *c,
 936                                 TDB_DATA indata,
 937                                 bool *async_reply)
 938 {
 939         int ret;
 940         struct release_ip_callback_state *state;
 941         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 942         struct ctdb_vnn *vnn;
 943         const char *iface;
 944
 945         /* update our vnn list */
 946         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 947         if (vnn == NULL) {
 948                 DEBUG(DEBUG_INFO,("releaseip called for an ip '%s' that is not a public address\n",
 949                         ctdb_addr_to_str(&pip->addr)));
 950                 return 0;
 951         }
 952
 953         /* stop any previous arps */
 954         talloc_free(vnn->takeover_ctx);
 955         vnn->takeover_ctx = NULL;
 956
 957         /* RELEASE_IP controls are sent to all nodes that should not
 958          * be hosting a particular IP.  This serves 2 purposes.  The
 959          * first is to help resolve any inconsistencies.  If a node
 960          * does unexpectedly host an IP then it will be released.  The
 961          * 2nd is to use a "redundant release" to tell non-takeover
 962          * nodes where an IP is moving to.  This is how "ctdb ip" can
 963          * report the (likely) location of an IP by only asking the
 964          * local node.  Redundant releases need to update the PNN but
 965          * are otherwise ignored.
 966          */
 967         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 968                 if (!ctdb_sys_have_ip(&pip->addr)) {
 969                         DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
 970                                 ctdb_addr_to_str(&pip->addr),
 971                                 vnn->public_netmask_bits,
 972                                 ctdb_vnn_iface_string(vnn)));
 973                         vnn->pnn = pip->pnn;
 974                         ctdb_vnn_unassign_iface(ctdb, vnn);
 975                         return 0;
 976                 }
 977         } else {
 978                 if (vnn->iface == NULL) {
 979                         DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
 980                                            ctdb_addr_to_str(&pip->addr),
 981                                            vnn->public_netmask_bits));
 982                         vnn->pnn = pip->pnn;
 983                         return 0;
 984                 }
 985         }
 986
 987         /* There is a potential race between take_ip and us because we
 988          * update the VNN via a callback that run when the
 989          * eventscripts have been run.  Avoid the race by allowing one
 990          * update to be in flight at a time.
 991          */
 992         if (vnn->update_in_flight) {
 993                 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
 994                                     "update for this IP already in flight\n",
 995                                     ctdb_addr_to_str(&vnn->public_address),
 996                                     vnn->public_netmask_bits));
 997                 return -1;
 998         }
 999
1000         iface = ctdb_vnn_iface_string(vnn);
1001
1002         DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s  node:%d\n",
1003                 ctdb_addr_to_str(&pip->addr),
1004                 vnn->public_netmask_bits,
1005                 iface,
1006                 pip->pnn));
1007
1008         state = talloc(ctdb, struct release_ip_callback_state);
1009         if (state == NULL) {
1010                 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1011                                __FILE__, __LINE__);
1012                 return -1;
1013         }
1014
1015         state->c = NULL;
1016         state->addr = talloc(state, ctdb_sock_addr);
1017         if (state->addr == NULL) {
1018                 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1019                                __FILE__, __LINE__);
1020                 talloc_free(state);
1021                 return -1;
1022         }
1023         *state->addr = pip->addr;
1024         state->target_pnn = pip->pnn;
1025         state->vnn   = vnn;
1026
1027         vnn->update_in_flight = true;
1028         talloc_set_destructor(state, ctdb_releaseip_destructor);
1029
1030         ret = ctdb_event_script_callback(ctdb,
1031                                          state, release_ip_callback, state,
1032                                          CTDB_EVENT_RELEASE_IP,
1033                                          "%s %s %u",
1034                                          iface,
1035                                          ctdb_addr_to_str(&pip->addr),
1036                                          vnn->public_netmask_bits);
1037         if (ret != 0) {
1038                 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
1039                         ctdb_addr_to_str(&pip->addr),
1040                         ctdb_vnn_iface_string(vnn)));
1041                 talloc_free(state);
1042                 return -1;
1043         }
1044
1045         /* tell the control that we will be reply asynchronously */
1046         *async_reply = true;
1047         state->c = talloc_steal(state, c);
1048         return 0;
1049 }
1050
1051 static int ctdb_add_public_address(struct ctdb_context *ctdb,
1052                                    ctdb_sock_addr *addr,
1053                                    unsigned mask, const char *ifaces,
1054                                    bool check_address)
1055 {
1056         struct ctdb_vnn      *vnn;
1057         char *tmp;
1058         const char *iface;
1059
1060         /* Verify that we don't have an entry for this IP yet */
1061         for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
1062                 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
1063                         D_ERR("Duplicate public IP address '%s'\n",
1064                               ctdb_addr_to_str(addr));
1065                         return -1;
1066                 }
1067         }
1068
1069         /* Create a new VNN structure for this IP address */
1070         vnn = talloc_zero(ctdb, struct ctdb_vnn);
1071         if (vnn == NULL) {
1072                 DBG_ERR("Memory allocation error\n");
1073                 return -1;
1074         }
1075         tmp = talloc_strdup(vnn, ifaces);
1076         if (tmp == NULL) {
1077                 DBG_ERR("Memory allocation error\n");
1078                 talloc_free(vnn);
1079                 return -1;
1080         }
1081         for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1082                 struct vnn_interface *vnn_iface;
1083                 struct ctdb_interface *i;
1084
1085                 if (!ctdb_sys_check_iface_exists(iface)) {
1086                         D_ERR("Unknown interface %s for public address %s\n",
1087                               iface,
1088                               ctdb_addr_to_str(addr));
1089                         talloc_free(vnn);
1090                         return -1;
1091                 }
1092
1093                 i = ctdb_add_local_iface(ctdb, iface);
1094                 if (i == NULL) {
1095                         D_ERR("Failed to add interface '%s' "
1096                               "for public address %s\n",
1097                               iface,
1098                               ctdb_addr_to_str(addr));
1099                         talloc_free(vnn);
1100                         return -1;
1101                 }
1102
1103                 vnn_iface = talloc_zero(vnn, struct vnn_interface);
1104                 if (vnn_iface == NULL) {
1105                         DBG_ERR("Memory allocation error\n");
1106                         talloc_free(vnn);
1107                         return -1;
1108                 }
1109
1110                 vnn_iface->iface = i;
1111                 DLIST_ADD_END(vnn->ifaces, vnn_iface);
1112         }
1113         talloc_free(tmp);
1114         vnn->public_address      = *addr;
1115         vnn->public_netmask_bits = mask;
1116         vnn->pnn                 = -1;
1117
1118         DLIST_ADD(ctdb->vnn, vnn);
1119
1120         return 0;
1121 }
1122
1123 /*
1124   setup the public address lists from a file
1125 */
1126 int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
1127 {
1128         bool ok;
1129         char **lines;
1130         int nlines;
1131         int i;
1132
1133         /* If no public addresses file given then try the default */
1134         if (ctdb->public_addresses_file == NULL) {
1135                 const char *b = getenv("CTDB_BASE");
1136                 if (b == NULL) {
1137                         DBG_ERR("CTDB_BASE not set\n");
1138                         return -1;
1139                 }
1140                 ctdb->public_addresses_file = talloc_asprintf(
1141                                         ctdb, "%s/%s", b, "public_addresses");
1142                 if (ctdb->public_addresses_file == NULL) {
1143                         DBG_ERR("Out of memory\n");
1144                         return -1;
1145                 }
1146         }
1147
1148         /* If the file doesn't exist then warn and do nothing */
1149         ok = file_exist(ctdb->public_addresses_file);
1150         if (!ok) {
1151                 D_WARNING("Not loading public addresses, no file %s\n",
1152                           ctdb->public_addresses_file);
1153                 return 0;
1154         }
1155
1156         lines = file_lines_load(ctdb->public_addresses_file, &nlines, 0, ctdb);
1157         if (lines == NULL) {
1158                 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
1159                 return -1;
1160         }
1161         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
1162                 nlines--;
1163         }
1164
1165         for (i=0;i<nlines;i++) {
1166                 unsigned mask;
1167                 ctdb_sock_addr addr;
1168                 const char *addrstr;
1169                 const char *ifaces;
1170                 char *tok, *line;
1171                 int ret;
1172
1173                 line = lines[i];
1174                 while ((*line == ' ') || (*line == '\t')) {
1175                         line++;
1176                 }
1177                 if (*line == '#') {
1178                         continue;
1179                 }
1180                 if (strcmp(line, "") == 0) {
1181                         continue;
1182                 }
1183                 tok = strtok(line, " \t");
1184                 addrstr = tok;
1185
1186                 tok = strtok(NULL, " \t");
1187                 if (tok == NULL) {
1188                         D_ERR("No interface specified at line %u "
1189                               "of public addresses file\n", i+1);
1190                         talloc_free(lines);
1191                         return -1;
1192                 }
1193                 ifaces = tok;
1194
1195                 if (addrstr == NULL) {
1196                         D_ERR("Badly formed line %u in public address list\n",
1197                               i+1);
1198                         talloc_free(lines);
1199                         return -1;
1200                 }
1201
1202                 ret = ctdb_sock_addr_mask_from_string(addrstr, &addr, &mask);
1203                 if (ret != 0) {
1204                         D_ERR("Badly formed line %u in public address list\n",
1205                               i+1);
1206                         talloc_free(lines);
1207                         return -1;
1208                 }
1209
1210                 if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
1211                         DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
1212                         talloc_free(lines);
1213                         return -1;
1214                 }
1215         }
1216
1217
1218         D_NOTICE("Loaded public addresses from %s\n",
1219                  ctdb->public_addresses_file);
1220
1221         talloc_free(lines);
1222         return 0;
1223 }
1224
1225 /*
1226   destroy a ctdb_tcp_list structure
1227  */
1228 static int ctdb_tcp_list_destructor(struct ctdb_tcp_list *tcp)
1229 {
1230         struct ctdb_client *client = tcp->client;
1231         struct ctdb_connection *conn = &tcp->connection;
1232         char conn_str[132] = { 0, };
1233         int ret;
1234
1235         ret = ctdb_connection_to_buf(conn_str,
1236                                      sizeof(conn_str),
1237                                      conn,
1238                                      false,
1239                                      " -> ");
1240         if (ret != 0) {
1241                 strlcpy(conn_str, "UNKNOWN", sizeof(conn_str));
1242         }
1243
1244         D_DEBUG("removing client TCP connection %s "
1245                 "(client_id %u pid %d)\n",
1246                 conn_str, client->client_id, client->pid);
1247
1248         DLIST_REMOVE(client->tcp_list, tcp);
1249
1250         /*
1251          * We don't call ctdb_remove_connection(vnn, conn) here
1252          * as we want the caller to decide if it's called
1253          * directly (local only) or indirectly via a
1254          * CTDB_CONTROL_TCP_REMOVE broadcast
1255          */
1256
1257         return 0;
1258 }
1259
1260 /*
1261   called by a client to inform us of a TCP connection that it is managing
1262   that should tickled with an ACK when IP takeover is done
1263  */
1264 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
1265                                 TDB_DATA indata)
1266 {
1267         struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
1268         struct ctdb_connection *tcp_sock = NULL;
1269         struct ctdb_tcp_list *tcp;
1270         struct ctdb_connection t;
1271         int ret;
1272         TDB_DATA data;
1273         struct ctdb_vnn *vnn;
1274         ctdb_sock_addr src_addr;
1275         ctdb_sock_addr dst_addr;
1276
1277         /* If we don't have public IPs, tickles are useless */
1278         if (ctdb->vnn == NULL) {
1279                 return 0;
1280         }
1281
1282         tcp_sock = (struct ctdb_connection *)indata.dptr;
1283
1284         ctdb_canonicalize_ip_inplace(&tcp_sock->src);
1285         src_addr = tcp_sock->src;
1286
1287         ctdb_canonicalize_ip_inplace(&tcp_sock->dst);
1288         dst_addr = tcp_sock->dst;
1289
1290         vnn = find_public_ip_vnn(ctdb, &dst_addr);
1291         if (vnn == NULL) {
1292                 char *src_addr_str = NULL;
1293                 char *dst_addr_str = NULL;
1294
1295                 switch (dst_addr.sa.sa_family) {
1296                 case AF_INET:
1297                         if (ntohl(dst_addr.ip.sin_addr.s_addr) == INADDR_LOOPBACK) {
1298                                 /* ignore ... */
1299                                 return 0;
1300                         }
1301                         break;
1302                 case AF_INET6:
1303                         break;
1304                 default:
1305                         DEBUG(DEBUG_ERR,(__location__ " Unknown family type %d\n",
1306                               dst_addr.sa.sa_family));
1307                         return 0;
1308                 }
1309
1310                 src_addr_str = ctdb_sock_addr_to_string(client, &src_addr, false);
1311                 dst_addr_str = ctdb_sock_addr_to_string(client, &dst_addr, false);
1312                 DEBUG(DEBUG_ERR,(
1313                       "Could not register TCP connection from "
1314                       "%s to %s (not a public address) (port %u) "
1315                       "(client_id %u pid %u).\n",
1316                       src_addr_str,
1317                       dst_addr_str,
1318                       ctdb_sock_addr_port(&dst_addr),
1319                       client_id, client->pid));
1320                 TALLOC_FREE(src_addr_str);
1321                 TALLOC_FREE(dst_addr_str);
1322                 return 0;
1323         }
1324
1325         if (vnn->pnn != ctdb->pnn) {
1326                 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
1327                         ctdb_addr_to_str(&dst_addr),
1328                         client_id, client->pid));
1329                 /* failing this call will tell smbd to die */
1330                 return -1;
1331         }
1332
1333         tcp = talloc(client, struct ctdb_tcp_list);
1334         CTDB_NO_MEMORY(ctdb, tcp);
1335         tcp->client = client;
1336
1337         tcp->connection.src = tcp_sock->src;
1338         tcp->connection.dst = tcp_sock->dst;
1339
1340         DLIST_ADD(client->tcp_list, tcp);
1341         talloc_set_destructor(tcp, ctdb_tcp_list_destructor);
1342
1343         t.src = tcp_sock->src;
1344         t.dst = tcp_sock->dst;
1345
1346         data.dptr = (uint8_t *)&t;
1347         data.dsize = sizeof(t);
1348
1349         switch (dst_addr.sa.sa_family) {
1350         case AF_INET:
1351                 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
1352                         (unsigned)ntohs(tcp_sock->dst.ip.sin_port),
1353                         ctdb_addr_to_str(&tcp_sock->src),
1354                         (unsigned)ntohs(tcp_sock->src.ip.sin_port), client_id, client->pid));
1355                 break;
1356         case AF_INET6:
1357                 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
1358                         (unsigned)ntohs(tcp_sock->dst.ip6.sin6_port),
1359                         ctdb_addr_to_str(&tcp_sock->src),
1360                         (unsigned)ntohs(tcp_sock->src.ip6.sin6_port), client_id, client->pid));
1361                 break;
1362         default:
1363                 DEBUG(DEBUG_ERR,(__location__ " Unknown family %d\n",
1364                       dst_addr.sa.sa_family));
1365         }
1366
1367
1368         /* tell all nodes about this tcp connection */
1369         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
1370                                        CTDB_CONTROL_TCP_ADD,
1371                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
1372         if (ret != 0) {
1373                 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
1374                 return -1;
1375         }
1376
1377         return 0;
1378 }
1379
1380 static bool ctdb_client_remove_tcp(struct ctdb_client *client,
1381                                    const struct ctdb_connection *conn)
1382 {
1383         struct ctdb_tcp_list *tcp = NULL;
1384         struct ctdb_tcp_list *tcp_next = NULL;
1385         bool found = false;
1386
1387         for (tcp = client->tcp_list; tcp != NULL; tcp = tcp_next) {
1388                 bool same;
1389
1390                 tcp_next = tcp->next;
1391
1392                 same = ctdb_connection_same(conn, &tcp->connection);
1393                 if (!same) {
1394                         continue;
1395                 }
1396
1397                 TALLOC_FREE(tcp);
1398                 found = true;
1399         }
1400
1401         return found;
1402 }
1403
1404 /*
1405   called by a client to inform us of a TCP connection that was disconnected
1406  */
1407 int32_t ctdb_control_tcp_client_disconnected(struct ctdb_context *ctdb,
1408                                              uint32_t client_id,
1409                                              TDB_DATA indata)
1410 {
1411         struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
1412         struct ctdb_connection *tcp_sock = NULL;
1413         int ret;
1414         TDB_DATA data;
1415         char conn_str[132] = { 0, };
1416         bool found = false;
1417
1418         tcp_sock = (struct ctdb_connection *)indata.dptr;
1419
1420         ctdb_canonicalize_ip_inplace(&tcp_sock->src);
1421         ctdb_canonicalize_ip_inplace(&tcp_sock->dst);
1422
1423         ret = ctdb_connection_to_buf(conn_str,
1424                                      sizeof(conn_str),
1425                                      tcp_sock,
1426                                      false,
1427                                      " -> ");
1428         if (ret != 0) {
1429                 strlcpy(conn_str, "UNKNOWN", sizeof(conn_str));
1430         }
1431
1432         found = ctdb_client_remove_tcp(client, tcp_sock);
1433         if (!found) {
1434                 DBG_DEBUG("TCP connection %s not found "
1435                           "(client_id %u pid %u).\n",
1436                           conn_str, client_id, client->pid);
1437                 return 0;
1438         }
1439
1440         D_INFO("deregistered TCP connection %s "
1441                "(client_id %u pid %u)\n",
1442                conn_str, client_id, client->pid);
1443
1444         data.dptr = (uint8_t *)tcp_sock;
1445         data.dsize = sizeof(*tcp_sock);
1446
1447         /* tell all nodes about this tcp connection is gone */
1448         ret = ctdb_daemon_send_control(ctdb,
1449                                        CTDB_BROADCAST_CONNECTED,
1450                                        0,
1451                                        CTDB_CONTROL_TCP_REMOVE,
1452                                        0,
1453                                        CTDB_CTRL_FLAG_NOREPLY,
1454                                        data,
1455                                        NULL,
1456                                        NULL);
1457         if (ret != 0) {
1458                 DBG_ERR("Failed to send CTDB_CONTROL_TCP_REMOVE: %s\n",
1459                         conn_str);
1460                 return -1;
1461         }
1462
1463         return 0;
1464 }
1465
1466 /*
1467   called by a client to inform us of a TCP connection was passed to a different
1468   "client" (typically with multichannel to another smbd process).
1469  */
1470 int32_t ctdb_control_tcp_client_passed(struct ctdb_context *ctdb,
1471                                        uint32_t client_id,
1472                                        TDB_DATA indata)
1473 {
1474         struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
1475         struct ctdb_connection *tcp_sock = NULL;
1476         int ret;
1477         char conn_str[132] = { 0, };
1478         bool found = false;
1479
1480         tcp_sock = (struct ctdb_connection *)indata.dptr;
1481
1482         ctdb_canonicalize_ip_inplace(&tcp_sock->src);
1483         ctdb_canonicalize_ip_inplace(&tcp_sock->dst);
1484
1485         ret = ctdb_connection_to_buf(conn_str,
1486                                      sizeof(conn_str),
1487                                      tcp_sock,
1488                                      false,
1489                                      " -> ");
1490         if (ret != 0) {
1491                 strlcpy(conn_str, "UNKNOWN", sizeof(conn_str));
1492         }
1493
1494         found = ctdb_client_remove_tcp(client, tcp_sock);
1495         if (!found) {
1496                 DBG_DEBUG("TCP connection from %s not found "
1497                           "(client_id %u pid %u).\n",
1498                           conn_str, client_id, client->pid);
1499                 return 0;
1500         }
1501
1502         D_INFO("TCP connection from %s "
1503                "(client_id %u pid %u) passed to another client\n",
1504                conn_str, client_id, client->pid);
1505
1506         /*
1507          * We don't call CTDB_CONTROL_TCP_REMOVE
1508          * nor ctdb_remove_connection() as the connection
1509          * is still alive, but handled by another client
1510          */
1511
1512         return 0;
1513 }
1514
1515 /*
1516   find a tcp address on a list
1517  */
1518 static struct ctdb_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
1519                                            struct ctdb_connection *tcp)
1520 {
1521         unsigned int i;
1522
1523         if (array == NULL) {
1524                 return NULL;
1525         }
1526
1527         for (i=0;i<array->num;i++) {
1528                 if (ctdb_same_sockaddr(&array->connections[i].src, &tcp->src) &&
1529                     ctdb_same_sockaddr(&array->connections[i].dst, &tcp->dst)) {
1530                         return &array->connections[i];
1531                 }
1532         }
1533         return NULL;
1534 }
1535
1536
1537
1538 /*
1539   called by a daemon to inform us of a TCP connection that one of its
1540   clients managing that should tickled with an ACK when IP takeover is
1541   done
1542  */
1543 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed)
1544 {
1545         struct ctdb_connection *p = (struct ctdb_connection *)indata.dptr;
1546         struct ctdb_tcp_array *tcparray;
1547         struct ctdb_connection tcp;
1548         struct ctdb_vnn *vnn;
1549
1550         /* If we don't have public IPs, tickles are useless */
1551         if (ctdb->vnn == NULL) {
1552                 return 0;
1553         }
1554
1555         vnn = find_public_ip_vnn(ctdb, &p->dst);
1556         if (vnn == NULL) {
1557                 DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
1558                         ctdb_addr_to_str(&p->dst)));
1559
1560                 return -1;
1561         }
1562
1563
1564         tcparray = vnn->tcp_array;
1565
1566         /* If this is the first tickle */
1567         if (tcparray == NULL) {
1568                 tcparray = talloc(vnn, struct ctdb_tcp_array);
1569                 CTDB_NO_MEMORY(ctdb, tcparray);
1570                 vnn->tcp_array = tcparray;
1571
1572                 tcparray->num = 0;
1573                 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_connection));
1574                 CTDB_NO_MEMORY(ctdb, tcparray->connections);
1575
1576                 tcparray->connections[tcparray->num].src = p->src;
1577                 tcparray->connections[tcparray->num].dst = p->dst;
1578                 tcparray->num++;
1579
1580                 if (tcp_update_needed) {
1581                         vnn->tcp_update_needed = true;
1582                 }
1583                 return 0;
1584         }
1585
1586
1587         /* Do we already have this tickle ?*/
1588         tcp.src = p->src;
1589         tcp.dst = p->dst;
1590         if (ctdb_tcp_find(tcparray, &tcp) != NULL) {
1591                 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
1592                         ctdb_addr_to_str(&tcp.dst),
1593                         ntohs(tcp.dst.ip.sin_port),
1594                         vnn->pnn));
1595                 return 0;
1596         }
1597
1598         /* A new tickle, we must add it to the array */
1599         tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
1600                                         struct ctdb_connection,
1601                                         tcparray->num+1);
1602         CTDB_NO_MEMORY(ctdb, tcparray->connections);
1603
1604         tcparray->connections[tcparray->num].src = p->src;
1605         tcparray->connections[tcparray->num].dst = p->dst;
1606         tcparray->num++;
1607
1608         DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
1609                 ctdb_addr_to_str(&tcp.dst),
1610                 ntohs(tcp.dst.ip.sin_port),
1611                 vnn->pnn));
1612
1613         if (tcp_update_needed) {
1614                 vnn->tcp_update_needed = true;
1615         }
1616
1617         return 0;
1618 }
1619
1620
1621 static void ctdb_remove_connection(struct ctdb_vnn *vnn, struct ctdb_connection *conn)
1622 {
1623         struct ctdb_connection *tcpp;
1624
1625         if (vnn == NULL) {
1626                 return;
1627         }
1628
1629         /* if the array is empty we can't remove it
1630            and we don't need to do anything
1631          */
1632         if (vnn->tcp_array == NULL) {
1633                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesn't exist (array is empty) %s:%u\n",
1634                         ctdb_addr_to_str(&conn->dst),
1635                         ntohs(conn->dst.ip.sin_port)));
1636                 return;
1637         }
1638
1639
1640         /* See if we know this connection
1641            if we don't know this connection  then we don't need to do anything
1642          */
1643         tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
1644         if (tcpp == NULL) {
1645                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesn't exist %s:%u\n",
1646                         ctdb_addr_to_str(&conn->dst),
1647                         ntohs(conn->dst.ip.sin_port)));
1648                 return;
1649         }
1650
1651
1652         /* We need to remove this entry from the array.
1653            Instead of allocating a new array and copying data to it
1654            we cheat and just copy the last entry in the existing array
1655            to the entry that is to be removed and just shring the
1656            ->num field
1657          */
1658         *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
1659         vnn->tcp_array->num--;
1660
1661         /* If we deleted the last entry we also need to remove the entire array
1662          */
1663         if (vnn->tcp_array->num == 0) {
1664                 talloc_free(vnn->tcp_array);
1665                 vnn->tcp_array = NULL;
1666         }
1667
1668         vnn->tcp_update_needed = true;
1669
1670         DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
1671                 ctdb_addr_to_str(&conn->src),
1672                 ntohs(conn->src.ip.sin_port)));
1673 }
1674
1675
1676 /*
1677   called by a daemon to inform us of a TCP connection that one of its
1678   clients used are no longer needed in the tickle database
1679  */
1680 int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
1681 {
1682         struct ctdb_vnn *vnn;
1683         struct ctdb_connection *conn = (struct ctdb_connection *)indata.dptr;
1684
1685         /* If we don't have public IPs, tickles are useless */
1686         if (ctdb->vnn == NULL) {
1687                 return 0;
1688         }
1689
1690         vnn = find_public_ip_vnn(ctdb, &conn->dst);
1691         if (vnn == NULL) {
1692                 DEBUG(DEBUG_ERR,
1693                       (__location__ " unable to find public address %s\n",
1694                        ctdb_addr_to_str(&conn->dst)));
1695                 return 0;
1696         }
1697
1698         ctdb_remove_connection(vnn, conn);
1699
1700         return 0;
1701 }
1702
1703
1704 static void ctdb_send_set_tcp_tickles_for_all(struct ctdb_context *ctdb,
1705                                               bool force);
1706
1707 /*
1708   Called when another daemon starts - causes all tickles for all
1709   public addresses we are serving to be sent to the new node on the
1710   next check.  This actually causes the tickles to be sent to the
1711   other node immediately.  In case there is an error, the periodic
1712   timer will send the updates on timer event.  This is simple and
1713   doesn't require careful error handling.
1714  */
1715 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t pnn)
1716 {
1717         DEBUG(DEBUG_INFO, ("Received startup control from node %lu\n",
1718                            (unsigned long) pnn));
1719
1720         ctdb_send_set_tcp_tickles_for_all(ctdb, true);
1721         return 0;
1722 }
1723
1724
1725 /*
1726   called when a client structure goes away - hook to remove
1727   elements from the tcp_list in all daemons
1728  */
1729 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
1730 {
1731         while (client->tcp_list) {
1732                 struct ctdb_vnn *vnn;
1733                 struct ctdb_tcp_list *tcp = client->tcp_list;
1734                 struct ctdb_connection *conn = &tcp->connection;
1735
1736                 vnn = find_public_ip_vnn(client->ctdb,
1737                                          &conn->dst);
1738
1739                 /* If the IP address is hosted on this node then
1740                  * remove the connection. */
1741                 if (vnn != NULL && vnn->pnn == client->ctdb->pnn) {
1742                         ctdb_remove_connection(vnn, conn);
1743                 }
1744
1745                 /* Otherwise this function has been called because the
1746                  * server IP address has been released to another node
1747                  * and the client has exited.  This means that we
1748                  * should not delete the connection information.  The
1749                  * takeover node processes connections too. */
1750
1751                 /*
1752                  * The destructor removes from the list
1753                  */
1754                 TALLOC_FREE(tcp);
1755         }
1756 }
1757
1758
1759 void ctdb_release_all_ips(struct ctdb_context *ctdb)
1760 {
1761         struct ctdb_vnn *vnn, *next;
1762         int count = 0;
1763
1764         if (ctdb_config.failover_disabled == 1) {
1765                 return;
1766         }
1767
1768         for (vnn = ctdb->vnn; vnn != NULL; vnn = next) {
1769                 /* vnn can be freed below in release_ip_post() */
1770                 next = vnn->next;
1771
1772                 if (!ctdb_sys_have_ip(&vnn->public_address)) {
1773                         ctdb_vnn_unassign_iface(ctdb, vnn);
1774                         continue;
1775                 }
1776
1777                 /* Don't allow multiple releases at once.  Some code,
1778                  * particularly ctdb_tickle_sentenced_connections() is
1779                  * not re-entrant */
1780                 if (vnn->update_in_flight) {
1781                         DEBUG(DEBUG_WARNING,
1782                               (__location__
1783                                " Not releasing IP %s/%u on interface %s, an update is already in progress\n",
1784                                     ctdb_addr_to_str(&vnn->public_address),
1785                                     vnn->public_netmask_bits,
1786                                     ctdb_vnn_iface_string(vnn)));
1787                         continue;
1788                 }
1789                 vnn->update_in_flight = true;
1790
1791                 DEBUG(DEBUG_INFO,("Release of IP %s/%u on interface %s node:-1\n",
1792                                     ctdb_addr_to_str(&vnn->public_address),
1793                                     vnn->public_netmask_bits,
1794                                     ctdb_vnn_iface_string(vnn)));
1795
1796                 ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
1797                                        ctdb_vnn_iface_string(vnn),
1798                                        ctdb_addr_to_str(&vnn->public_address),
1799                                        vnn->public_netmask_bits);
1800                 /* releaseip timeouts are converted to success, so to
1801                  * detect failures just check if the IP address is
1802                  * still there...
1803                  */
1804                 if (ctdb_sys_have_ip(&vnn->public_address)) {
1805                         DEBUG(DEBUG_ERR,
1806                               (__location__
1807                                " IP address %s not released\n",
1808                                ctdb_addr_to_str(&vnn->public_address)));
1809                         vnn->update_in_flight = false;
1810                         continue;
1811                 }
1812
1813                 vnn = release_ip_post(ctdb, vnn, &vnn->public_address);
1814                 if (vnn != NULL) {
1815                         vnn->update_in_flight = false;
1816                 }
1817                 count++;
1818         }
1819
1820         DEBUG(DEBUG_NOTICE,(__location__ " Released %d public IPs\n", count));
1821 }
1822
1823
1824 /*
1825   get list of public IPs
1826  */
1827 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
1828                                     struct ctdb_req_control_old *c, TDB_DATA *outdata)
1829 {
1830         int i, num, len;
1831         struct ctdb_public_ip_list_old *ips;
1832         struct ctdb_vnn *vnn;
1833         bool only_available = false;
1834
1835         if (c->flags & CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE) {
1836                 only_available = true;
1837         }
1838
1839         /* count how many public ip structures we have */
1840         num = 0;
1841         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1842                 num++;
1843         }
1844
1845         len = offsetof(struct ctdb_public_ip_list_old, ips) +
1846                 num*sizeof(struct ctdb_public_ip);
1847         ips = talloc_zero_size(outdata, len);
1848         CTDB_NO_MEMORY(ctdb, ips);
1849
1850         i = 0;
1851         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1852                 if (only_available && !ctdb_vnn_available(ctdb, vnn)) {
1853                         continue;
1854                 }
1855                 ips->ips[i].pnn  = vnn->pnn;
1856                 ips->ips[i].addr = vnn->public_address;
1857                 i++;
1858         }
1859         ips->num = i;
1860         len = offsetof(struct ctdb_public_ip_list_old, ips) +
1861                 i*sizeof(struct ctdb_public_ip);
1862
1863         outdata->dsize = len;
1864         outdata->dptr  = (uint8_t *)ips;
1865
1866         return 0;
1867 }
1868
1869
1870 int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
1871                                         struct ctdb_req_control_old *c,
1872                                         TDB_DATA indata,
1873                                         TDB_DATA *outdata)
1874 {
1875         int i, num, len;
1876         ctdb_sock_addr *addr;
1877         struct ctdb_public_ip_info_old *info;
1878         struct ctdb_vnn *vnn;
1879         struct vnn_interface *iface;
1880
1881         addr = (ctdb_sock_addr *)indata.dptr;
1882
1883         vnn = find_public_ip_vnn(ctdb, addr);
1884         if (vnn == NULL) {
1885                 DEBUG(DEBUG_ERR,(__location__ " Could not get public ip info, "
1886                                  "'%s'not a public address\n",
1887                                  ctdb_addr_to_str(addr)));
1888                 return -1;
1889         }
1890
1891         /* count how many public ip structures we have */
1892         num = 0;
1893         for (iface = vnn->ifaces; iface != NULL; iface = iface->next) {
1894                 num++;
1895         }
1896
1897         len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
1898                 num*sizeof(struct ctdb_iface);
1899         info = talloc_zero_size(outdata, len);
1900         CTDB_NO_MEMORY(ctdb, info);
1901
1902         info->ip.addr = vnn->public_address;
1903         info->ip.pnn = vnn->pnn;
1904         info->active_idx = 0xFFFFFFFF;
1905
1906         i = 0;
1907         for (iface = vnn->ifaces; iface != NULL; iface = iface->next) {
1908                 struct ctdb_interface *cur;
1909
1910                 cur = iface->iface;
1911                 if (vnn->iface == cur) {
1912                         info->active_idx = i;
1913                 }
1914                 strncpy(info->ifaces[i].name, cur->name,
1915                         sizeof(info->ifaces[i].name));
1916                 info->ifaces[i].name[sizeof(info->ifaces[i].name)-1] = '\0';
1917                 info->ifaces[i].link_state = cur->link_up;
1918                 info->ifaces[i].references = cur->references;
1919
1920                 i++;
1921         }
1922         info->num = i;
1923         len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
1924                 i*sizeof(struct ctdb_iface);
1925
1926         outdata->dsize = len;
1927         outdata->dptr  = (uint8_t *)info;
1928
1929         return 0;
1930 }
1931
1932 int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
1933                                 struct ctdb_req_control_old *c,
1934                                 TDB_DATA *outdata)
1935 {
1936         int i, num, len;
1937         struct ctdb_iface_list_old *ifaces;
1938         struct ctdb_interface *cur;
1939
1940         /* count how many public ip structures we have */
1941         num = 0;
1942         for (cur=ctdb->ifaces;cur;cur=cur->next) {
1943                 num++;
1944         }
1945
1946         len = offsetof(struct ctdb_iface_list_old, ifaces) +
1947                 num*sizeof(struct ctdb_iface);
1948         ifaces = talloc_zero_size(outdata, len);
1949         CTDB_NO_MEMORY(ctdb, ifaces);
1950
1951         i = 0;
1952         for (cur=ctdb->ifaces;cur;cur=cur->next) {
1953                 strncpy(ifaces->ifaces[i].name, cur->name,
1954                         sizeof(ifaces->ifaces[i].name));
1955                 ifaces->ifaces[i].name[sizeof(ifaces->ifaces[i].name)-1] = '\0';
1956                 ifaces->ifaces[i].link_state = cur->link_up;
1957                 ifaces->ifaces[i].references = cur->references;
1958                 i++;
1959         }
1960         ifaces->num = i;
1961         len = offsetof(struct ctdb_iface_list_old, ifaces) +
1962                 i*sizeof(struct ctdb_iface);
1963
1964         outdata->dsize = len;
1965         outdata->dptr  = (uint8_t *)ifaces;
1966
1967         return 0;
1968 }
1969
1970 int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
1971                                     struct ctdb_req_control_old *c,
1972                                     TDB_DATA indata)
1973 {
1974         struct ctdb_iface *info;
1975         struct ctdb_interface *iface;
1976         bool link_up = false;
1977
1978         info = (struct ctdb_iface *)indata.dptr;
1979
1980         if (info->name[CTDB_IFACE_SIZE] != '\0') {
1981                 int len = strnlen(info->name, CTDB_IFACE_SIZE);
1982                 DEBUG(DEBUG_ERR, (__location__ " name[%*.*s] not terminated\n",
1983                                   len, len, info->name));
1984                 return -1;
1985         }
1986
1987         switch (info->link_state) {
1988         case 0:
1989                 link_up = false;
1990                 break;
1991         case 1:
1992                 link_up = true;
1993                 break;
1994         default:
1995                 DEBUG(DEBUG_ERR, (__location__ " link_state[%u] invalid\n",
1996                                   (unsigned int)info->link_state));
1997                 return -1;
1998         }
1999
2000         if (info->references != 0) {
2001                 DEBUG(DEBUG_ERR, (__location__ " references[%u] should be 0\n",
2002                                   (unsigned int)info->references));
2003                 return -1;
2004         }
2005
2006         iface = ctdb_find_iface(ctdb, info->name);
2007         if (iface == NULL) {
2008                 return -1;
2009         }
2010
2011         if (link_up == iface->link_up) {
2012                 return 0;
2013         }
2014
2015         DEBUG(DEBUG_ERR,
2016               ("iface[%s] has changed it's link status %s => %s\n",
2017                iface->name,
2018                iface->link_up?"up":"down",
2019                link_up?"up":"down"));
2020
2021         iface->link_up = link_up;
2022         return 0;
2023 }
2024
2025
2026 /*
2027   called by a daemon to inform us of the entire list of TCP tickles for
2028   a particular public address.
2029   this control should only be sent by the node that is currently serving
2030   that public address.
2031  */
2032 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
2033 {
2034         struct ctdb_tickle_list_old *list = (struct ctdb_tickle_list_old *)indata.dptr;
2035         struct ctdb_tcp_array *tcparray;
2036         struct ctdb_vnn *vnn;
2037
2038         /* We must at least have tickles.num or else we can't verify the size
2039            of the received data blob
2040          */
2041         if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)) {
2042                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list. Not enough data for the tickle.num field\n"));
2043                 return -1;
2044         }
2045
2046         /* verify that the size of data matches what we expect */
2047         if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)
2048                          + sizeof(struct ctdb_connection) * list->num) {
2049                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list\n"));
2050                 return -1;
2051         }
2052
2053         DEBUG(DEBUG_INFO, ("Received tickle update for public address %s\n",
2054                            ctdb_addr_to_str(&list->addr)));
2055
2056         vnn = find_public_ip_vnn(ctdb, &list->addr);
2057         if (vnn == NULL) {
2058                 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
2059                         ctdb_addr_to_str(&list->addr)));
2060
2061                 return 1;
2062         }
2063
2064         if (vnn->pnn == ctdb->pnn) {
2065                 DEBUG(DEBUG_INFO,
2066                       ("Ignoring redundant set tcp tickle list, this node hosts '%s'\n",
2067                        ctdb_addr_to_str(&list->addr)));
2068                 return 0;
2069         }
2070
2071         /* remove any old ticklelist we might have */
2072         talloc_free(vnn->tcp_array);
2073         vnn->tcp_array = NULL;
2074
2075         tcparray = talloc(vnn, struct ctdb_tcp_array);
2076         CTDB_NO_MEMORY(ctdb, tcparray);
2077
2078         tcparray->num = list->num;
2079
2080         tcparray->connections = talloc_array(tcparray, struct ctdb_connection, tcparray->num);
2081         CTDB_NO_MEMORY(ctdb, tcparray->connections);
2082
2083         memcpy(tcparray->connections, &list->connections[0],
2084                sizeof(struct ctdb_connection)*tcparray->num);
2085
2086         /* We now have a new fresh tickle list array for this vnn */
2087         vnn->tcp_array = tcparray;
2088
2089         return 0;
2090 }
2091
2092 /*
2093   called to return the full list of tickles for the puclic address associated
2094   with the provided vnn
2095  */
2096 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
2097 {
2098         ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
2099         struct ctdb_tickle_list_old *list;
2100         struct ctdb_tcp_array *tcparray;
2101         unsigned int num, i;
2102         struct ctdb_vnn *vnn;
2103         unsigned port;
2104
2105         vnn = find_public_ip_vnn(ctdb, addr);
2106         if (vnn == NULL) {
2107                 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
2108                         ctdb_addr_to_str(addr)));
2109
2110                 return 1;
2111         }
2112
2113         port = ctdb_addr_to_port(addr);
2114
2115         tcparray = vnn->tcp_array;
2116         num = 0;
2117         if (tcparray != NULL) {
2118                 if (port == 0) {
2119                         /* All connections */
2120                         num = tcparray->num;
2121                 } else {
2122                         /* Count connections for port */
2123                         for (i = 0; i < tcparray->num; i++) {
2124                                 if (port == ctdb_addr_to_port(&tcparray->connections[i].dst)) {
2125                                         num++;
2126                                 }
2127                         }
2128                 }
2129         }
2130
2131         outdata->dsize = offsetof(struct ctdb_tickle_list_old, connections)
2132                         + sizeof(struct ctdb_connection) * num;
2133
2134         outdata->dptr  = talloc_size(outdata, outdata->dsize);
2135         CTDB_NO_MEMORY(ctdb, outdata->dptr);
2136         list = (struct ctdb_tickle_list_old *)outdata->dptr;
2137
2138         list->addr = *addr;
2139         list->num = num;
2140
2141         if (num == 0) {
2142                 return 0;
2143         }
2144
2145         num = 0;
2146         for (i = 0; i < tcparray->num; i++) {
2147                 if (port == 0 || \
2148                     port == ctdb_addr_to_port(&tcparray->connections[i].dst)) {
2149                         list->connections[num] = tcparray->connections[i];
2150                         num++;
2151                 }
2152         }
2153
2154         return 0;
2155 }
2156
2157
2158 /*
2159   set the list of all tcp tickles for a public address
2160  */
2161 static int ctdb_send_set_tcp_tickles_for_ip(struct ctdb_context *ctdb,
2162                                             ctdb_sock_addr *addr,
2163                                             struct ctdb_tcp_array *tcparray)
2164 {
2165         int ret, num;
2166         TDB_DATA data;
2167         struct ctdb_tickle_list_old *list;
2168
2169         if (tcparray) {
2170                 num = tcparray->num;
2171         } else {
2172                 num = 0;
2173         }
2174
2175         data.dsize = offsetof(struct ctdb_tickle_list_old, connections) +
2176                         sizeof(struct ctdb_connection) * num;
2177         data.dptr = talloc_size(ctdb, data.dsize);
2178         CTDB_NO_MEMORY(ctdb, data.dptr);
2179
2180         list = (struct ctdb_tickle_list_old *)data.dptr;
2181         list->addr = *addr;
2182         list->num = num;
2183         if (tcparray) {
2184                 memcpy(&list->connections[0], tcparray->connections, sizeof(struct ctdb_connection) * num);
2185         }
2186
2187         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
2188                                        CTDB_CONTROL_SET_TCP_TICKLE_LIST,
2189                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
2190         if (ret != 0) {
2191                 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
2192                 return -1;
2193         }
2194
2195         talloc_free(data.dptr);
2196
2197         return ret;
2198 }
2199
2200 static void ctdb_send_set_tcp_tickles_for_all(struct ctdb_context *ctdb,
2201                                               bool force)
2202 {
2203         struct ctdb_vnn *vnn;
2204         int ret;
2205
2206         for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
2207                 /* we only send out updates for public addresses that
2208                    we have taken over
2209                  */
2210                 if (ctdb->pnn != vnn->pnn) {
2211                         continue;
2212                 }
2213
2214                 /* We only send out the updates if we need to */
2215                 if (!force && !vnn->tcp_update_needed) {
2216                         continue;
2217                 }
2218
2219                 ret = ctdb_send_set_tcp_tickles_for_ip(ctdb,
2220                                                        &vnn->public_address,
2221                                                        vnn->tcp_array);
2222                 if (ret != 0) {
2223                         D_ERR("Failed to send the tickle update for ip %s\n",
2224                               ctdb_addr_to_str(&vnn->public_address));
2225                         vnn->tcp_update_needed = true;
2226                 } else {
2227                         D_INFO("Sent tickle update for ip %s\n",
2228                                ctdb_addr_to_str(&vnn->public_address));
2229                         vnn->tcp_update_needed = false;
2230                 }
2231         }
2232
2233 }
2234
2235 /*
2236   perform tickle updates if required
2237  */
2238 static void ctdb_update_tcp_tickles(struct tevent_context *ev,
2239                                     struct tevent_timer *te,
2240                                     struct timeval t, void *private_data)
2241 {
2242         struct ctdb_context *ctdb = talloc_get_type(
2243                 private_data, struct ctdb_context);
2244
2245         ctdb_send_set_tcp_tickles_for_all(ctdb, false);
2246
2247         tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
2248                          timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
2249                          ctdb_update_tcp_tickles, ctdb);
2250 }
2251
2252 /*
2253   start periodic update of tcp tickles
2254  */
2255 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
2256 {
2257         ctdb->tickle_update_context = talloc_new(ctdb);
2258
2259         tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
2260                          timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
2261                          ctdb_update_tcp_tickles, ctdb);
2262 }
2263
2264
2265
2266
2267 struct control_gratious_arp {
2268         struct ctdb_context *ctdb;
2269         ctdb_sock_addr addr;
2270         const char *iface;
2271         int count;
2272 };
2273
2274 /*
2275   send a control_gratuitous arp
2276  */
2277 static void send_gratious_arp(struct tevent_context *ev,
2278                               struct tevent_timer *te,
2279                               struct timeval t, void *private_data)
2280 {
2281         int ret;
2282         struct control_gratious_arp *arp = talloc_get_type(private_data,
2283                                                         struct control_gratious_arp);
2284
2285         ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
2286         if (ret != 0) {
2287                 DBG_ERR("Failed to send gratuitous ARP on iface %s: %s\n",
2288                         arp->iface, strerror(ret));
2289         }
2290
2291
2292         arp->count++;
2293         if (arp->count == CTDB_ARP_REPEAT) {
2294                 talloc_free(arp);
2295                 return;
2296         }
2297
2298         tevent_add_timer(arp->ctdb->ev, arp,
2299                          timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
2300                          send_gratious_arp, arp);
2301 }
2302
2303
2304 /*
2305   send a gratious arp
2306  */
2307 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
2308 {
2309         struct ctdb_addr_info_old *gratious_arp = (struct ctdb_addr_info_old *)indata.dptr;
2310         struct control_gratious_arp *arp;
2311
2312         /* verify the size of indata */
2313         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2314                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
2315                                  (unsigned)indata.dsize,
2316                                  (unsigned)offsetof(struct ctdb_addr_info_old, iface)));
2317                 return -1;
2318         }
2319         if (indata.dsize !=
2320                 ( offsetof(struct ctdb_addr_info_old, iface)
2321                 + gratious_arp->len ) ){
2322
2323                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2324                         "but should be %u bytes\n",
2325                          (unsigned)indata.dsize,
2326                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+gratious_arp->len)));
2327                 return -1;
2328         }
2329
2330
2331         arp = talloc(ctdb, struct control_gratious_arp);
2332         CTDB_NO_MEMORY(ctdb, arp);
2333
2334         arp->ctdb  = ctdb;
2335         arp->addr   = gratious_arp->addr;
2336         arp->iface = talloc_strdup(arp, gratious_arp->iface);
2337         CTDB_NO_MEMORY(ctdb, arp->iface);
2338         arp->count = 0;
2339
2340         tevent_add_timer(arp->ctdb->ev, arp,
2341                          timeval_zero(), send_gratious_arp, arp);
2342
2343         return 0;
2344 }
2345
2346 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
2347 {
2348         struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
2349         int ret;
2350
2351         /* verify the size of indata */
2352         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2353                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
2354                 return -1;
2355         }
2356         if (indata.dsize !=
2357                 ( offsetof(struct ctdb_addr_info_old, iface)
2358                 + pub->len ) ){
2359
2360                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2361                         "but should be %u bytes\n",
2362                          (unsigned)indata.dsize,
2363                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
2364                 return -1;
2365         }
2366
2367         DEBUG(DEBUG_NOTICE,("Add IP %s\n", ctdb_addr_to_str(&pub->addr)));
2368
2369         ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
2370
2371         if (ret != 0) {
2372                 DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
2373                 return -1;
2374         }
2375
2376         return 0;
2377 }
2378
2379 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
2380 {
2381         struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
2382         struct ctdb_vnn *vnn;
2383
2384         /* verify the size of indata */
2385         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2386                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
2387                 return -1;
2388         }
2389         if (indata.dsize !=
2390                 ( offsetof(struct ctdb_addr_info_old, iface)
2391                 + pub->len ) ){
2392
2393                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2394                         "but should be %u bytes\n",
2395                          (unsigned)indata.dsize,
2396                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
2397                 return -1;
2398         }
2399
2400         DEBUG(DEBUG_NOTICE,("Delete IP %s\n", ctdb_addr_to_str(&pub->addr)));
2401
2402         /* walk over all public addresses until we find a match */
2403         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2404                 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
2405                         if (vnn->pnn == ctdb->pnn) {
2406                                 /* This IP is currently being hosted.
2407                                  * Defer the deletion until the next
2408                                  * takeover run. "ctdb reloadips" will
2409                                  * always cause a takeover run.  "ctdb
2410                                  * delip" will now need an explicit
2411                                  * "ctdb ipreallocated" afterwards. */
2412                                 vnn->delete_pending = true;
2413                         } else {
2414                                 /* This IP is not hosted on the
2415                                  * current node so just delete it
2416                                  * now. */
2417                                 do_delete_ip(ctdb, vnn);
2418                         }
2419
2420                         return 0;
2421                 }
2422         }
2423
2424         DEBUG(DEBUG_ERR,("Delete IP of unknown public IP address %s\n",
2425                          ctdb_addr_to_str(&pub->addr)));
2426         return -1;
2427 }
2428
2429
2430 struct ipreallocated_callback_state {
2431         struct ctdb_req_control_old *c;
2432 };
2433
2434 static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
2435                                         int status, void *p)
2436 {
2437         struct ipreallocated_callback_state *state =
2438                 talloc_get_type(p, struct ipreallocated_callback_state);
2439
2440         if (status != 0) {
2441                 DEBUG(DEBUG_ERR,
2442                       (" \"ipreallocated\" event script failed (status %d)\n",
2443                        status));
2444                 if (status == -ETIMEDOUT) {
2445                         ctdb_ban_self(ctdb);
2446                 }
2447         }
2448
2449         ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
2450         talloc_free(state);
2451 }
2452
2453 /* A control to run the ipreallocated event */
2454 int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
2455                                    struct ctdb_req_control_old *c,
2456                                    bool *async_reply)
2457 {
2458         int ret;
2459         struct ipreallocated_callback_state *state;
2460
2461         state = talloc(ctdb, struct ipreallocated_callback_state);
2462         CTDB_NO_MEMORY(ctdb, state);
2463
2464         DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
2465
2466         ret = ctdb_event_script_callback(ctdb, state,
2467                                          ctdb_ipreallocated_callback, state,
2468                                          CTDB_EVENT_IPREALLOCATED,
2469                                          "%s", "");
2470
2471         if (ret != 0) {
2472                 DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
2473                 talloc_free(state);
2474                 return -1;
2475         }
2476
2477         /* tell the control that we will be reply asynchronously */
2478         state->c    = talloc_steal(state, c);
2479         *async_reply = true;
2480
2481         return 0;
2482 }
2483
2484
2485 struct ctdb_reloadips_handle {
2486         struct ctdb_context *ctdb;
2487         struct ctdb_req_control_old *c;
2488         int status;
2489         int fd[2];
2490         pid_t child;
2491         struct tevent_fd *fde;
2492 };
2493
2494 static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
2495 {
2496         if (h == h->ctdb->reload_ips) {
2497                 h->ctdb->reload_ips = NULL;
2498         }
2499         if (h->c != NULL) {
2500                 ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
2501                 h->c = NULL;
2502         }
2503         ctdb_kill(h->ctdb, h->child, SIGKILL);
2504         return 0;
2505 }
2506
2507 static void ctdb_reloadips_timeout_event(struct tevent_context *ev,
2508                                          struct tevent_timer *te,
2509                                          struct timeval t, void *private_data)
2510 {
2511         struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
2512
2513         talloc_free(h);
2514 }
2515
2516 static void ctdb_reloadips_child_handler(struct tevent_context *ev,
2517                                          struct tevent_fd *fde,
2518                                          uint16_t flags, void *private_data)
2519 {
2520         struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
2521
2522         char res;
2523         int ret;
2524
2525         ret = sys_read(h->fd[0], &res, 1);
2526         if (ret < 1 || res != 0) {
2527                 DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
2528                 res = 1;
2529         }
2530         h->status = res;
2531
2532         talloc_free(h);
2533 }
2534
2535 static int ctdb_reloadips_child(struct ctdb_context *ctdb)
2536 {
2537         TALLOC_CTX *mem_ctx = talloc_new(NULL);
2538         struct ctdb_public_ip_list_old *ips;
2539         struct ctdb_vnn *vnn;
2540         struct client_async_data *async_data;
2541         struct timeval timeout;
2542         TDB_DATA data;
2543         struct ctdb_client_control_state *state;
2544         bool first_add;
2545         unsigned int i;
2546         int ret;
2547
2548         CTDB_NO_MEMORY(ctdb, mem_ctx);
2549
2550         /* Read IPs from local node */
2551         ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(),
2552                                        CTDB_CURRENT_NODE, mem_ctx, &ips);
2553         if (ret != 0) {
2554                 DEBUG(DEBUG_ERR,
2555                       ("Unable to fetch public IPs from local node\n"));
2556                 talloc_free(mem_ctx);
2557                 return -1;
2558         }
2559
2560         /* Read IPs file - this is safe since this is a child process */
2561         ctdb->vnn = NULL;
2562         if (ctdb_set_public_addresses(ctdb, false) != 0) {
2563                 DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
2564                 talloc_free(mem_ctx);
2565                 return -1;
2566         }
2567
2568         async_data = talloc_zero(mem_ctx, struct client_async_data);
2569         CTDB_NO_MEMORY(ctdb, async_data);
2570
2571         /* Compare IPs between node and file for IPs to be deleted */
2572         for (i = 0; i < ips->num; i++) {
2573                 /* */
2574                 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
2575                         if (ctdb_same_ip(&vnn->public_address,
2576                                          &ips->ips[i].addr)) {
2577                                 /* IP is still in file */
2578                                 break;
2579                         }
2580                 }
2581
2582                 if (vnn == NULL) {
2583                         /* Delete IP ips->ips[i] */
2584                         struct ctdb_addr_info_old *pub;
2585
2586                         DEBUG(DEBUG_NOTICE,
2587                               ("IP %s no longer configured, deleting it\n",
2588                                ctdb_addr_to_str(&ips->ips[i].addr)));
2589
2590                         pub = talloc_zero(mem_ctx, struct ctdb_addr_info_old);
2591                         CTDB_NO_MEMORY(ctdb, pub);
2592
2593                         pub->addr  = ips->ips[i].addr;
2594                         pub->mask  = 0;
2595                         pub->len   = 0;
2596
2597                         timeout = TAKEOVER_TIMEOUT();
2598
2599                         data.dsize = offsetof(struct ctdb_addr_info_old,
2600                                               iface) + pub->len;
2601                         data.dptr = (uint8_t *)pub;
2602
2603                         state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
2604                                                   CTDB_CONTROL_DEL_PUBLIC_IP,
2605                                                   0, data, async_data,
2606                                                   &timeout, NULL);
2607                         if (state == NULL) {
2608                                 DEBUG(DEBUG_ERR,
2609                                       (__location__
2610                                        " failed sending CTDB_CONTROL_DEL_PUBLIC_IP\n"));
2611                                 goto failed;
2612                         }
2613
2614                         ctdb_client_async_add(async_data, state);
2615                 }
2616         }
2617
2618         /* Compare IPs between node and file for IPs to be added */
2619         first_add = true;
2620         for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
2621                 for (i = 0; i < ips->num; i++) {
2622                         if (ctdb_same_ip(&vnn->public_address,
2623                                          &ips->ips[i].addr)) {
2624                                 /* IP already on node */
2625                                 break;
2626                         }
2627                 }
2628                 if (i == ips->num) {
2629                         /* Add IP ips->ips[i] */
2630                         struct ctdb_addr_info_old *pub;
2631                         const char *ifaces = NULL;
2632                         uint32_t len;
2633                         struct vnn_interface *iface = NULL;
2634
2635                         DEBUG(DEBUG_NOTICE,
2636                               ("New IP %s configured, adding it\n",
2637                                ctdb_addr_to_str(&vnn->public_address)));
2638                         if (first_add) {
2639                                 uint32_t pnn = ctdb_get_pnn(ctdb);
2640
2641                                 data.dsize = sizeof(pnn);
2642                                 data.dptr  = (uint8_t *)&pnn;
2643
2644                                 ret = ctdb_client_send_message(
2645                                         ctdb,
2646                                         CTDB_BROADCAST_CONNECTED,
2647                                         CTDB_SRVID_REBALANCE_NODE,
2648                                         data);
2649                                 if (ret != 0) {
2650                                         DEBUG(DEBUG_WARNING,
2651                                               ("Failed to send message to force node reallocation - IPs may be unbalanced\n"));
2652                                 }
2653
2654                                 first_add = false;
2655                         }
2656
2657                         ifaces = vnn->ifaces->iface->name;
2658                         iface = vnn->ifaces->next;
2659                         while (iface != NULL) {
2660                                 ifaces = talloc_asprintf(vnn, "%s,%s", ifaces,
2661                                                          iface->iface->name);
2662                                 iface = iface->next;
2663                         }
2664
2665                         len   = strlen(ifaces) + 1;
2666                         pub = talloc_zero_size(mem_ctx,
2667                                                offsetof(struct ctdb_addr_info_old, iface) + len);
2668                         CTDB_NO_MEMORY(ctdb, pub);
2669
2670                         pub->addr  = vnn->public_address;
2671                         pub->mask  = vnn->public_netmask_bits;
2672                         pub->len   = len;
2673                         memcpy(&pub->iface[0], ifaces, pub->len);
2674
2675                         timeout = TAKEOVER_TIMEOUT();
2676
2677                         data.dsize = offsetof(struct ctdb_addr_info_old,
2678                                               iface) + pub->len;
2679                         data.dptr = (uint8_t *)pub;
2680
2681                         state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
2682                                                   CTDB_CONTROL_ADD_PUBLIC_IP,
2683                                                   0, data, async_data,
2684                                                   &timeout, NULL);
2685                         if (state == NULL) {
2686                                 DEBUG(DEBUG_ERR,
2687                                       (__location__
2688                                        " failed sending CTDB_CONTROL_ADD_PUBLIC_IP\n"));
2689                                 goto failed;
2690                         }
2691
2692                         ctdb_client_async_add(async_data, state);
2693                 }
2694         }
2695
2696         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2697                 DEBUG(DEBUG_ERR,(__location__ " Add/delete IPs failed\n"));
2698                 goto failed;
2699         }
2700
2701         talloc_free(mem_ctx);
2702         return 0;
2703
2704 failed:
2705         talloc_free(mem_ctx);
2706         return -1;
2707 }
2708
2709 /* This control is sent to force the node to re-read the public addresses file
2710    and drop any addresses we should nnot longer host, and add new addresses
2711    that we are now able to host
2712 */
2713 int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control_old *c, bool *async_reply)
2714 {
2715         struct ctdb_reloadips_handle *h;
2716         pid_t parent = getpid();
2717
2718         if (ctdb->reload_ips != NULL) {
2719                 talloc_free(ctdb->reload_ips);
2720                 ctdb->reload_ips = NULL;
2721         }
2722
2723         h = talloc(ctdb, struct ctdb_reloadips_handle);
2724         CTDB_NO_MEMORY(ctdb, h);
2725         h->ctdb     = ctdb;
2726         h->c        = NULL;
2727         h->status   = -1;
2728
2729         if (pipe(h->fd) == -1) {
2730                 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
2731                 talloc_free(h);
2732                 return -1;
2733         }
2734
2735         h->child = ctdb_fork(ctdb);
2736         if (h->child == (pid_t)-1) {
2737                 DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
2738                 close(h->fd[0]);
2739                 close(h->fd[1]);
2740                 talloc_free(h);
2741                 return -1;
2742         }
2743
2744         /* child process */
2745         if (h->child == 0) {
2746                 signed char res = 0;
2747
2748                 close(h->fd[0]);
2749
2750                 prctl_set_comment("ctdb_reloadips");
2751                 if (switch_from_server_to_client(ctdb) != 0) {
2752                         DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
2753                         res = -1;
2754                 } else {
2755                         res = ctdb_reloadips_child(ctdb);
2756                         if (res != 0) {
2757                                 DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
2758                         }
2759                 }
2760
2761                 sys_write(h->fd[1], &res, 1);
2762                 ctdb_wait_for_process_to_exit(parent);
2763                 _exit(0);
2764         }
2765
2766         h->c             = talloc_steal(h, c);
2767
2768         close(h->fd[1]);
2769         set_close_on_exec(h->fd[0]);
2770
2771         talloc_set_destructor(h, ctdb_reloadips_destructor);
2772
2773
2774         h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ,
2775                                ctdb_reloadips_child_handler, (void *)h);
2776         tevent_fd_set_auto_close(h->fde);
2777
2778         tevent_add_timer(ctdb->ev, h, timeval_current_ofs(120, 0),
2779                          ctdb_reloadips_timeout_event, h);
2780
2781         /* we reply later */
2782         *async_reply = true;
2783         return 0;
2784 }