ctdb/server/ctdb_takeover.c

   1 /*
   2    ctdb ip takeover code
   3
   4    Copyright (C) Ronnie Sahlberg  2007
   5    Copyright (C) Andrew Tridgell  2007
   6    Copyright (C) Martin Schwenke  2011
   7
   8    This program is free software; you can redistribute it and/or modify
   9    it under the terms of the GNU General Public License as published by
  10    the Free Software Foundation; either version 3 of the License, or
  11    (at your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful,
  14    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16    GNU General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, see <http://www.gnu.org/licenses/>.
  20 */
  21 #include "replace.h"
  22 #include "system/network.h"
  23 #include "system/filesys.h"
  24 #include "system/time.h"
  25 #include "system/wait.h"
  26
  27 #include <talloc.h>
  28 #include <tevent.h>
  29
  30 #include "lib/util/dlinklist.h"
  31 #include "lib/util/debug.h"
  32 #include "lib/util/samba_util.h"
  33 #include "lib/util/sys_rw.h"
  34 #include "lib/util/util_process.h"
  35
  36 #include "protocol/protocol_util.h"
  37
  38 #include "ctdb_private.h"
  39 #include "ctdb_client.h"
  40
  41 #include "common/reqid.h"
  42 #include "common/system.h"
  43 #include "common/system_socket.h"
  44 #include "common/common.h"
  45 #include "common/logging.h"
  46
  47 #include "server/ctdb_config.h"
  48
  49 #include "server/ipalloc.h"
  50
  51 #define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
  52
  53 #define CTDB_ARP_INTERVAL 1
  54 #define CTDB_ARP_REPEAT   3
  55
  56 struct ctdb_interface {
  57         struct ctdb_interface *prev, *next;
  58         const char *name;
  59         bool link_up;
  60         uint32_t references;
  61 };
  62
  63 struct vnn_interface {
  64         struct vnn_interface *prev, *next;
  65         struct ctdb_interface *iface;
  66 };
  67
  68 /* state associated with a public ip address */
  69 struct ctdb_vnn {
  70         struct ctdb_vnn *prev, *next;
  71
  72         struct ctdb_interface *iface;
  73         struct vnn_interface *ifaces;
  74         ctdb_sock_addr public_address;
  75         uint8_t public_netmask_bits;
  76
  77         /*
  78          * The node number that is serving this public address - set
  79          * to CTDB_UNKNOWN_PNN if node is serving it
  80          */
  81         uint32_t pnn;
  82
  83         /* List of clients to tickle for this public address */
  84         struct ctdb_tcp_array *tcp_array;
  85
  86         /* whether we need to update the other nodes with changes to our list
  87            of connected clients */
  88         bool tcp_update_needed;
  89
  90         /* a context to hang sending gratious arp events off */
  91         TALLOC_CTX *takeover_ctx;
  92
  93         /* Set to true any time an update to this VNN is in flight.
  94            This helps to avoid races. */
  95         bool update_in_flight;
  96
  97         /* If CTDB_CONTROL_DEL_PUBLIC_IP is received for this IP
  98          * address then this flag is set.  It will be deleted in the
  99          * release IP callback. */
 100         bool delete_pending;
 101 };
 102
 103 static const char *iface_string(const struct ctdb_interface *iface)
 104 {
 105         return (iface != NULL ? iface->name : "__none__");
 106 }
 107
 108 static const char *ctdb_vnn_iface_string(const struct ctdb_vnn *vnn)
 109 {
 110         return iface_string(vnn->iface);
 111 }
 112
 113 static struct ctdb_interface *ctdb_find_iface(struct ctdb_context *ctdb,
 114                                               const char *iface);
 115
 116 static struct ctdb_interface *
 117 ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
 118 {
 119         struct ctdb_interface *i;
 120
 121         if (strlen(iface) > CTDB_IFACE_SIZE) {
 122                 DEBUG(DEBUG_ERR, ("Interface name too long \"%s\"\n", iface));
 123                 return NULL;
 124         }
 125
 126         /* Verify that we don't have an entry for this ip yet */
 127         i = ctdb_find_iface(ctdb, iface);
 128         if (i != NULL) {
 129                 return i;
 130         }
 131
 132         /* create a new structure for this interface */
 133         i = talloc_zero(ctdb, struct ctdb_interface);
 134         if (i == NULL) {
 135                 DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
 136                 return NULL;
 137         }
 138         i->name = talloc_strdup(i, iface);
 139         if (i->name == NULL) {
 140                 DEBUG(DEBUG_ERR, (__location__ " out of memory\n"));
 141                 talloc_free(i);
 142                 return NULL;
 143         }
 144
 145         i->link_up = true;
 146
 147         DLIST_ADD(ctdb->ifaces, i);
 148
 149         return i;
 150 }
 151
 152 static bool vnn_has_interface(struct ctdb_vnn *vnn,
 153                               const struct ctdb_interface *iface)
 154 {
 155         struct vnn_interface *i;
 156
 157         for (i = vnn->ifaces; i != NULL; i = i->next) {
 158                 if (iface == i->iface) {
 159                         return true;
 160                 }
 161         }
 162
 163         return false;
 164 }
 165
 166 /* If any interfaces now have no possible IPs then delete them.  This
 167  * implementation is naive (i.e. simple) rather than clever
 168  * (i.e. complex).  Given that this is run on delip and that operation
 169  * is rare, this doesn't need to be efficient - it needs to be
 170  * foolproof.  One alternative is reference counting, where the logic
 171  * is distributed and can, therefore, be broken in multiple places.
 172  * Another alternative is to build a red-black tree of interfaces that
 173  * can have addresses (by walking ctdb->vnn once) and then walking
 174  * ctdb->ifaces once and deleting those not in the tree.  Let's go to
 175  * one of those if the naive implementation causes problems...  :-)
 176  */
 177 static void ctdb_remove_orphaned_ifaces(struct ctdb_context *ctdb,
 178                                         struct ctdb_vnn *vnn)
 179 {
 180         struct ctdb_interface *i, *next;
 181
 182         /* For each interface, check if there's an IP using it. */
 183         for (i = ctdb->ifaces; i != NULL; i = next) {
 184                 struct ctdb_vnn *tv;
 185                 bool found;
 186                 next = i->next;
 187
 188                 /* Only consider interfaces named in the given VNN. */
 189                 if (!vnn_has_interface(vnn, i)) {
 190                         continue;
 191                 }
 192
 193                 /* Search for a vnn with this interface. */
 194                 found = false;
 195                 for (tv=ctdb->vnn; tv; tv=tv->next) {
 196                         if (vnn_has_interface(tv, i)) {
 197                                 found = true;
 198                                 break;
 199                         }
 200                 }
 201
 202                 if (!found) {
 203                         /* None of the VNNs are using this interface. */
 204                         DLIST_REMOVE(ctdb->ifaces, i);
 205                         talloc_free(i);
 206                 }
 207         }
 208 }
 209
 210
 211 static struct ctdb_interface *ctdb_find_iface(struct ctdb_context *ctdb,
 212                                               const char *iface)
 213 {
 214         struct ctdb_interface *i;
 215
 216         for (i=ctdb->ifaces;i;i=i->next) {
 217                 if (strcmp(i->name, iface) == 0) {
 218                         return i;
 219                 }
 220         }
 221
 222         return NULL;
 223 }
 224
 225 static struct ctdb_interface *ctdb_vnn_best_iface(struct ctdb_context *ctdb,
 226                                                   struct ctdb_vnn *vnn)
 227 {
 228         struct vnn_interface *i;
 229         struct ctdb_interface *cur = NULL;
 230         struct ctdb_interface *best = NULL;
 231
 232         for (i = vnn->ifaces; i != NULL; i = i->next) {
 233
 234                 cur = i->iface;
 235
 236                 if (!cur->link_up) {
 237                         continue;
 238                 }
 239
 240                 if (best == NULL) {
 241                         best = cur;
 242                         continue;
 243                 }
 244
 245                 if (cur->references < best->references) {
 246                         best = cur;
 247                         continue;
 248                 }
 249         }
 250
 251         return best;
 252 }
 253
 254 static int32_t ctdb_vnn_assign_iface(struct ctdb_context *ctdb,
 255                                      struct ctdb_vnn *vnn)
 256 {
 257         struct ctdb_interface *best = NULL;
 258
 259         if (vnn->iface) {
 260                 DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 261                                    "still assigned to iface '%s'\n",
 262                                    ctdb_addr_to_str(&vnn->public_address),
 263                                    ctdb_vnn_iface_string(vnn)));
 264                 return 0;
 265         }
 266
 267         best = ctdb_vnn_best_iface(ctdb, vnn);
 268         if (best == NULL) {
 269                 DEBUG(DEBUG_ERR, (__location__ " public address '%s' "
 270                                   "cannot assign to iface any iface\n",
 271                                   ctdb_addr_to_str(&vnn->public_address)));
 272                 return -1;
 273         }
 274
 275         vnn->iface = best;
 276         best->references++;
 277         vnn->pnn = ctdb->pnn;
 278
 279         DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 280                            "now assigned to iface '%s' refs[%d]\n",
 281                            ctdb_addr_to_str(&vnn->public_address),
 282                            ctdb_vnn_iface_string(vnn),
 283                            best->references));
 284         return 0;
 285 }
 286
 287 static void ctdb_vnn_unassign_iface(struct ctdb_context *ctdb,
 288                                     struct ctdb_vnn *vnn)
 289 {
 290         DEBUG(DEBUG_INFO, (__location__ " public address '%s' "
 291                            "now unassigned (old iface '%s' refs[%d])\n",
 292                            ctdb_addr_to_str(&vnn->public_address),
 293                            ctdb_vnn_iface_string(vnn),
 294                            vnn->iface?vnn->iface->references:0));
 295         if (vnn->iface) {
 296                 vnn->iface->references--;
 297         }
 298         vnn->iface = NULL;
 299         if (vnn->pnn == ctdb->pnn) {
 300                 vnn->pnn = CTDB_UNKNOWN_PNN;
 301         }
 302 }
 303
 304 static bool ctdb_vnn_available(struct ctdb_context *ctdb,
 305                                struct ctdb_vnn *vnn)
 306 {
 307         uint32_t flags;
 308         struct vnn_interface *i;
 309
 310         /* Nodes that are not RUNNING can not host IPs */
 311         if (ctdb->runstate != CTDB_RUNSTATE_RUNNING) {
 312                 return false;
 313         }
 314
 315         flags = ctdb->nodes[ctdb->pnn]->flags;
 316         if ((flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED)) != 0) {
 317                 return false;
 318         }
 319
 320         if (vnn->delete_pending) {
 321                 return false;
 322         }
 323
 324         if (vnn->iface && vnn->iface->link_up) {
 325                 return true;
 326         }
 327
 328         for (i = vnn->ifaces; i != NULL; i = i->next) {
 329                 if (i->iface->link_up) {
 330                         return true;
 331                 }
 332         }
 333
 334         return false;
 335 }
 336
 337 struct ctdb_takeover_arp {
 338         struct ctdb_context *ctdb;
 339         uint32_t count;
 340         ctdb_sock_addr addr;
 341         struct ctdb_tcp_array *tcparray;
 342         struct ctdb_vnn *vnn;
 343 };
 344
 345
 346 /*
 347   lists of tcp endpoints
 348  */
 349 struct ctdb_tcp_list {
 350         struct ctdb_tcp_list *prev, *next;
 351         struct ctdb_client *client;
 352         struct ctdb_connection connection;
 353 };
 354
 355 /*
 356   send a gratuitous arp
 357  */
 358 static void ctdb_control_send_arp(struct tevent_context *ev,
 359                                   struct tevent_timer *te,
 360                                   struct timeval t, void *private_data)
 361 {
 362         struct ctdb_takeover_arp *arp = talloc_get_type(private_data,
 363                                                         struct ctdb_takeover_arp);
 364         int ret;
 365         struct ctdb_tcp_array *tcparray;
 366         const char *iface;
 367
 368         /* IP address might have been released between sends */
 369         if (arp->vnn->iface == NULL) {
 370                 DBG_INFO("Cancelling ARP send for released IP %s\n",
 371                          ctdb_addr_to_str(&arp->vnn->public_address));
 372                 talloc_free(arp);
 373                 return;
 374         }
 375
 376         iface = ctdb_vnn_iface_string(arp->vnn);
 377         ret = ctdb_sys_send_arp(&arp->addr, iface);
 378         if (ret != 0) {
 379                 DBG_ERR("Failed to send ARP on interface %s: %s\n",
 380                         iface, strerror(ret));
 381         }
 382
 383         tcparray = arp->tcparray;
 384         if (tcparray) {
 385                 unsigned int i;
 386
 387                 for (i=0;i<tcparray->num;i++) {
 388                         struct ctdb_connection *tcon;
 389                         char buf[128];
 390
 391                         tcon = &tcparray->connections[i];
 392                         ret = ctdb_connection_to_buf(buf,
 393                                                      sizeof(buf),
 394                                                      tcon,
 395                                                      false,
 396                                                      " -> ");
 397                         if (ret != 0) {
 398                                 strlcpy(buf, "UNKNOWN", sizeof(buf));
 399                         }
 400                         D_INFO("Send TCP tickle ACK: %s\n", buf);
 401                         ret = ctdb_sys_send_tcp(
 402                                 &tcon->src,
 403                                 &tcon->dst,
 404                                 0, 0, 0);
 405                         if (ret != 0) {
 406                                 DBG_ERR("Failed to send TCP tickle ACK: %s\n",
 407                                         buf);
 408                         }
 409                 }
 410         }
 411
 412         arp->count++;
 413
 414         if (arp->count == CTDB_ARP_REPEAT) {
 415                 talloc_free(arp);
 416                 return;
 417         }
 418
 419         tevent_add_timer(arp->ctdb->ev, arp->vnn->takeover_ctx,
 420                          timeval_current_ofs(CTDB_ARP_INTERVAL, 100000),
 421                          ctdb_control_send_arp, arp);
 422 }
 423
 424 static int32_t ctdb_announce_vnn_iface(struct ctdb_context *ctdb,
 425                                        struct ctdb_vnn *vnn)
 426 {
 427         struct ctdb_takeover_arp *arp;
 428         struct ctdb_tcp_array *tcparray;
 429
 430         if (!vnn->takeover_ctx) {
 431                 vnn->takeover_ctx = talloc_new(vnn);
 432                 if (!vnn->takeover_ctx) {
 433                         return -1;
 434                 }
 435         }
 436
 437         arp = talloc_zero(vnn->takeover_ctx, struct ctdb_takeover_arp);
 438         if (!arp) {
 439                 return -1;
 440         }
 441
 442         arp->ctdb = ctdb;
 443         arp->addr = vnn->public_address;
 444         arp->vnn  = vnn;
 445
 446         tcparray = vnn->tcp_array;
 447         if (tcparray) {
 448                 /* add all of the known tcp connections for this IP to the
 449                    list of tcp connections to send tickle acks for */
 450                 arp->tcparray = talloc_steal(arp, tcparray);
 451
 452                 vnn->tcp_array = NULL;
 453                 vnn->tcp_update_needed = true;
 454         }
 455
 456         tevent_add_timer(arp->ctdb->ev, vnn->takeover_ctx,
 457                          timeval_zero(), ctdb_control_send_arp, arp);
 458
 459         return 0;
 460 }
 461
 462 struct ctdb_do_takeip_state {
 463         struct ctdb_req_control_old *c;
 464         struct ctdb_vnn *vnn;
 465 };
 466
 467 /*
 468   called when takeip event finishes
 469  */
 470 static void ctdb_do_takeip_callback(struct ctdb_context *ctdb, int status,
 471                                     void *private_data)
 472 {
 473         struct ctdb_do_takeip_state *state =
 474                 talloc_get_type(private_data, struct ctdb_do_takeip_state);
 475         int32_t ret;
 476         TDB_DATA data;
 477
 478         if (status != 0) {
 479                 if (status == -ETIMEDOUT) {
 480                         ctdb_ban_self(ctdb);
 481                 }
 482                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 483                                  ctdb_addr_to_str(&state->vnn->public_address),
 484                                  ctdb_vnn_iface_string(state->vnn)));
 485                 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
 486
 487                 talloc_free(state);
 488                 return;
 489         }
 490
 491         if (ctdb->do_checkpublicip) {
 492
 493         ret = ctdb_announce_vnn_iface(ctdb, state->vnn);
 494         if (ret != 0) {
 495                 ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
 496                 talloc_free(state);
 497                 return;
 498         }
 499
 500         }
 501
 502         data.dptr  = (uint8_t *)ctdb_addr_to_str(&state->vnn->public_address);
 503         data.dsize = strlen((char *)data.dptr) + 1;
 504         DEBUG(DEBUG_INFO,(__location__ " sending TAKE_IP for '%s'\n", data.dptr));
 505
 506         ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_TAKE_IP, data);
 507
 508
 509         /* the control succeeded */
 510         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 511         talloc_free(state);
 512         return;
 513 }
 514
 515 static int ctdb_takeip_destructor(struct ctdb_do_takeip_state *state)
 516 {
 517         state->vnn->update_in_flight = false;
 518         return 0;
 519 }
 520
 521 /*
 522   take over an ip address
 523  */
 524 static int32_t ctdb_do_takeip(struct ctdb_context *ctdb,
 525                               struct ctdb_req_control_old *c,
 526                               struct ctdb_vnn *vnn)
 527 {
 528         int ret;
 529         struct ctdb_do_takeip_state *state;
 530
 531         if (vnn->update_in_flight) {
 532                 DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u rejected "
 533                                     "update for this IP already in flight\n",
 534                                     ctdb_addr_to_str(&vnn->public_address),
 535                                     vnn->public_netmask_bits));
 536                 return -1;
 537         }
 538
 539         ret = ctdb_vnn_assign_iface(ctdb, vnn);
 540         if (ret != 0) {
 541                 DEBUG(DEBUG_ERR,("Takeover of IP %s/%u failed to "
 542                                  "assign a usable interface\n",
 543                                  ctdb_addr_to_str(&vnn->public_address),
 544                                  vnn->public_netmask_bits));
 545                 return -1;
 546         }
 547
 548         state = talloc(vnn, struct ctdb_do_takeip_state);
 549         CTDB_NO_MEMORY(ctdb, state);
 550
 551         state->c = NULL;
 552         state->vnn   = vnn;
 553
 554         vnn->update_in_flight = true;
 555         talloc_set_destructor(state, ctdb_takeip_destructor);
 556
 557         DEBUG(DEBUG_NOTICE,("Takeover of IP %s/%u on interface %s\n",
 558                             ctdb_addr_to_str(&vnn->public_address),
 559                             vnn->public_netmask_bits,
 560                             ctdb_vnn_iface_string(vnn)));
 561
 562         ret = ctdb_event_script_callback(ctdb,
 563                                          state,
 564                                          ctdb_do_takeip_callback,
 565                                          state,
 566                                          CTDB_EVENT_TAKE_IP,
 567                                          "%s %s %u",
 568                                          ctdb_vnn_iface_string(vnn),
 569                                          ctdb_addr_to_str(&vnn->public_address),
 570                                          vnn->public_netmask_bits);
 571
 572         if (ret != 0) {
 573                 DEBUG(DEBUG_ERR,(__location__ " Failed to takeover IP %s on interface %s\n",
 574                         ctdb_addr_to_str(&vnn->public_address),
 575                         ctdb_vnn_iface_string(vnn)));
 576                 talloc_free(state);
 577                 return -1;
 578         }
 579
 580         state->c = talloc_steal(ctdb, c);
 581         return 0;
 582 }
 583
 584 struct ctdb_do_updateip_state {
 585         struct ctdb_req_control_old *c;
 586         struct ctdb_interface *old;
 587         struct ctdb_vnn *vnn;
 588 };
 589
 590 /*
 591   called when updateip event finishes
 592  */
 593 static void ctdb_do_updateip_callback(struct ctdb_context *ctdb, int status,
 594                                       void *private_data)
 595 {
 596         struct ctdb_do_updateip_state *state =
 597                 talloc_get_type(private_data, struct ctdb_do_updateip_state);
 598
 599         if (status != 0) {
 600                 if (status == -ETIMEDOUT) {
 601                         ctdb_ban_self(ctdb);
 602                 }
 603                 DEBUG(DEBUG_ERR,
 604                       ("Failed update of IP %s from interface %s to %s\n",
 605                        ctdb_addr_to_str(&state->vnn->public_address),
 606                        iface_string(state->old),
 607                        ctdb_vnn_iface_string(state->vnn)));
 608
 609                 /*
 610                  * All we can do is reset the old interface
 611                  * and let the next run fix it
 612                  */
 613                 ctdb_vnn_unassign_iface(ctdb, state->vnn);
 614                 state->vnn->iface = state->old;
 615                 state->vnn->iface->references++;
 616
 617                 ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
 618                 talloc_free(state);
 619                 return;
 620         }
 621
 622         /* the control succeeded */
 623         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 624         talloc_free(state);
 625         return;
 626 }
 627
 628 static int ctdb_updateip_destructor(struct ctdb_do_updateip_state *state)
 629 {
 630         state->vnn->update_in_flight = false;
 631         return 0;
 632 }
 633
 634 /*
 635   update (move) an ip address
 636  */
 637 static int32_t ctdb_do_updateip(struct ctdb_context *ctdb,
 638                                 struct ctdb_req_control_old *c,
 639                                 struct ctdb_vnn *vnn)
 640 {
 641         int ret;
 642         struct ctdb_do_updateip_state *state;
 643         struct ctdb_interface *old = vnn->iface;
 644         const char *old_name = iface_string(old);
 645         const char *new_name;
 646
 647         if (vnn->update_in_flight) {
 648                 DEBUG(DEBUG_NOTICE,("Update of IP %s/%u rejected "
 649                                     "update for this IP already in flight\n",
 650                                     ctdb_addr_to_str(&vnn->public_address),
 651                                     vnn->public_netmask_bits));
 652                 return -1;
 653         }
 654
 655         ctdb_vnn_unassign_iface(ctdb, vnn);
 656         ret = ctdb_vnn_assign_iface(ctdb, vnn);
 657         if (ret != 0) {
 658                 DEBUG(DEBUG_ERR,("Update of IP %s/%u failed to "
 659                                  "assign a usable interface (old iface '%s')\n",
 660                                  ctdb_addr_to_str(&vnn->public_address),
 661                                  vnn->public_netmask_bits,
 662                                  old_name));
 663                 return -1;
 664         }
 665
 666         if (old == vnn->iface) {
 667                 /* A benign update from one interface onto itself.
 668                  * no need to run the eventscripts in this case, just return
 669                  * success.
 670                  */
 671                 ctdb_request_control_reply(ctdb, c, NULL, 0, NULL);
 672                 return 0;
 673         }
 674
 675         state = talloc(vnn, struct ctdb_do_updateip_state);
 676         CTDB_NO_MEMORY(ctdb, state);
 677
 678         state->c = NULL;
 679         state->old = old;
 680         state->vnn = vnn;
 681
 682         vnn->update_in_flight = true;
 683         talloc_set_destructor(state, ctdb_updateip_destructor);
 684
 685         new_name = ctdb_vnn_iface_string(vnn);
 686         DEBUG(DEBUG_NOTICE,("Update of IP %s/%u from "
 687                             "interface %s to %s\n",
 688                             ctdb_addr_to_str(&vnn->public_address),
 689                             vnn->public_netmask_bits,
 690                             old_name,
 691                             new_name));
 692
 693         ret = ctdb_event_script_callback(ctdb,
 694                                          state,
 695                                          ctdb_do_updateip_callback,
 696                                          state,
 697                                          CTDB_EVENT_UPDATE_IP,
 698                                          "%s %s %s %u",
 699                                          old_name,
 700                                          new_name,
 701                                          ctdb_addr_to_str(&vnn->public_address),
 702                                          vnn->public_netmask_bits);
 703         if (ret != 0) {
 704                 DEBUG(DEBUG_ERR,
 705                       ("Failed update IP %s from interface %s to %s\n",
 706                        ctdb_addr_to_str(&vnn->public_address),
 707                        old_name, new_name));
 708                 talloc_free(state);
 709                 return -1;
 710         }
 711
 712         state->c = talloc_steal(ctdb, c);
 713         return 0;
 714 }
 715
 716 /*
 717   Find the vnn of the node that has a public ip address
 718   returns -1 if the address is not known as a public address
 719  */
 720 static struct ctdb_vnn *find_public_ip_vnn(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
 721 {
 722         struct ctdb_vnn *vnn;
 723
 724         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
 725                 if (ctdb_same_ip(&vnn->public_address, addr)) {
 726                         return vnn;
 727                 }
 728         }
 729
 730         return NULL;
 731 }
 732
 733 /*
 734   take over an ip address
 735  */
 736 int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb,
 737                                  struct ctdb_req_control_old *c,
 738                                  TDB_DATA indata,
 739                                  bool *async_reply)
 740 {
 741         int ret;
 742         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 743         struct ctdb_vnn *vnn;
 744         bool have_ip = false;
 745         bool do_updateip = false;
 746         bool do_takeip = false;
 747         struct ctdb_interface *best_iface = NULL;
 748
 749         if (pip->pnn != ctdb->pnn) {
 750                 DEBUG(DEBUG_ERR,(__location__" takeoverip called for an ip '%s' "
 751                                  "with pnn %d, but we're node %d\n",
 752                                  ctdb_addr_to_str(&pip->addr),
 753                                  pip->pnn, ctdb->pnn));
 754                 return -1;
 755         }
 756
 757         /* update out vnn list */
 758         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 759         if (vnn == NULL) {
 760                 DEBUG(DEBUG_INFO,("takeoverip called for an ip '%s' that is not a public address\n",
 761                         ctdb_addr_to_str(&pip->addr)));
 762                 return 0;
 763         }
 764
 765         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 766                 have_ip = ctdb_sys_have_ip(&pip->addr);
 767         }
 768         best_iface = ctdb_vnn_best_iface(ctdb, vnn);
 769         if (best_iface == NULL) {
 770                 DEBUG(DEBUG_ERR,("takeoverip of IP %s/%u failed to find"
 771                                  "a usable interface (old %s, have_ip %d)\n",
 772                                  ctdb_addr_to_str(&vnn->public_address),
 773                                  vnn->public_netmask_bits,
 774                                  ctdb_vnn_iface_string(vnn),
 775                                  have_ip));
 776                 return -1;
 777         }
 778
 779         if (vnn->pnn != ctdb->pnn && have_ip && vnn->pnn != CTDB_UNKNOWN_PNN) {
 780                 DEBUG(DEBUG_CRIT,(__location__ " takeoverip of IP %s is known to the kernel, "
 781                                   "and we have it on iface[%s], but it was assigned to node %d"
 782                                   "and we are node %d, banning ourself\n",
 783                                  ctdb_addr_to_str(&vnn->public_address),
 784                                  ctdb_vnn_iface_string(vnn), vnn->pnn, ctdb->pnn));
 785                 ctdb_ban_self(ctdb);
 786                 return -1;
 787         }
 788
 789         if (vnn->pnn == CTDB_UNKNOWN_PNN && have_ip) {
 790                 /* This will cause connections to be reset and
 791                  * reestablished.  However, this is a very unusual
 792                  * situation and doing this will completely repair the
 793                  * inconsistency in the VNN.
 794                  */
 795                 DEBUG(DEBUG_WARNING,
 796                       (__location__
 797                        " Doing updateip for IP %s already on an interface\n",
 798                        ctdb_addr_to_str(&vnn->public_address)));
 799                 do_updateip = true;
 800         }
 801
 802         if (vnn->iface) {
 803                 if (vnn->iface != best_iface) {
 804                         if (!vnn->iface->link_up) {
 805                                 do_updateip = true;
 806                         } else if (vnn->iface->references > (best_iface->references + 1)) {
 807                                 /* only move when the rebalance gains something */
 808                                         do_updateip = true;
 809                         }
 810                 }
 811         }
 812
 813         if (!have_ip) {
 814                 if (do_updateip) {
 815                         ctdb_vnn_unassign_iface(ctdb, vnn);
 816                         do_updateip = false;
 817                 }
 818                 do_takeip = true;
 819         }
 820
 821         if (do_takeip) {
 822                 ret = ctdb_do_takeip(ctdb, c, vnn);
 823                 if (ret != 0) {
 824                         return -1;
 825                 }
 826         } else if (do_updateip) {
 827                 ret = ctdb_do_updateip(ctdb, c, vnn);
 828                 if (ret != 0) {
 829                         return -1;
 830                 }
 831         } else {
 832                 /*
 833                  * The interface is up and the kernel known the ip
 834                  * => do nothing
 835                  */
 836                 DEBUG(DEBUG_INFO,("Redundant takeover of IP %s/%u on interface %s (ip already held)\n",
 837                         ctdb_addr_to_str(&pip->addr),
 838                         vnn->public_netmask_bits,
 839                         ctdb_vnn_iface_string(vnn)));
 840                 return 0;
 841         }
 842
 843         /* tell ctdb_control.c that we will be replying asynchronously */
 844         *async_reply = true;
 845
 846         return 0;
 847 }
 848
 849 static void do_delete_ip(struct ctdb_context *ctdb, struct ctdb_vnn *vnn)
 850 {
 851         DLIST_REMOVE(ctdb->vnn, vnn);
 852         ctdb_vnn_unassign_iface(ctdb, vnn);
 853         ctdb_remove_orphaned_ifaces(ctdb, vnn);
 854         talloc_free(vnn);
 855 }
 856
 857 static struct ctdb_vnn *release_ip_post(struct ctdb_context *ctdb,
 858                                         struct ctdb_vnn *vnn,
 859                                         ctdb_sock_addr *addr)
 860 {
 861         TDB_DATA data;
 862
 863         /* Send a message to all clients of this node telling them
 864          * that the cluster has been reconfigured and they should
 865          * close any connections on this IP address
 866          */
 867         data.dptr = (uint8_t *)ctdb_addr_to_str(addr);
 868         data.dsize = strlen((char *)data.dptr)+1;
 869         DEBUG(DEBUG_INFO, ("Sending RELEASE_IP message for %s\n", data.dptr));
 870         ctdb_daemon_send_message(ctdb, ctdb->pnn, CTDB_SRVID_RELEASE_IP, data);
 871
 872         ctdb_vnn_unassign_iface(ctdb, vnn);
 873
 874         /* Process the IP if it has been marked for deletion */
 875         if (vnn->delete_pending) {
 876                 do_delete_ip(ctdb, vnn);
 877                 return NULL;
 878         }
 879
 880         return vnn;
 881 }
 882
 883 struct release_ip_callback_state {
 884         struct ctdb_req_control_old *c;
 885         ctdb_sock_addr *addr;
 886         struct ctdb_vnn *vnn;
 887         uint32_t target_pnn;
 888 };
 889
 890 /*
 891   called when releaseip event finishes
 892  */
 893 static void release_ip_callback(struct ctdb_context *ctdb, int status,
 894                                 void *private_data)
 895 {
 896         struct release_ip_callback_state *state =
 897                 talloc_get_type(private_data, struct release_ip_callback_state);
 898
 899         if (status == -ETIMEDOUT) {
 900                 ctdb_ban_self(ctdb);
 901         }
 902
 903         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 904                 if  (ctdb_sys_have_ip(state->addr)) {
 905                         DEBUG(DEBUG_ERR,
 906                               ("IP %s still hosted during release IP callback, failing\n",
 907                                ctdb_addr_to_str(state->addr)));
 908                         ctdb_request_control_reply(ctdb, state->c,
 909                                                    NULL, -1, NULL);
 910                         talloc_free(state);
 911                         return;
 912                 }
 913         }
 914
 915         state->vnn->pnn = state->target_pnn;
 916         state->vnn = release_ip_post(ctdb, state->vnn, state->addr);
 917
 918         /* the control succeeded */
 919         ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
 920         talloc_free(state);
 921 }
 922
 923 static int ctdb_releaseip_destructor(struct release_ip_callback_state *state)
 924 {
 925         if (state->vnn != NULL) {
 926                 state->vnn->update_in_flight = false;
 927         }
 928         return 0;
 929 }
 930
 931 /*
 932   release an ip address
 933  */
 934 int32_t ctdb_control_release_ip(struct ctdb_context *ctdb,
 935                                 struct ctdb_req_control_old *c,
 936                                 TDB_DATA indata,
 937                                 bool *async_reply)
 938 {
 939         int ret;
 940         struct release_ip_callback_state *state;
 941         struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
 942         struct ctdb_vnn *vnn;
 943         const char *iface;
 944
 945         /* update our vnn list */
 946         vnn = find_public_ip_vnn(ctdb, &pip->addr);
 947         if (vnn == NULL) {
 948                 DEBUG(DEBUG_INFO,("releaseip called for an ip '%s' that is not a public address\n",
 949                         ctdb_addr_to_str(&pip->addr)));
 950                 return 0;
 951         }
 952
 953         /* stop any previous arps */
 954         talloc_free(vnn->takeover_ctx);
 955         vnn->takeover_ctx = NULL;
 956
 957         /* RELEASE_IP controls are sent to all nodes that should not
 958          * be hosting a particular IP.  This serves 2 purposes.  The
 959          * first is to help resolve any inconsistencies.  If a node
 960          * does unexpectedly host an IP then it will be released.  The
 961          * 2nd is to use a "redundant release" to tell non-takeover
 962          * nodes where an IP is moving to.  This is how "ctdb ip" can
 963          * report the (likely) location of an IP by only asking the
 964          * local node.  Redundant releases need to update the PNN but
 965          * are otherwise ignored.
 966          */
 967         if (ctdb_config.failover_disabled == 0 && ctdb->do_checkpublicip) {
 968                 if (!ctdb_sys_have_ip(&pip->addr)) {
 969                         DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u on interface %s (ip not held)\n",
 970                                 ctdb_addr_to_str(&pip->addr),
 971                                 vnn->public_netmask_bits,
 972                                 ctdb_vnn_iface_string(vnn)));
 973                         vnn->pnn = pip->pnn;
 974                         ctdb_vnn_unassign_iface(ctdb, vnn);
 975                         return 0;
 976                 }
 977         } else {
 978                 if (vnn->iface == NULL) {
 979                         DEBUG(DEBUG_DEBUG,("Redundant release of IP %s/%u (ip not held)\n",
 980                                            ctdb_addr_to_str(&pip->addr),
 981                                            vnn->public_netmask_bits));
 982                         vnn->pnn = pip->pnn;
 983                         return 0;
 984                 }
 985         }
 986
 987         /* There is a potential race between take_ip and us because we
 988          * update the VNN via a callback that run when the
 989          * eventscripts have been run.  Avoid the race by allowing one
 990          * update to be in flight at a time.
 991          */
 992         if (vnn->update_in_flight) {
 993                 DEBUG(DEBUG_NOTICE,("Release of IP %s/%u rejected "
 994                                     "update for this IP already in flight\n",
 995                                     ctdb_addr_to_str(&vnn->public_address),
 996                                     vnn->public_netmask_bits));
 997                 return -1;
 998         }
 999
1000         iface = ctdb_vnn_iface_string(vnn);
1001
1002         DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s  node:%d\n",
1003                 ctdb_addr_to_str(&pip->addr),
1004                 vnn->public_netmask_bits,
1005                 iface,
1006                 pip->pnn));
1007
1008         state = talloc(ctdb, struct release_ip_callback_state);
1009         if (state == NULL) {
1010                 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1011                                __FILE__, __LINE__);
1012                 return -1;
1013         }
1014
1015         state->c = NULL;
1016         state->addr = talloc(state, ctdb_sock_addr);
1017         if (state->addr == NULL) {
1018                 ctdb_set_error(ctdb, "Out of memory at %s:%d",
1019                                __FILE__, __LINE__);
1020                 talloc_free(state);
1021                 return -1;
1022         }
1023         *state->addr = pip->addr;
1024         state->target_pnn = pip->pnn;
1025         state->vnn   = vnn;
1026
1027         vnn->update_in_flight = true;
1028         talloc_set_destructor(state, ctdb_releaseip_destructor);
1029
1030         ret = ctdb_event_script_callback(ctdb,
1031                                          state, release_ip_callback, state,
1032                                          CTDB_EVENT_RELEASE_IP,
1033                                          "%s %s %u",
1034                                          iface,
1035                                          ctdb_addr_to_str(&pip->addr),
1036                                          vnn->public_netmask_bits);
1037         if (ret != 0) {
1038                 DEBUG(DEBUG_ERR,(__location__ " Failed to release IP %s on interface %s\n",
1039                         ctdb_addr_to_str(&pip->addr),
1040                         ctdb_vnn_iface_string(vnn)));
1041                 talloc_free(state);
1042                 return -1;
1043         }
1044
1045         /* tell the control that we will be reply asynchronously */
1046         *async_reply = true;
1047         state->c = talloc_steal(state, c);
1048         return 0;
1049 }
1050
1051 static int ctdb_add_public_address(struct ctdb_context *ctdb,
1052                                    ctdb_sock_addr *addr,
1053                                    unsigned mask, const char *ifaces,
1054                                    bool check_address)
1055 {
1056         struct ctdb_vnn      *vnn;
1057         char *tmp;
1058         const char *iface;
1059
1060         /* Verify that we don't have an entry for this IP yet */
1061         for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
1062                 if (ctdb_same_sockaddr(addr, &vnn->public_address)) {
1063                         D_ERR("Duplicate public IP address '%s'\n",
1064                               ctdb_addr_to_str(addr));
1065                         return -1;
1066                 }
1067         }
1068
1069         /* Create a new VNN structure for this IP address */
1070         vnn = talloc_zero(ctdb, struct ctdb_vnn);
1071         if (vnn == NULL) {
1072                 DBG_ERR("Memory allocation error\n");
1073                 return -1;
1074         }
1075         tmp = talloc_strdup(vnn, ifaces);
1076         if (tmp == NULL) {
1077                 DBG_ERR("Memory allocation error\n");
1078                 talloc_free(vnn);
1079                 return -1;
1080         }
1081         for (iface = strtok(tmp, ","); iface; iface = strtok(NULL, ",")) {
1082                 struct vnn_interface *vnn_iface;
1083                 struct ctdb_interface *i;
1084
1085                 if (!ctdb_sys_check_iface_exists(iface)) {
1086                         D_ERR("Unknown interface %s for public address %s\n",
1087                               iface,
1088                               ctdb_addr_to_str(addr));
1089                         talloc_free(vnn);
1090                         return -1;
1091                 }
1092
1093                 i = ctdb_add_local_iface(ctdb, iface);
1094                 if (i == NULL) {
1095                         D_ERR("Failed to add interface '%s' "
1096                               "for public address %s\n",
1097                               iface,
1098                               ctdb_addr_to_str(addr));
1099                         talloc_free(vnn);
1100                         return -1;
1101                 }
1102
1103                 vnn_iface = talloc_zero(vnn, struct vnn_interface);
1104                 if (vnn_iface == NULL) {
1105                         DBG_ERR("Memory allocation error\n");
1106                         talloc_free(vnn);
1107                         return -1;
1108                 }
1109
1110                 vnn_iface->iface = i;
1111                 DLIST_ADD_END(vnn->ifaces, vnn_iface);
1112         }
1113         talloc_free(tmp);
1114         vnn->public_address      = *addr;
1115         vnn->public_netmask_bits = mask;
1116         vnn->pnn                 = -1;
1117
1118         DLIST_ADD(ctdb->vnn, vnn);
1119
1120         return 0;
1121 }
1122
1123 /*
1124   setup the public address lists from a file
1125 */
1126 int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses)
1127 {
1128         bool ok;
1129         char **lines;
1130         int nlines;
1131         int i;
1132
1133         /* If no public addresses file given then try the default */
1134         if (ctdb->public_addresses_file == NULL) {
1135                 const char *b = getenv("CTDB_BASE");
1136                 if (b == NULL) {
1137                         DBG_ERR("CTDB_BASE not set\n");
1138                         return -1;
1139                 }
1140                 ctdb->public_addresses_file = talloc_asprintf(
1141                                         ctdb, "%s/%s", b, "public_addresses");
1142                 if (ctdb->public_addresses_file == NULL) {
1143                         DBG_ERR("Out of memory\n");
1144                         return -1;
1145                 }
1146         }
1147
1148         /* If the file doesn't exist then warn and do nothing */
1149         ok = file_exist(ctdb->public_addresses_file);
1150         if (!ok) {
1151                 D_WARNING("Not loading public addresses, no file %s\n",
1152                           ctdb->public_addresses_file);
1153                 return 0;
1154         }
1155
1156         lines = file_lines_load(ctdb->public_addresses_file, &nlines, 0, ctdb);
1157         if (lines == NULL) {
1158                 ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", ctdb->public_addresses_file);
1159                 return -1;
1160         }
1161         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
1162                 nlines--;
1163         }
1164
1165         for (i=0;i<nlines;i++) {
1166                 unsigned mask;
1167                 ctdb_sock_addr addr;
1168                 const char *addrstr;
1169                 const char *ifaces;
1170                 char *tok, *line;
1171                 int ret;
1172
1173                 line = lines[i];
1174                 while ((*line == ' ') || (*line == '\t')) {
1175                         line++;
1176                 }
1177                 if (*line == '#') {
1178                         continue;
1179                 }
1180                 if (strcmp(line, "") == 0) {
1181                         continue;
1182                 }
1183                 tok = strtok(line, " \t");
1184                 addrstr = tok;
1185
1186                 tok = strtok(NULL, " \t");
1187                 if (tok == NULL) {
1188                         D_ERR("No interface specified at line %u "
1189                               "of public addresses file\n", i+1);
1190                         talloc_free(lines);
1191                         return -1;
1192                 }
1193                 ifaces = tok;
1194
1195                 if (addrstr == NULL) {
1196                         D_ERR("Badly formed line %u in public address list\n",
1197                               i+1);
1198                         talloc_free(lines);
1199                         return -1;
1200                 }
1201
1202                 ret = ctdb_sock_addr_mask_from_string(addrstr, &addr, &mask);
1203                 if (ret != 0) {
1204                         D_ERR("Badly formed line %u in public address list\n",
1205                               i+1);
1206                         talloc_free(lines);
1207                         return -1;
1208                 }
1209
1210                 if (ctdb_add_public_address(ctdb, &addr, mask, ifaces, check_addresses)) {
1211                         DEBUG(DEBUG_CRIT,("Failed to add line %u to the public address list\n", i+1));
1212                         talloc_free(lines);
1213                         return -1;
1214                 }
1215         }
1216
1217
1218         D_NOTICE("Loaded public addresses from %s\n",
1219                  ctdb->public_addresses_file);
1220
1221         talloc_free(lines);
1222         return 0;
1223 }
1224
1225 /*
1226   destroy a ctdb_tcp_list structure
1227  */
1228 static int ctdb_tcp_list_destructor(struct ctdb_tcp_list *tcp)
1229 {
1230         struct ctdb_client *client = tcp->client;
1231         struct ctdb_connection *conn = &tcp->connection;
1232         char conn_str[132] = { 0, };
1233         int ret;
1234
1235         ret = ctdb_connection_to_buf(conn_str,
1236                                      sizeof(conn_str),
1237                                      conn,
1238                                      false,
1239                                      " -> ");
1240         if (ret != 0) {
1241                 strlcpy(conn_str, "UNKNOWN", sizeof(conn_str));
1242         }
1243
1244         D_DEBUG("removing client TCP connection %s "
1245                 "(client_id %u pid %d)\n",
1246                 conn_str, client->client_id, client->pid);
1247
1248         DLIST_REMOVE(client->tcp_list, tcp);
1249
1250         /*
1251          * We don't call ctdb_remove_connection(vnn, conn) here
1252          * as we want the caller to decide if it's called
1253          * directly (local only) or indirectly via a
1254          * CTDB_CONTROL_TCP_REMOVE broadcast
1255          */
1256
1257         return 0;
1258 }
1259
1260 /*
1261   called by a client to inform us of a TCP connection that it is managing
1262   that should tickled with an ACK when IP takeover is done
1263  */
1264 int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id,
1265                                 TDB_DATA indata)
1266 {
1267         struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
1268         struct ctdb_connection *tcp_sock = NULL;
1269         struct ctdb_tcp_list *tcp;
1270         struct ctdb_connection t;
1271         int ret;
1272         TDB_DATA data;
1273         struct ctdb_vnn *vnn;
1274         ctdb_sock_addr src_addr;
1275         ctdb_sock_addr dst_addr;
1276
1277         /* If we don't have public IPs, tickles are useless */
1278         if (ctdb->vnn == NULL) {
1279                 return 0;
1280         }
1281
1282         tcp_sock = (struct ctdb_connection *)indata.dptr;
1283
1284         ctdb_canonicalize_ip_inplace(&tcp_sock->src);
1285         src_addr = tcp_sock->src;
1286
1287         ctdb_canonicalize_ip_inplace(&tcp_sock->dst);
1288         dst_addr = tcp_sock->dst;
1289
1290         vnn = find_public_ip_vnn(ctdb, &dst_addr);
1291         if (vnn == NULL) {
1292                 char *src_addr_str = NULL;
1293                 char *dst_addr_str = NULL;
1294
1295                 switch (dst_addr.sa.sa_family) {
1296                 case AF_INET:
1297                         if (ntohl(dst_addr.ip.sin_addr.s_addr) == INADDR_LOOPBACK) {
1298                                 /* ignore ... */
1299                                 return 0;
1300                         }
1301                         break;
1302                 case AF_INET6:
1303                         break;
1304                 default:
1305                         DEBUG(DEBUG_ERR,(__location__ " Unknown family type %d\n",
1306                               dst_addr.sa.sa_family));
1307                         return 0;
1308                 }
1309
1310                 src_addr_str = ctdb_sock_addr_to_string(client, &src_addr, false);
1311                 dst_addr_str = ctdb_sock_addr_to_string(client, &dst_addr, false);
1312                 DEBUG(DEBUG_ERR,(
1313                       "Could not register TCP connection from "
1314                       "%s to %s (not a public address) (port %u) "
1315                       "(client_id %u pid %u).\n",
1316                       src_addr_str,
1317                       dst_addr_str,
1318                       ctdb_sock_addr_port(&dst_addr),
1319                       client_id, client->pid));
1320                 TALLOC_FREE(src_addr_str);
1321                 TALLOC_FREE(dst_addr_str);
1322                 return 0;
1323         }
1324
1325         if (vnn->pnn != ctdb->pnn) {
1326                 DEBUG(DEBUG_ERR,("Attempt to register tcp client for IP %s we don't hold - failing (client_id %u pid %u)\n",
1327                         ctdb_addr_to_str(&dst_addr),
1328                         client_id, client->pid));
1329                 /* failing this call will tell smbd to die */
1330                 return -1;
1331         }
1332
1333         tcp = talloc(client, struct ctdb_tcp_list);
1334         CTDB_NO_MEMORY(ctdb, tcp);
1335         tcp->client = client;
1336
1337         tcp->connection.src = tcp_sock->src;
1338         tcp->connection.dst = tcp_sock->dst;
1339
1340         DLIST_ADD(client->tcp_list, tcp);
1341         talloc_set_destructor(tcp, ctdb_tcp_list_destructor);
1342
1343         t.src = tcp_sock->src;
1344         t.dst = tcp_sock->dst;
1345
1346         data.dptr = (uint8_t *)&t;
1347         data.dsize = sizeof(t);
1348
1349         switch (dst_addr.sa.sa_family) {
1350         case AF_INET:
1351                 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
1352                         (unsigned)ntohs(tcp_sock->dst.ip.sin_port),
1353                         ctdb_addr_to_str(&tcp_sock->src),
1354                         (unsigned)ntohs(tcp_sock->src.ip.sin_port), client_id, client->pid));
1355                 break;
1356         case AF_INET6:
1357                 DEBUG(DEBUG_INFO,("registered tcp client for %u->%s:%u (client_id %u pid %u)\n",
1358                         (unsigned)ntohs(tcp_sock->dst.ip6.sin6_port),
1359                         ctdb_addr_to_str(&tcp_sock->src),
1360                         (unsigned)ntohs(tcp_sock->src.ip6.sin6_port), client_id, client->pid));
1361                 break;
1362         default:
1363                 DEBUG(DEBUG_ERR,(__location__ " Unknown family %d\n",
1364                       dst_addr.sa.sa_family));
1365         }
1366
1367
1368         /* tell all nodes about this tcp connection */
1369         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0,
1370                                        CTDB_CONTROL_TCP_ADD,
1371                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
1372         if (ret != 0) {
1373                 DEBUG(DEBUG_ERR,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
1374                 return -1;
1375         }
1376
1377         return 0;
1378 }
1379
1380 static bool ctdb_client_remove_tcp(struct ctdb_client *client,
1381                                    const struct ctdb_connection *conn)
1382 {
1383         struct ctdb_tcp_list *tcp = NULL;
1384         struct ctdb_tcp_list *tcp_next = NULL;
1385         bool found = false;
1386
1387         for (tcp = client->tcp_list; tcp != NULL; tcp = tcp_next) {
1388                 bool same;
1389
1390                 tcp_next = tcp->next;
1391
1392                 same = ctdb_connection_same(conn, &tcp->connection);
1393                 if (!same) {
1394                         continue;
1395                 }
1396
1397                 TALLOC_FREE(tcp);
1398                 found = true;
1399         }
1400
1401         return found;
1402 }
1403
1404 /*
1405   called by a client to inform us of a TCP connection that was disconnected
1406  */
1407 int32_t ctdb_control_tcp_client_disconnected(struct ctdb_context *ctdb,
1408                                              uint32_t client_id,
1409                                              TDB_DATA indata)
1410 {
1411         struct ctdb_client *client = reqid_find(ctdb->idr, client_id, struct ctdb_client);
1412         struct ctdb_connection *tcp_sock = NULL;
1413         int ret;
1414         TDB_DATA data;
1415         char conn_str[132] = { 0, };
1416         bool found = false;
1417
1418         tcp_sock = (struct ctdb_connection *)indata.dptr;
1419
1420         ctdb_canonicalize_ip_inplace(&tcp_sock->src);
1421         ctdb_canonicalize_ip_inplace(&tcp_sock->dst);
1422
1423         ret = ctdb_connection_to_buf(conn_str,
1424                                      sizeof(conn_str),
1425                                      tcp_sock,
1426                                      false,
1427                                      " -> ");
1428         if (ret != 0) {
1429                 strlcpy(conn_str, "UNKNOWN", sizeof(conn_str));
1430         }
1431
1432         found = ctdb_client_remove_tcp(client, tcp_sock);
1433         if (!found) {
1434                 DBG_DEBUG("TCP connection %s not found "
1435                           "(client_id %u pid %u).\n",
1436                           conn_str, client_id, client->pid);
1437                 return 0;
1438         }
1439
1440         D_INFO("deregistered TCP connection %s "
1441                "(client_id %u pid %u)\n",
1442                conn_str, client_id, client->pid);
1443
1444         data.dptr = (uint8_t *)tcp_sock;
1445         data.dsize = sizeof(*tcp_sock);
1446
1447         /* tell all nodes about this tcp connection is gone */
1448         ret = ctdb_daemon_send_control(ctdb,
1449                                        CTDB_BROADCAST_CONNECTED,
1450                                        0,
1451                                        CTDB_CONTROL_TCP_REMOVE,
1452                                        0,
1453                                        CTDB_CTRL_FLAG_NOREPLY,
1454                                        data,
1455                                        NULL,
1456                                        NULL);
1457         if (ret != 0) {
1458                 DBG_ERR("Failed to send CTDB_CONTROL_TCP_REMOVE: %s\n",
1459                         conn_str);
1460                 return -1;
1461         }
1462
1463         return 0;
1464 }
1465
1466 /*
1467   find a tcp address on a list
1468  */
1469 static struct ctdb_connection *ctdb_tcp_find(struct ctdb_tcp_array *array,
1470                                            struct ctdb_connection *tcp)
1471 {
1472         unsigned int i;
1473
1474         if (array == NULL) {
1475                 return NULL;
1476         }
1477
1478         for (i=0;i<array->num;i++) {
1479                 if (ctdb_same_sockaddr(&array->connections[i].src, &tcp->src) &&
1480                     ctdb_same_sockaddr(&array->connections[i].dst, &tcp->dst)) {
1481                         return &array->connections[i];
1482                 }
1483         }
1484         return NULL;
1485 }
1486
1487
1488
1489 /*
1490   called by a daemon to inform us of a TCP connection that one of its
1491   clients managing that should tickled with an ACK when IP takeover is
1492   done
1493  */
1494 int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed)
1495 {
1496         struct ctdb_connection *p = (struct ctdb_connection *)indata.dptr;
1497         struct ctdb_tcp_array *tcparray;
1498         struct ctdb_connection tcp;
1499         struct ctdb_vnn *vnn;
1500
1501         /* If we don't have public IPs, tickles are useless */
1502         if (ctdb->vnn == NULL) {
1503                 return 0;
1504         }
1505
1506         vnn = find_public_ip_vnn(ctdb, &p->dst);
1507         if (vnn == NULL) {
1508                 DEBUG(DEBUG_INFO,(__location__ " got TCP_ADD control for an address which is not a public address '%s'\n",
1509                         ctdb_addr_to_str(&p->dst)));
1510
1511                 return -1;
1512         }
1513
1514
1515         tcparray = vnn->tcp_array;
1516
1517         /* If this is the first tickle */
1518         if (tcparray == NULL) {
1519                 tcparray = talloc(vnn, struct ctdb_tcp_array);
1520                 CTDB_NO_MEMORY(ctdb, tcparray);
1521                 vnn->tcp_array = tcparray;
1522
1523                 tcparray->num = 0;
1524                 tcparray->connections = talloc_size(tcparray, sizeof(struct ctdb_connection));
1525                 CTDB_NO_MEMORY(ctdb, tcparray->connections);
1526
1527                 tcparray->connections[tcparray->num].src = p->src;
1528                 tcparray->connections[tcparray->num].dst = p->dst;
1529                 tcparray->num++;
1530
1531                 if (tcp_update_needed) {
1532                         vnn->tcp_update_needed = true;
1533                 }
1534                 return 0;
1535         }
1536
1537
1538         /* Do we already have this tickle ?*/
1539         tcp.src = p->src;
1540         tcp.dst = p->dst;
1541         if (ctdb_tcp_find(tcparray, &tcp) != NULL) {
1542                 DEBUG(DEBUG_DEBUG,("Already had tickle info for %s:%u for vnn:%u\n",
1543                         ctdb_addr_to_str(&tcp.dst),
1544                         ntohs(tcp.dst.ip.sin_port),
1545                         vnn->pnn));
1546                 return 0;
1547         }
1548
1549         /* A new tickle, we must add it to the array */
1550         tcparray->connections = talloc_realloc(tcparray, tcparray->connections,
1551                                         struct ctdb_connection,
1552                                         tcparray->num+1);
1553         CTDB_NO_MEMORY(ctdb, tcparray->connections);
1554
1555         tcparray->connections[tcparray->num].src = p->src;
1556         tcparray->connections[tcparray->num].dst = p->dst;
1557         tcparray->num++;
1558
1559         DEBUG(DEBUG_INFO,("Added tickle info for %s:%u from vnn %u\n",
1560                 ctdb_addr_to_str(&tcp.dst),
1561                 ntohs(tcp.dst.ip.sin_port),
1562                 vnn->pnn));
1563
1564         if (tcp_update_needed) {
1565                 vnn->tcp_update_needed = true;
1566         }
1567
1568         return 0;
1569 }
1570
1571
1572 static void ctdb_remove_connection(struct ctdb_vnn *vnn, struct ctdb_connection *conn)
1573 {
1574         struct ctdb_connection *tcpp;
1575
1576         if (vnn == NULL) {
1577                 return;
1578         }
1579
1580         /* if the array is empty we can't remove it
1581            and we don't need to do anything
1582          */
1583         if (vnn->tcp_array == NULL) {
1584                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesn't exist (array is empty) %s:%u\n",
1585                         ctdb_addr_to_str(&conn->dst),
1586                         ntohs(conn->dst.ip.sin_port)));
1587                 return;
1588         }
1589
1590
1591         /* See if we know this connection
1592            if we don't know this connection  then we don't need to do anything
1593          */
1594         tcpp = ctdb_tcp_find(vnn->tcp_array, conn);
1595         if (tcpp == NULL) {
1596                 DEBUG(DEBUG_INFO,("Trying to remove tickle that doesn't exist %s:%u\n",
1597                         ctdb_addr_to_str(&conn->dst),
1598                         ntohs(conn->dst.ip.sin_port)));
1599                 return;
1600         }
1601
1602
1603         /* We need to remove this entry from the array.
1604            Instead of allocating a new array and copying data to it
1605            we cheat and just copy the last entry in the existing array
1606            to the entry that is to be removed and just shring the
1607            ->num field
1608          */
1609         *tcpp = vnn->tcp_array->connections[vnn->tcp_array->num - 1];
1610         vnn->tcp_array->num--;
1611
1612         /* If we deleted the last entry we also need to remove the entire array
1613          */
1614         if (vnn->tcp_array->num == 0) {
1615                 talloc_free(vnn->tcp_array);
1616                 vnn->tcp_array = NULL;
1617         }
1618
1619         vnn->tcp_update_needed = true;
1620
1621         DEBUG(DEBUG_INFO,("Removed tickle info for %s:%u\n",
1622                 ctdb_addr_to_str(&conn->src),
1623                 ntohs(conn->src.ip.sin_port)));
1624 }
1625
1626
1627 /*
1628   called by a daemon to inform us of a TCP connection that one of its
1629   clients used are no longer needed in the tickle database
1630  */
1631 int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
1632 {
1633         struct ctdb_vnn *vnn;
1634         struct ctdb_connection *conn = (struct ctdb_connection *)indata.dptr;
1635
1636         /* If we don't have public IPs, tickles are useless */
1637         if (ctdb->vnn == NULL) {
1638                 return 0;
1639         }
1640
1641         vnn = find_public_ip_vnn(ctdb, &conn->dst);
1642         if (vnn == NULL) {
1643                 DEBUG(DEBUG_ERR,
1644                       (__location__ " unable to find public address %s\n",
1645                        ctdb_addr_to_str(&conn->dst)));
1646                 return 0;
1647         }
1648
1649         ctdb_remove_connection(vnn, conn);
1650
1651         return 0;
1652 }
1653
1654
1655 static void ctdb_send_set_tcp_tickles_for_all(struct ctdb_context *ctdb,
1656                                               bool force);
1657
1658 /*
1659   Called when another daemon starts - causes all tickles for all
1660   public addresses we are serving to be sent to the new node on the
1661   next check.  This actually causes the tickles to be sent to the
1662   other node immediately.  In case there is an error, the periodic
1663   timer will send the updates on timer event.  This is simple and
1664   doesn't require careful error handling.
1665  */
1666 int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t pnn)
1667 {
1668         DEBUG(DEBUG_INFO, ("Received startup control from node %lu\n",
1669                            (unsigned long) pnn));
1670
1671         ctdb_send_set_tcp_tickles_for_all(ctdb, true);
1672         return 0;
1673 }
1674
1675
1676 /*
1677   called when a client structure goes away - hook to remove
1678   elements from the tcp_list in all daemons
1679  */
1680 void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
1681 {
1682         while (client->tcp_list) {
1683                 struct ctdb_vnn *vnn;
1684                 struct ctdb_tcp_list *tcp = client->tcp_list;
1685                 struct ctdb_connection *conn = &tcp->connection;
1686
1687                 vnn = find_public_ip_vnn(client->ctdb,
1688                                          &conn->dst);
1689
1690                 /* If the IP address is hosted on this node then
1691                  * remove the connection. */
1692                 if (vnn != NULL && vnn->pnn == client->ctdb->pnn) {
1693                         ctdb_remove_connection(vnn, conn);
1694                 }
1695
1696                 /* Otherwise this function has been called because the
1697                  * server IP address has been released to another node
1698                  * and the client has exited.  This means that we
1699                  * should not delete the connection information.  The
1700                  * takeover node processes connections too. */
1701
1702                 /*
1703                  * The destructor removes from the list
1704                  */
1705                 TALLOC_FREE(tcp);
1706         }
1707 }
1708
1709
1710 void ctdb_release_all_ips(struct ctdb_context *ctdb)
1711 {
1712         struct ctdb_vnn *vnn, *next;
1713         int count = 0;
1714
1715         if (ctdb_config.failover_disabled == 1) {
1716                 return;
1717         }
1718
1719         for (vnn = ctdb->vnn; vnn != NULL; vnn = next) {
1720                 /* vnn can be freed below in release_ip_post() */
1721                 next = vnn->next;
1722
1723                 if (!ctdb_sys_have_ip(&vnn->public_address)) {
1724                         ctdb_vnn_unassign_iface(ctdb, vnn);
1725                         continue;
1726                 }
1727
1728                 /* Don't allow multiple releases at once.  Some code,
1729                  * particularly ctdb_tickle_sentenced_connections() is
1730                  * not re-entrant */
1731                 if (vnn->update_in_flight) {
1732                         DEBUG(DEBUG_WARNING,
1733                               (__location__
1734                                " Not releasing IP %s/%u on interface %s, an update is already in progress\n",
1735                                     ctdb_addr_to_str(&vnn->public_address),
1736                                     vnn->public_netmask_bits,
1737                                     ctdb_vnn_iface_string(vnn)));
1738                         continue;
1739                 }
1740                 vnn->update_in_flight = true;
1741
1742                 DEBUG(DEBUG_INFO,("Release of IP %s/%u on interface %s node:-1\n",
1743                                     ctdb_addr_to_str(&vnn->public_address),
1744                                     vnn->public_netmask_bits,
1745                                     ctdb_vnn_iface_string(vnn)));
1746
1747                 ctdb_event_script_args(ctdb, CTDB_EVENT_RELEASE_IP, "%s %s %u",
1748                                        ctdb_vnn_iface_string(vnn),
1749                                        ctdb_addr_to_str(&vnn->public_address),
1750                                        vnn->public_netmask_bits);
1751                 /* releaseip timeouts are converted to success, so to
1752                  * detect failures just check if the IP address is
1753                  * still there...
1754                  */
1755                 if (ctdb_sys_have_ip(&vnn->public_address)) {
1756                         DEBUG(DEBUG_ERR,
1757                               (__location__
1758                                " IP address %s not released\n",
1759                                ctdb_addr_to_str(&vnn->public_address)));
1760                         vnn->update_in_flight = false;
1761                         continue;
1762                 }
1763
1764                 vnn = release_ip_post(ctdb, vnn, &vnn->public_address);
1765                 if (vnn != NULL) {
1766                         vnn->update_in_flight = false;
1767                 }
1768                 count++;
1769         }
1770
1771         DEBUG(DEBUG_NOTICE,(__location__ " Released %d public IPs\n", count));
1772 }
1773
1774
1775 /*
1776   get list of public IPs
1777  */
1778 int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb,
1779                                     struct ctdb_req_control_old *c, TDB_DATA *outdata)
1780 {
1781         int i, num, len;
1782         struct ctdb_public_ip_list_old *ips;
1783         struct ctdb_vnn *vnn;
1784         bool only_available = false;
1785
1786         if (c->flags & CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE) {
1787                 only_available = true;
1788         }
1789
1790         /* count how many public ip structures we have */
1791         num = 0;
1792         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1793                 num++;
1794         }
1795
1796         len = offsetof(struct ctdb_public_ip_list_old, ips) +
1797                 num*sizeof(struct ctdb_public_ip);
1798         ips = talloc_zero_size(outdata, len);
1799         CTDB_NO_MEMORY(ctdb, ips);
1800
1801         i = 0;
1802         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
1803                 if (only_available && !ctdb_vnn_available(ctdb, vnn)) {
1804                         continue;
1805                 }
1806                 ips->ips[i].pnn  = vnn->pnn;
1807                 ips->ips[i].addr = vnn->public_address;
1808                 i++;
1809         }
1810         ips->num = i;
1811         len = offsetof(struct ctdb_public_ip_list_old, ips) +
1812                 i*sizeof(struct ctdb_public_ip);
1813
1814         outdata->dsize = len;
1815         outdata->dptr  = (uint8_t *)ips;
1816
1817         return 0;
1818 }
1819
1820
1821 int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
1822                                         struct ctdb_req_control_old *c,
1823                                         TDB_DATA indata,
1824                                         TDB_DATA *outdata)
1825 {
1826         int i, num, len;
1827         ctdb_sock_addr *addr;
1828         struct ctdb_public_ip_info_old *info;
1829         struct ctdb_vnn *vnn;
1830         struct vnn_interface *iface;
1831
1832         addr = (ctdb_sock_addr *)indata.dptr;
1833
1834         vnn = find_public_ip_vnn(ctdb, addr);
1835         if (vnn == NULL) {
1836                 DEBUG(DEBUG_ERR,(__location__ " Could not get public ip info, "
1837                                  "'%s'not a public address\n",
1838                                  ctdb_addr_to_str(addr)));
1839                 return -1;
1840         }
1841
1842         /* count how many public ip structures we have */
1843         num = 0;
1844         for (iface = vnn->ifaces; iface != NULL; iface = iface->next) {
1845                 num++;
1846         }
1847
1848         len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
1849                 num*sizeof(struct ctdb_iface);
1850         info = talloc_zero_size(outdata, len);
1851         CTDB_NO_MEMORY(ctdb, info);
1852
1853         info->ip.addr = vnn->public_address;
1854         info->ip.pnn = vnn->pnn;
1855         info->active_idx = 0xFFFFFFFF;
1856
1857         i = 0;
1858         for (iface = vnn->ifaces; iface != NULL; iface = iface->next) {
1859                 struct ctdb_interface *cur;
1860
1861                 cur = iface->iface;
1862                 if (vnn->iface == cur) {
1863                         info->active_idx = i;
1864                 }
1865                 strncpy(info->ifaces[i].name, cur->name,
1866                         sizeof(info->ifaces[i].name));
1867                 info->ifaces[i].name[sizeof(info->ifaces[i].name)-1] = '\0';
1868                 info->ifaces[i].link_state = cur->link_up;
1869                 info->ifaces[i].references = cur->references;
1870
1871                 i++;
1872         }
1873         info->num = i;
1874         len = offsetof(struct ctdb_public_ip_info_old, ifaces) +
1875                 i*sizeof(struct ctdb_iface);
1876
1877         outdata->dsize = len;
1878         outdata->dptr  = (uint8_t *)info;
1879
1880         return 0;
1881 }
1882
1883 int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
1884                                 struct ctdb_req_control_old *c,
1885                                 TDB_DATA *outdata)
1886 {
1887         int i, num, len;
1888         struct ctdb_iface_list_old *ifaces;
1889         struct ctdb_interface *cur;
1890
1891         /* count how many public ip structures we have */
1892         num = 0;
1893         for (cur=ctdb->ifaces;cur;cur=cur->next) {
1894                 num++;
1895         }
1896
1897         len = offsetof(struct ctdb_iface_list_old, ifaces) +
1898                 num*sizeof(struct ctdb_iface);
1899         ifaces = talloc_zero_size(outdata, len);
1900         CTDB_NO_MEMORY(ctdb, ifaces);
1901
1902         i = 0;
1903         for (cur=ctdb->ifaces;cur;cur=cur->next) {
1904                 strncpy(ifaces->ifaces[i].name, cur->name,
1905                         sizeof(ifaces->ifaces[i].name));
1906                 ifaces->ifaces[i].name[sizeof(ifaces->ifaces[i].name)-1] = '\0';
1907                 ifaces->ifaces[i].link_state = cur->link_up;
1908                 ifaces->ifaces[i].references = cur->references;
1909                 i++;
1910         }
1911         ifaces->num = i;
1912         len = offsetof(struct ctdb_iface_list_old, ifaces) +
1913                 i*sizeof(struct ctdb_iface);
1914
1915         outdata->dsize = len;
1916         outdata->dptr  = (uint8_t *)ifaces;
1917
1918         return 0;
1919 }
1920
1921 int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
1922                                     struct ctdb_req_control_old *c,
1923                                     TDB_DATA indata)
1924 {
1925         struct ctdb_iface *info;
1926         struct ctdb_interface *iface;
1927         bool link_up = false;
1928
1929         info = (struct ctdb_iface *)indata.dptr;
1930
1931         if (info->name[CTDB_IFACE_SIZE] != '\0') {
1932                 int len = strnlen(info->name, CTDB_IFACE_SIZE);
1933                 DEBUG(DEBUG_ERR, (__location__ " name[%*.*s] not terminated\n",
1934                                   len, len, info->name));
1935                 return -1;
1936         }
1937
1938         switch (info->link_state) {
1939         case 0:
1940                 link_up = false;
1941                 break;
1942         case 1:
1943                 link_up = true;
1944                 break;
1945         default:
1946                 DEBUG(DEBUG_ERR, (__location__ " link_state[%u] invalid\n",
1947                                   (unsigned int)info->link_state));
1948                 return -1;
1949         }
1950
1951         if (info->references != 0) {
1952                 DEBUG(DEBUG_ERR, (__location__ " references[%u] should be 0\n",
1953                                   (unsigned int)info->references));
1954                 return -1;
1955         }
1956
1957         iface = ctdb_find_iface(ctdb, info->name);
1958         if (iface == NULL) {
1959                 return -1;
1960         }
1961
1962         if (link_up == iface->link_up) {
1963                 return 0;
1964         }
1965
1966         DEBUG(DEBUG_ERR,
1967               ("iface[%s] has changed it's link status %s => %s\n",
1968                iface->name,
1969                iface->link_up?"up":"down",
1970                link_up?"up":"down"));
1971
1972         iface->link_up = link_up;
1973         return 0;
1974 }
1975
1976
1977 /*
1978   called by a daemon to inform us of the entire list of TCP tickles for
1979   a particular public address.
1980   this control should only be sent by the node that is currently serving
1981   that public address.
1982  */
1983 int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata)
1984 {
1985         struct ctdb_tickle_list_old *list = (struct ctdb_tickle_list_old *)indata.dptr;
1986         struct ctdb_tcp_array *tcparray;
1987         struct ctdb_vnn *vnn;
1988
1989         /* We must at least have tickles.num or else we can't verify the size
1990            of the received data blob
1991          */
1992         if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)) {
1993                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list. Not enough data for the tickle.num field\n"));
1994                 return -1;
1995         }
1996
1997         /* verify that the size of data matches what we expect */
1998         if (indata.dsize < offsetof(struct ctdb_tickle_list_old, connections)
1999                          + sizeof(struct ctdb_connection) * list->num) {
2000                 DEBUG(DEBUG_ERR,("Bad indata in ctdb_tickle_list\n"));
2001                 return -1;
2002         }
2003
2004         DEBUG(DEBUG_INFO, ("Received tickle update for public address %s\n",
2005                            ctdb_addr_to_str(&list->addr)));
2006
2007         vnn = find_public_ip_vnn(ctdb, &list->addr);
2008         if (vnn == NULL) {
2009                 DEBUG(DEBUG_INFO,(__location__ " Could not set tcp tickle list, '%s' is not a public address\n",
2010                         ctdb_addr_to_str(&list->addr)));
2011
2012                 return 1;
2013         }
2014
2015         if (vnn->pnn == ctdb->pnn) {
2016                 DEBUG(DEBUG_INFO,
2017                       ("Ignoring redundant set tcp tickle list, this node hosts '%s'\n",
2018                        ctdb_addr_to_str(&list->addr)));
2019                 return 0;
2020         }
2021
2022         /* remove any old ticklelist we might have */
2023         talloc_free(vnn->tcp_array);
2024         vnn->tcp_array = NULL;
2025
2026         tcparray = talloc(vnn, struct ctdb_tcp_array);
2027         CTDB_NO_MEMORY(ctdb, tcparray);
2028
2029         tcparray->num = list->num;
2030
2031         tcparray->connections = talloc_array(tcparray, struct ctdb_connection, tcparray->num);
2032         CTDB_NO_MEMORY(ctdb, tcparray->connections);
2033
2034         memcpy(tcparray->connections, &list->connections[0],
2035                sizeof(struct ctdb_connection)*tcparray->num);
2036
2037         /* We now have a new fresh tickle list array for this vnn */
2038         vnn->tcp_array = tcparray;
2039
2040         return 0;
2041 }
2042
2043 /*
2044   called to return the full list of tickles for the puclic address associated
2045   with the provided vnn
2046  */
2047 int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
2048 {
2049         ctdb_sock_addr *addr = (ctdb_sock_addr *)indata.dptr;
2050         struct ctdb_tickle_list_old *list;
2051         struct ctdb_tcp_array *tcparray;
2052         unsigned int num, i;
2053         struct ctdb_vnn *vnn;
2054         unsigned port;
2055
2056         vnn = find_public_ip_vnn(ctdb, addr);
2057         if (vnn == NULL) {
2058                 DEBUG(DEBUG_ERR,(__location__ " Could not get tcp tickle list, '%s' is not a public address\n",
2059                         ctdb_addr_to_str(addr)));
2060
2061                 return 1;
2062         }
2063
2064         port = ctdb_addr_to_port(addr);
2065
2066         tcparray = vnn->tcp_array;
2067         num = 0;
2068         if (tcparray != NULL) {
2069                 if (port == 0) {
2070                         /* All connections */
2071                         num = tcparray->num;
2072                 } else {
2073                         /* Count connections for port */
2074                         for (i = 0; i < tcparray->num; i++) {
2075                                 if (port == ctdb_addr_to_port(&tcparray->connections[i].dst)) {
2076                                         num++;
2077                                 }
2078                         }
2079                 }
2080         }
2081
2082         outdata->dsize = offsetof(struct ctdb_tickle_list_old, connections)
2083                         + sizeof(struct ctdb_connection) * num;
2084
2085         outdata->dptr  = talloc_size(outdata, outdata->dsize);
2086         CTDB_NO_MEMORY(ctdb, outdata->dptr);
2087         list = (struct ctdb_tickle_list_old *)outdata->dptr;
2088
2089         list->addr = *addr;
2090         list->num = num;
2091
2092         if (num == 0) {
2093                 return 0;
2094         }
2095
2096         num = 0;
2097         for (i = 0; i < tcparray->num; i++) {
2098                 if (port == 0 || \
2099                     port == ctdb_addr_to_port(&tcparray->connections[i].dst)) {
2100                         list->connections[num] = tcparray->connections[i];
2101                         num++;
2102                 }
2103         }
2104
2105         return 0;
2106 }
2107
2108
2109 /*
2110   set the list of all tcp tickles for a public address
2111  */
2112 static int ctdb_send_set_tcp_tickles_for_ip(struct ctdb_context *ctdb,
2113                                             ctdb_sock_addr *addr,
2114                                             struct ctdb_tcp_array *tcparray)
2115 {
2116         int ret, num;
2117         TDB_DATA data;
2118         struct ctdb_tickle_list_old *list;
2119
2120         if (tcparray) {
2121                 num = tcparray->num;
2122         } else {
2123                 num = 0;
2124         }
2125
2126         data.dsize = offsetof(struct ctdb_tickle_list_old, connections) +
2127                         sizeof(struct ctdb_connection) * num;
2128         data.dptr = talloc_size(ctdb, data.dsize);
2129         CTDB_NO_MEMORY(ctdb, data.dptr);
2130
2131         list = (struct ctdb_tickle_list_old *)data.dptr;
2132         list->addr = *addr;
2133         list->num = num;
2134         if (tcparray) {
2135                 memcpy(&list->connections[0], tcparray->connections, sizeof(struct ctdb_connection) * num);
2136         }
2137
2138         ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
2139                                        CTDB_CONTROL_SET_TCP_TICKLE_LIST,
2140                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
2141         if (ret != 0) {
2142                 DEBUG(DEBUG_ERR,(__location__ " ctdb_control for set tcp tickles failed\n"));
2143                 return -1;
2144         }
2145
2146         talloc_free(data.dptr);
2147
2148         return ret;
2149 }
2150
2151 static void ctdb_send_set_tcp_tickles_for_all(struct ctdb_context *ctdb,
2152                                               bool force)
2153 {
2154         struct ctdb_vnn *vnn;
2155         int ret;
2156
2157         for (vnn = ctdb->vnn; vnn != NULL; vnn = vnn->next) {
2158                 /* we only send out updates for public addresses that
2159                    we have taken over
2160                  */
2161                 if (ctdb->pnn != vnn->pnn) {
2162                         continue;
2163                 }
2164
2165                 /* We only send out the updates if we need to */
2166                 if (!force && !vnn->tcp_update_needed) {
2167                         continue;
2168                 }
2169
2170                 ret = ctdb_send_set_tcp_tickles_for_ip(ctdb,
2171                                                        &vnn->public_address,
2172                                                        vnn->tcp_array);
2173                 if (ret != 0) {
2174                         D_ERR("Failed to send the tickle update for ip %s\n",
2175                               ctdb_addr_to_str(&vnn->public_address));
2176                         vnn->tcp_update_needed = true;
2177                 } else {
2178                         D_INFO("Sent tickle update for ip %s\n",
2179                                ctdb_addr_to_str(&vnn->public_address));
2180                         vnn->tcp_update_needed = false;
2181                 }
2182         }
2183
2184 }
2185
2186 /*
2187   perform tickle updates if required
2188  */
2189 static void ctdb_update_tcp_tickles(struct tevent_context *ev,
2190                                     struct tevent_timer *te,
2191                                     struct timeval t, void *private_data)
2192 {
2193         struct ctdb_context *ctdb = talloc_get_type(
2194                 private_data, struct ctdb_context);
2195
2196         ctdb_send_set_tcp_tickles_for_all(ctdb, false);
2197
2198         tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
2199                          timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
2200                          ctdb_update_tcp_tickles, ctdb);
2201 }
2202
2203 /*
2204   start periodic update of tcp tickles
2205  */
2206 void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb)
2207 {
2208         ctdb->tickle_update_context = talloc_new(ctdb);
2209
2210         tevent_add_timer(ctdb->ev, ctdb->tickle_update_context,
2211                          timeval_current_ofs(ctdb->tunable.tickle_update_interval, 0),
2212                          ctdb_update_tcp_tickles, ctdb);
2213 }
2214
2215
2216
2217
2218 struct control_gratious_arp {
2219         struct ctdb_context *ctdb;
2220         ctdb_sock_addr addr;
2221         const char *iface;
2222         int count;
2223 };
2224
2225 /*
2226   send a control_gratuitous arp
2227  */
2228 static void send_gratious_arp(struct tevent_context *ev,
2229                               struct tevent_timer *te,
2230                               struct timeval t, void *private_data)
2231 {
2232         int ret;
2233         struct control_gratious_arp *arp = talloc_get_type(private_data,
2234                                                         struct control_gratious_arp);
2235
2236         ret = ctdb_sys_send_arp(&arp->addr, arp->iface);
2237         if (ret != 0) {
2238                 DBG_ERR("Failed to send gratuitous ARP on iface %s: %s\n",
2239                         arp->iface, strerror(ret));
2240         }
2241
2242
2243         arp->count++;
2244         if (arp->count == CTDB_ARP_REPEAT) {
2245                 talloc_free(arp);
2246                 return;
2247         }
2248
2249         tevent_add_timer(arp->ctdb->ev, arp,
2250                          timeval_current_ofs(CTDB_ARP_INTERVAL, 0),
2251                          send_gratious_arp, arp);
2252 }
2253
2254
2255 /*
2256   send a gratious arp
2257  */
2258 int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata)
2259 {
2260         struct ctdb_addr_info_old *gratious_arp = (struct ctdb_addr_info_old *)indata.dptr;
2261         struct control_gratious_arp *arp;
2262
2263         /* verify the size of indata */
2264         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2265                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_control_gratious_arp structure. Got %u require %u bytes\n",
2266                                  (unsigned)indata.dsize,
2267                                  (unsigned)offsetof(struct ctdb_addr_info_old, iface)));
2268                 return -1;
2269         }
2270         if (indata.dsize !=
2271                 ( offsetof(struct ctdb_addr_info_old, iface)
2272                 + gratious_arp->len ) ){
2273
2274                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2275                         "but should be %u bytes\n",
2276                          (unsigned)indata.dsize,
2277                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+gratious_arp->len)));
2278                 return -1;
2279         }
2280
2281
2282         arp = talloc(ctdb, struct control_gratious_arp);
2283         CTDB_NO_MEMORY(ctdb, arp);
2284
2285         arp->ctdb  = ctdb;
2286         arp->addr   = gratious_arp->addr;
2287         arp->iface = talloc_strdup(arp, gratious_arp->iface);
2288         CTDB_NO_MEMORY(ctdb, arp->iface);
2289         arp->count = 0;
2290
2291         tevent_add_timer(arp->ctdb->ev, arp,
2292                          timeval_zero(), send_gratious_arp, arp);
2293
2294         return 0;
2295 }
2296
2297 int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
2298 {
2299         struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
2300         int ret;
2301
2302         /* verify the size of indata */
2303         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2304                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
2305                 return -1;
2306         }
2307         if (indata.dsize !=
2308                 ( offsetof(struct ctdb_addr_info_old, iface)
2309                 + pub->len ) ){
2310
2311                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2312                         "but should be %u bytes\n",
2313                          (unsigned)indata.dsize,
2314                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
2315                 return -1;
2316         }
2317
2318         DEBUG(DEBUG_NOTICE,("Add IP %s\n", ctdb_addr_to_str(&pub->addr)));
2319
2320         ret = ctdb_add_public_address(ctdb, &pub->addr, pub->mask, &pub->iface[0], true);
2321
2322         if (ret != 0) {
2323                 DEBUG(DEBUG_ERR,(__location__ " Failed to add public address\n"));
2324                 return -1;
2325         }
2326
2327         return 0;
2328 }
2329
2330 int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata)
2331 {
2332         struct ctdb_addr_info_old *pub = (struct ctdb_addr_info_old *)indata.dptr;
2333         struct ctdb_vnn *vnn;
2334
2335         /* verify the size of indata */
2336         if (indata.dsize < offsetof(struct ctdb_addr_info_old, iface)) {
2337                 DEBUG(DEBUG_ERR,(__location__ " Too small indata to hold a ctdb_addr_info structure\n"));
2338                 return -1;
2339         }
2340         if (indata.dsize !=
2341                 ( offsetof(struct ctdb_addr_info_old, iface)
2342                 + pub->len ) ){
2343
2344                 DEBUG(DEBUG_ERR,(__location__ " Wrong size of indata. Was %u bytes "
2345                         "but should be %u bytes\n",
2346                          (unsigned)indata.dsize,
2347                          (unsigned)(offsetof(struct ctdb_addr_info_old, iface)+pub->len)));
2348                 return -1;
2349         }
2350
2351         DEBUG(DEBUG_NOTICE,("Delete IP %s\n", ctdb_addr_to_str(&pub->addr)));
2352
2353         /* walk over all public addresses until we find a match */
2354         for (vnn=ctdb->vnn;vnn;vnn=vnn->next) {
2355                 if (ctdb_same_ip(&vnn->public_address, &pub->addr)) {
2356                         if (vnn->pnn == ctdb->pnn) {
2357                                 /* This IP is currently being hosted.
2358                                  * Defer the deletion until the next
2359                                  * takeover run. "ctdb reloadips" will
2360                                  * always cause a takeover run.  "ctdb
2361                                  * delip" will now need an explicit
2362                                  * "ctdb ipreallocated" afterwards. */
2363                                 vnn->delete_pending = true;
2364                         } else {
2365                                 /* This IP is not hosted on the
2366                                  * current node so just delete it
2367                                  * now. */
2368                                 do_delete_ip(ctdb, vnn);
2369                         }
2370
2371                         return 0;
2372                 }
2373         }
2374
2375         DEBUG(DEBUG_ERR,("Delete IP of unknown public IP address %s\n",
2376                          ctdb_addr_to_str(&pub->addr)));
2377         return -1;
2378 }
2379
2380
2381 struct ipreallocated_callback_state {
2382         struct ctdb_req_control_old *c;
2383 };
2384
2385 static void ctdb_ipreallocated_callback(struct ctdb_context *ctdb,
2386                                         int status, void *p)
2387 {
2388         struct ipreallocated_callback_state *state =
2389                 talloc_get_type(p, struct ipreallocated_callback_state);
2390
2391         if (status != 0) {
2392                 DEBUG(DEBUG_ERR,
2393                       (" \"ipreallocated\" event script failed (status %d)\n",
2394                        status));
2395                 if (status == -ETIMEDOUT) {
2396                         ctdb_ban_self(ctdb);
2397                 }
2398         }
2399
2400         ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
2401         talloc_free(state);
2402 }
2403
2404 /* A control to run the ipreallocated event */
2405 int32_t ctdb_control_ipreallocated(struct ctdb_context *ctdb,
2406                                    struct ctdb_req_control_old *c,
2407                                    bool *async_reply)
2408 {
2409         int ret;
2410         struct ipreallocated_callback_state *state;
2411
2412         state = talloc(ctdb, struct ipreallocated_callback_state);
2413         CTDB_NO_MEMORY(ctdb, state);
2414
2415         DEBUG(DEBUG_INFO,(__location__ " Running \"ipreallocated\" event\n"));
2416
2417         ret = ctdb_event_script_callback(ctdb, state,
2418                                          ctdb_ipreallocated_callback, state,
2419                                          CTDB_EVENT_IPREALLOCATED,
2420                                          "%s", "");
2421
2422         if (ret != 0) {
2423                 DEBUG(DEBUG_ERR,("Failed to run \"ipreallocated\" event \n"));
2424                 talloc_free(state);
2425                 return -1;
2426         }
2427
2428         /* tell the control that we will be reply asynchronously */
2429         state->c    = talloc_steal(state, c);
2430         *async_reply = true;
2431
2432         return 0;
2433 }
2434
2435
2436 struct ctdb_reloadips_handle {
2437         struct ctdb_context *ctdb;
2438         struct ctdb_req_control_old *c;
2439         int status;
2440         int fd[2];
2441         pid_t child;
2442         struct tevent_fd *fde;
2443 };
2444
2445 static int ctdb_reloadips_destructor(struct ctdb_reloadips_handle *h)
2446 {
2447         if (h == h->ctdb->reload_ips) {
2448                 h->ctdb->reload_ips = NULL;
2449         }
2450         if (h->c != NULL) {
2451                 ctdb_request_control_reply(h->ctdb, h->c, NULL, h->status, NULL);
2452                 h->c = NULL;
2453         }
2454         ctdb_kill(h->ctdb, h->child, SIGKILL);
2455         return 0;
2456 }
2457
2458 static void ctdb_reloadips_timeout_event(struct tevent_context *ev,
2459                                          struct tevent_timer *te,
2460                                          struct timeval t, void *private_data)
2461 {
2462         struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
2463
2464         talloc_free(h);
2465 }
2466
2467 static void ctdb_reloadips_child_handler(struct tevent_context *ev,
2468                                          struct tevent_fd *fde,
2469                                          uint16_t flags, void *private_data)
2470 {
2471         struct ctdb_reloadips_handle *h = talloc_get_type(private_data, struct ctdb_reloadips_handle);
2472
2473         char res;
2474         int ret;
2475
2476         ret = sys_read(h->fd[0], &res, 1);
2477         if (ret < 1 || res != 0) {
2478                 DEBUG(DEBUG_ERR, (__location__ " Reloadips child process returned error\n"));
2479                 res = 1;
2480         }
2481         h->status = res;
2482
2483         talloc_free(h);
2484 }
2485
2486 static int ctdb_reloadips_child(struct ctdb_context *ctdb)
2487 {
2488         TALLOC_CTX *mem_ctx = talloc_new(NULL);
2489         struct ctdb_public_ip_list_old *ips;
2490         struct ctdb_vnn *vnn;
2491         struct client_async_data *async_data;
2492         struct timeval timeout;
2493         TDB_DATA data;
2494         struct ctdb_client_control_state *state;
2495         bool first_add;
2496         unsigned int i;
2497         int ret;
2498
2499         CTDB_NO_MEMORY(ctdb, mem_ctx);
2500
2501         /* Read IPs from local node */
2502         ret = ctdb_ctrl_get_public_ips(ctdb, TAKEOVER_TIMEOUT(),
2503                                        CTDB_CURRENT_NODE, mem_ctx, &ips);
2504         if (ret != 0) {
2505                 DEBUG(DEBUG_ERR,
2506                       ("Unable to fetch public IPs from local node\n"));
2507                 talloc_free(mem_ctx);
2508                 return -1;
2509         }
2510
2511         /* Read IPs file - this is safe since this is a child process */
2512         ctdb->vnn = NULL;
2513         if (ctdb_set_public_addresses(ctdb, false) != 0) {
2514                 DEBUG(DEBUG_ERR,("Failed to re-read public addresses file\n"));
2515                 talloc_free(mem_ctx);
2516                 return -1;
2517         }
2518
2519         async_data = talloc_zero(mem_ctx, struct client_async_data);
2520         CTDB_NO_MEMORY(ctdb, async_data);
2521
2522         /* Compare IPs between node and file for IPs to be deleted */
2523         for (i = 0; i < ips->num; i++) {
2524                 /* */
2525                 for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
2526                         if (ctdb_same_ip(&vnn->public_address,
2527                                          &ips->ips[i].addr)) {
2528                                 /* IP is still in file */
2529                                 break;
2530                         }
2531                 }
2532
2533                 if (vnn == NULL) {
2534                         /* Delete IP ips->ips[i] */
2535                         struct ctdb_addr_info_old *pub;
2536
2537                         DEBUG(DEBUG_NOTICE,
2538                               ("IP %s no longer configured, deleting it\n",
2539                                ctdb_addr_to_str(&ips->ips[i].addr)));
2540
2541                         pub = talloc_zero(mem_ctx, struct ctdb_addr_info_old);
2542                         CTDB_NO_MEMORY(ctdb, pub);
2543
2544                         pub->addr  = ips->ips[i].addr;
2545                         pub->mask  = 0;
2546                         pub->len   = 0;
2547
2548                         timeout = TAKEOVER_TIMEOUT();
2549
2550                         data.dsize = offsetof(struct ctdb_addr_info_old,
2551                                               iface) + pub->len;
2552                         data.dptr = (uint8_t *)pub;
2553
2554                         state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
2555                                                   CTDB_CONTROL_DEL_PUBLIC_IP,
2556                                                   0, data, async_data,
2557                                                   &timeout, NULL);
2558                         if (state == NULL) {
2559                                 DEBUG(DEBUG_ERR,
2560                                       (__location__
2561                                        " failed sending CTDB_CONTROL_DEL_PUBLIC_IP\n"));
2562                                 goto failed;
2563                         }
2564
2565                         ctdb_client_async_add(async_data, state);
2566                 }
2567         }
2568
2569         /* Compare IPs between node and file for IPs to be added */
2570         first_add = true;
2571         for (vnn = ctdb->vnn; vnn; vnn = vnn->next) {
2572                 for (i = 0; i < ips->num; i++) {
2573                         if (ctdb_same_ip(&vnn->public_address,
2574                                          &ips->ips[i].addr)) {
2575                                 /* IP already on node */
2576                                 break;
2577                         }
2578                 }
2579                 if (i == ips->num) {
2580                         /* Add IP ips->ips[i] */
2581                         struct ctdb_addr_info_old *pub;
2582                         const char *ifaces = NULL;
2583                         uint32_t len;
2584                         struct vnn_interface *iface = NULL;
2585
2586                         DEBUG(DEBUG_NOTICE,
2587                               ("New IP %s configured, adding it\n",
2588                                ctdb_addr_to_str(&vnn->public_address)));
2589                         if (first_add) {
2590                                 uint32_t pnn = ctdb_get_pnn(ctdb);
2591
2592                                 data.dsize = sizeof(pnn);
2593                                 data.dptr  = (uint8_t *)&pnn;
2594
2595                                 ret = ctdb_client_send_message(
2596                                         ctdb,
2597                                         CTDB_BROADCAST_CONNECTED,
2598                                         CTDB_SRVID_REBALANCE_NODE,
2599                                         data);
2600                                 if (ret != 0) {
2601                                         DEBUG(DEBUG_WARNING,
2602                                               ("Failed to send message to force node reallocation - IPs may be unbalanced\n"));
2603                                 }
2604
2605                                 first_add = false;
2606                         }
2607
2608                         ifaces = vnn->ifaces->iface->name;
2609                         iface = vnn->ifaces->next;
2610                         while (iface != NULL) {
2611                                 ifaces = talloc_asprintf(vnn, "%s,%s", ifaces,
2612                                                          iface->iface->name);
2613                                 iface = iface->next;
2614                         }
2615
2616                         len   = strlen(ifaces) + 1;
2617                         pub = talloc_zero_size(mem_ctx,
2618                                                offsetof(struct ctdb_addr_info_old, iface) + len);
2619                         CTDB_NO_MEMORY(ctdb, pub);
2620
2621                         pub->addr  = vnn->public_address;
2622                         pub->mask  = vnn->public_netmask_bits;
2623                         pub->len   = len;
2624                         memcpy(&pub->iface[0], ifaces, pub->len);
2625
2626                         timeout = TAKEOVER_TIMEOUT();
2627
2628                         data.dsize = offsetof(struct ctdb_addr_info_old,
2629                                               iface) + pub->len;
2630                         data.dptr = (uint8_t *)pub;
2631
2632                         state = ctdb_control_send(ctdb, CTDB_CURRENT_NODE, 0,
2633                                                   CTDB_CONTROL_ADD_PUBLIC_IP,
2634                                                   0, data, async_data,
2635                                                   &timeout, NULL);
2636                         if (state == NULL) {
2637                                 DEBUG(DEBUG_ERR,
2638                                       (__location__
2639                                        " failed sending CTDB_CONTROL_ADD_PUBLIC_IP\n"));
2640                                 goto failed;
2641                         }
2642
2643                         ctdb_client_async_add(async_data, state);
2644                 }
2645         }
2646
2647         if (ctdb_client_async_wait(ctdb, async_data) != 0) {
2648                 DEBUG(DEBUG_ERR,(__location__ " Add/delete IPs failed\n"));
2649                 goto failed;
2650         }
2651
2652         talloc_free(mem_ctx);
2653         return 0;
2654
2655 failed:
2656         talloc_free(mem_ctx);
2657         return -1;
2658 }
2659
2660 /* This control is sent to force the node to re-read the public addresses file
2661    and drop any addresses we should nnot longer host, and add new addresses
2662    that we are now able to host
2663 */
2664 int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control_old *c, bool *async_reply)
2665 {
2666         struct ctdb_reloadips_handle *h;
2667         pid_t parent = getpid();
2668
2669         if (ctdb->reload_ips != NULL) {
2670                 talloc_free(ctdb->reload_ips);
2671                 ctdb->reload_ips = NULL;
2672         }
2673
2674         h = talloc(ctdb, struct ctdb_reloadips_handle);
2675         CTDB_NO_MEMORY(ctdb, h);
2676         h->ctdb     = ctdb;
2677         h->c        = NULL;
2678         h->status   = -1;
2679
2680         if (pipe(h->fd) == -1) {
2681                 DEBUG(DEBUG_ERR,("Failed to create pipe for ctdb_freeze_lock\n"));
2682                 talloc_free(h);
2683                 return -1;
2684         }
2685
2686         h->child = ctdb_fork(ctdb);
2687         if (h->child == (pid_t)-1) {
2688                 DEBUG(DEBUG_ERR, ("Failed to fork a child for reloadips\n"));
2689                 close(h->fd[0]);
2690                 close(h->fd[1]);
2691                 talloc_free(h);
2692                 return -1;
2693         }
2694
2695         /* child process */
2696         if (h->child == 0) {
2697                 signed char res = 0;
2698
2699                 close(h->fd[0]);
2700
2701                 prctl_set_comment("ctdb_reloadips");
2702                 if (switch_from_server_to_client(ctdb) != 0) {
2703                         DEBUG(DEBUG_CRIT,("ERROR: Failed to switch reloadips child into client mode\n"));
2704                         res = -1;
2705                 } else {
2706                         res = ctdb_reloadips_child(ctdb);
2707                         if (res != 0) {
2708                                 DEBUG(DEBUG_ERR,("Failed to reload ips on local node\n"));
2709                         }
2710                 }
2711
2712                 sys_write(h->fd[1], &res, 1);
2713                 ctdb_wait_for_process_to_exit(parent);
2714                 _exit(0);
2715         }
2716
2717         h->c             = talloc_steal(h, c);
2718
2719         close(h->fd[1]);
2720         set_close_on_exec(h->fd[0]);
2721
2722         talloc_set_destructor(h, ctdb_reloadips_destructor);
2723
2724
2725         h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ,
2726                                ctdb_reloadips_child_handler, (void *)h);
2727         tevent_fd_set_auto_close(h->fde);
2728
2729         tevent_add_timer(ctdb->ev, h, timeval_current_ofs(120, 0),
2730                          ctdb_reloadips_timeout_event, h);
2731
2732         /* we reply later */
2733         *async_reply = true;
2734         return 0;
2735 }