tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "lib/tevent/tevent.h"
  23 #include "system/time.h"
  24 #include "system/filesys.h"
  25 #include "system/network.h"
  26 #include "system/locale.h"
  27 #include "popt.h"
  28 #include "cmdline.h"
  29 #include "../include/ctdb.h"
  30 #include "../include/ctdb_client.h"
  31 #include "../include/ctdb_private.h"
  32 #include "../common/rb_tree.h"
  33 #include "db_wrap.h"
  34
  35 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  36 #define ERR_NONODE      21      /* node does not exist */
  37 #define ERR_DISNODE     22      /* node is disconnected */
  38
  39 struct ctdb_connection *ctdb_connection;
  40
  41 static void usage(void);
  42
  43 static struct {
  44         int timelimit;
  45         uint32_t pnn;
  46         int machinereadable;
  47         int verbose;
  48         int maxruntime;
  49 } options;
  50
  51 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  52 #define LONGTIMELIMIT() timeval_current_ofs(options.timelimit*10, 0)
  53
  54 #ifdef CTDB_VERS
  55 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  56 {
  57 #define STR(x) #x
  58 #define XSTR(x) STR(x)
  59         printf("CTDB version: %s\n", XSTR(CTDB_VERS));
  60         return 0;
  61 }
  62 #endif
  63
  64
  65 /*
  66   verify that a node exists and is reachable
  67  */
  68 static void verify_node(struct ctdb_context *ctdb)
  69 {
  70         int ret;
  71         struct ctdb_node_map *nodemap=NULL;
  72
  73         if (options.pnn == CTDB_CURRENT_NODE) {
  74                 return;
  75         }
  76         if (options.pnn == CTDB_BROADCAST_ALL) {
  77                 return;
  78         }
  79
  80         /* verify the node exists */
  81         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
  82                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
  83                 exit(10);
  84         }
  85         if (options.pnn >= nodemap->num) {
  86                 DEBUG(DEBUG_ERR, ("Node %u does not exist\n", options.pnn));
  87                 exit(ERR_NONODE);
  88         }
  89         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_DELETED) {
  90                 DEBUG(DEBUG_ERR, ("Node %u is DELETED\n", options.pnn));
  91                 exit(ERR_DISNODE);
  92         }
  93         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_DISCONNECTED) {
  94                 DEBUG(DEBUG_ERR, ("Node %u is DISCONNECTED\n", options.pnn));
  95                 exit(ERR_DISNODE);
  96         }
  97
  98         /* verify we can access the node */
  99         ret = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
 100         if (ret == -1) {
 101                 DEBUG(DEBUG_ERR,("Can not access node. Node is not operational.\n"));
 102                 exit(10);
 103         }
 104 }
 105
 106 /*
 107  check if a database exists
 108 */
 109 static int db_exists(struct ctdb_context *ctdb, const char *db_name)
 110 {
 111         int i, ret;
 112         struct ctdb_dbid_map *dbmap=NULL;
 113
 114         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
 115         if (ret != 0) {
 116                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 117                 return -1;
 118         }
 119
 120         for(i=0;i<dbmap->num;i++){
 121                 const char *name;
 122
 123                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
 124                 if (!strcmp(name, db_name)) {
 125                         return 0;
 126                 }
 127         }
 128
 129         return -1;
 130 }
 131
 132 /*
 133   see if a process exists
 134  */
 135 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 136 {
 137         uint32_t pnn, pid;
 138         int ret;
 139         if (argc < 1) {
 140                 usage();
 141         }
 142
 143         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 144                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 145                 return -1;
 146         }
 147
 148         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 149         if (ret == 0) {
 150                 printf("%u:%u exists\n", pnn, pid);
 151         } else {
 152                 printf("%u:%u does not exist\n", pnn, pid);
 153         }
 154         return ret;
 155 }
 156
 157 /*
 158   display statistics structure
 159  */
 160 static void show_statistics(struct ctdb_statistics *s)
 161 {
 162         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 163         int i;
 164         const char *prefix=NULL;
 165         int preflen=0;
 166         int tmp, days, hours, minutes, seconds;
 167         const struct {
 168                 const char *name;
 169                 uint32_t offset;
 170         } fields[] = {
 171 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 172                 STATISTICS_FIELD(num_clients),
 173                 STATISTICS_FIELD(frozen),
 174                 STATISTICS_FIELD(recovering),
 175                 STATISTICS_FIELD(num_recoveries),
 176                 STATISTICS_FIELD(client_packets_sent),
 177                 STATISTICS_FIELD(client_packets_recv),
 178                 STATISTICS_FIELD(node_packets_sent),
 179                 STATISTICS_FIELD(node_packets_recv),
 180                 STATISTICS_FIELD(keepalive_packets_sent),
 181                 STATISTICS_FIELD(keepalive_packets_recv),
 182                 STATISTICS_FIELD(node.req_call),
 183                 STATISTICS_FIELD(node.reply_call),
 184                 STATISTICS_FIELD(node.req_dmaster),
 185                 STATISTICS_FIELD(node.reply_dmaster),
 186                 STATISTICS_FIELD(node.reply_error),
 187                 STATISTICS_FIELD(node.req_message),
 188                 STATISTICS_FIELD(node.req_control),
 189                 STATISTICS_FIELD(node.reply_control),
 190                 STATISTICS_FIELD(client.req_call),
 191                 STATISTICS_FIELD(client.req_message),
 192                 STATISTICS_FIELD(client.req_control),
 193                 STATISTICS_FIELD(timeouts.call),
 194                 STATISTICS_FIELD(timeouts.control),
 195                 STATISTICS_FIELD(timeouts.traverse),
 196                 STATISTICS_FIELD(total_calls),
 197                 STATISTICS_FIELD(pending_calls),
 198                 STATISTICS_FIELD(lockwait_calls),
 199                 STATISTICS_FIELD(pending_lockwait_calls),
 200                 STATISTICS_FIELD(childwrite_calls),
 201                 STATISTICS_FIELD(pending_childwrite_calls),
 202                 STATISTICS_FIELD(memory_used),
 203                 STATISTICS_FIELD(max_hop_count),
 204         };
 205         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 206         seconds = tmp%60;
 207         tmp    /= 60;
 208         minutes = tmp%60;
 209         tmp    /= 60;
 210         hours   = tmp%24;
 211         tmp    /= 24;
 212         days    = tmp;
 213
 214         printf("CTDB version %u\n", CTDB_VERSION);
 215         printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 216         printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 217
 218         for (i=0;i<ARRAY_SIZE(fields);i++) {
 219                 if (strchr(fields[i].name, '.')) {
 220                         preflen = strcspn(fields[i].name, ".")+1;
 221                         if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 222                                 prefix = fields[i].name;
 223                                 printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 224                         }
 225                 } else {
 226                         preflen = 0;
 227                 }
 228                 printf(" %*s%-22s%*s%10u\n",
 229                        preflen?4:0, "",
 230                        fields[i].name+preflen,
 231                        preflen?0:4, "",
 232                        *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 233         }
 234         printf(" %-30s     %.6f sec\n", "max_reclock_ctdbd", s->reclock.ctdbd);
 235         printf(" %-30s     %.6f sec\n", "max_reclock_recd", s->reclock.recd);
 236
 237         printf(" %-30s     %.6f sec\n", "max_call_latency", s->max_call_latency);
 238         printf(" %-30s     %.6f sec\n", "max_lockwait_latency", s->max_lockwait_latency);
 239         printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
 240         printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
 241
 242         talloc_free(tmp_ctx);
 243 }
 244
 245 /*
 246   display remote ctdb statistics combined from all nodes
 247  */
 248 static int control_statistics_all(struct ctdb_context *ctdb)
 249 {
 250         int ret, i;
 251         struct ctdb_statistics statistics;
 252         uint32_t *nodes;
 253         uint32_t num_nodes;
 254
 255         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 256         CTDB_NO_MEMORY(ctdb, nodes);
 257
 258         ZERO_STRUCT(statistics);
 259
 260         for (i=0;i<num_nodes;i++) {
 261                 struct ctdb_statistics s1;
 262                 int j;
 263                 uint32_t *v1 = (uint32_t *)&s1;
 264                 uint32_t *v2 = (uint32_t *)&statistics;
 265                 uint32_t num_ints =
 266                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 267                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 268                 if (ret != 0) {
 269                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 270                         return ret;
 271                 }
 272                 for (j=0;j<num_ints;j++) {
 273                         v2[j] += v1[j];
 274                 }
 275                 statistics.max_hop_count =
 276                         MAX(statistics.max_hop_count, s1.max_hop_count);
 277                 statistics.max_call_latency =
 278                         MAX(statistics.max_call_latency, s1.max_call_latency);
 279                 statistics.max_lockwait_latency =
 280                         MAX(statistics.max_lockwait_latency, s1.max_lockwait_latency);
 281         }
 282         talloc_free(nodes);
 283         printf("Gathered statistics for %u nodes\n", num_nodes);
 284         show_statistics(&statistics);
 285         return 0;
 286 }
 287
 288 /*
 289   display remote ctdb statistics
 290  */
 291 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 292 {
 293         int ret;
 294         struct ctdb_statistics statistics;
 295
 296         if (options.pnn == CTDB_BROADCAST_ALL) {
 297                 return control_statistics_all(ctdb);
 298         }
 299
 300         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 301         if (ret != 0) {
 302                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 303                 return ret;
 304         }
 305         show_statistics(&statistics);
 306         return 0;
 307 }
 308
 309
 310 /*
 311   reset remote ctdb statistics
 312  */
 313 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 314 {
 315         int ret;
 316
 317         ret = ctdb_statistics_reset(ctdb, options.pnn);
 318         if (ret != 0) {
 319                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 320                 return ret;
 321         }
 322         return 0;
 323 }
 324
 325
 326 /*
 327   display uptime of remote node
 328  */
 329 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 330 {
 331         int ret;
 332         struct ctdb_uptime *uptime = NULL;
 333         int tmp, days, hours, minutes, seconds;
 334
 335         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 336         if (ret != 0) {
 337                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 338                 return ret;
 339         }
 340
 341         if (options.machinereadable){
 342                 printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 343                 printf(":%u:%u:%u:%lf\n",
 344                         (unsigned int)uptime->current_time.tv_sec,
 345                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 346                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 347                         timeval_delta(&uptime->last_recovery_finished,
 348                                       &uptime->last_recovery_started)
 349                 );
 350                 return 0;
 351         }
 352
 353         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 354
 355         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 356         seconds = tmp%60;
 357         tmp    /= 60;
 358         minutes = tmp%60;
 359         tmp    /= 60;
 360         hours   = tmp%24;
 361         tmp    /= 24;
 362         days    = tmp;
 363         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 364
 365         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 366         seconds = tmp%60;
 367         tmp    /= 60;
 368         minutes = tmp%60;
 369         tmp    /= 60;
 370         hours   = tmp%24;
 371         tmp    /= 24;
 372         days    = tmp;
 373         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 374
 375         printf("Duration of last recovery/failover: %lf seconds\n",
 376                 timeval_delta(&uptime->last_recovery_finished,
 377                               &uptime->last_recovery_started));
 378
 379         return 0;
 380 }
 381
 382 /*
 383   show the PNN of the current node
 384  */
 385 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 386 {
 387         uint32_t mypnn;
 388         bool ret;
 389
 390         ret = ctdb_getpnn(ctdb_connection, options.pnn, &mypnn);
 391         if (!ret) {
 392                 DEBUG(DEBUG_ERR, ("Unable to get pnn from node."));
 393                 return -1;
 394         }
 395
 396         printf("PNN:%d\n", mypnn);
 397         return 0;
 398 }
 399
 400
 401 struct pnn_node {
 402         struct pnn_node *next;
 403         const char *addr;
 404         int pnn;
 405 };
 406
 407 static struct pnn_node *read_nodes_file(TALLOC_CTX *mem_ctx)
 408 {
 409         const char *nodes_list;
 410         int nlines;
 411         char **lines;
 412         int i, pnn;
 413         struct pnn_node *pnn_nodes = NULL;
 414         struct pnn_node *pnn_node;
 415         struct pnn_node *tmp_node;
 416
 417         /* read the nodes file */
 418         nodes_list = getenv("CTDB_NODES");
 419         if (nodes_list == NULL) {
 420                 nodes_list = "/etc/ctdb/nodes";
 421         }
 422         lines = file_lines_load(nodes_list, &nlines, mem_ctx);
 423         if (lines == NULL) {
 424                 return NULL;
 425         }
 426         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 427                 nlines--;
 428         }
 429         for (i=0, pnn=0; i<nlines; i++) {
 430                 char *node;
 431
 432                 node = lines[i];
 433                 /* strip leading spaces */
 434                 while((*node == ' ') || (*node == '\t')) {
 435                         node++;
 436                 }
 437                 if (*node == '#') {
 438                         pnn++;
 439                         continue;
 440                 }
 441                 if (strcmp(node, "") == 0) {
 442                         continue;
 443                 }
 444                 pnn_node = talloc(mem_ctx, struct pnn_node);
 445                 pnn_node->pnn = pnn++;
 446                 pnn_node->addr = talloc_strdup(pnn_node, node);
 447                 pnn_node->next = pnn_nodes;
 448                 pnn_nodes = pnn_node;
 449         }
 450
 451         /* swap them around so we return them in incrementing order */
 452         pnn_node = pnn_nodes;
 453         pnn_nodes = NULL;
 454         while (pnn_node) {
 455                 tmp_node = pnn_node;
 456                 pnn_node = pnn_node->next;
 457
 458                 tmp_node->next = pnn_nodes;
 459                 pnn_nodes = tmp_node;
 460         }
 461
 462         return pnn_nodes;
 463 }
 464
 465 /*
 466   show the PNN of the current node
 467   discover the pnn by loading the nodes file and try to bind to all
 468   addresses one at a time until the ip address is found.
 469  */
 470 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 471 {
 472         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 473         struct pnn_node *pnn_nodes;
 474         struct pnn_node *pnn_node;
 475
 476         pnn_nodes = read_nodes_file(mem_ctx);
 477         if (pnn_nodes == NULL) {
 478                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
 479                 talloc_free(mem_ctx);
 480                 return -1;
 481         }
 482
 483         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
 484                 ctdb_sock_addr addr;
 485
 486                 if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
 487                         DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
 488                         talloc_free(mem_ctx);
 489                         return -1;
 490                 }
 491
 492                 if (ctdb_sys_have_ip(&addr)) {
 493                         printf("PNN:%d\n", pnn_node->pnn);
 494                         talloc_free(mem_ctx);
 495                         return 0;
 496                 }
 497         }
 498
 499         printf("Failed to detect which PNN this node is\n");
 500         talloc_free(mem_ctx);
 501         return -1;
 502 }
 503
 504 /*
 505   display remote ctdb status
 506  */
 507 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
 508 {
 509         int i, ret;
 510         struct ctdb_vnn_map *vnnmap=NULL;
 511         struct ctdb_node_map *nodemap=NULL;
 512         uint32_t recmode, recmaster;
 513         int mypnn;
 514
 515         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
 516         if (mypnn == -1) {
 517                 return -1;
 518         }
 519
 520         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
 521         if (ret != 0) {
 522                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
 523                 return ret;
 524         }
 525
 526         if(options.machinereadable){
 527                 printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:\n");
 528                 for(i=0;i<nodemap->num;i++){
 529                         int partially_online = 0;
 530                         int j;
 531
 532                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 533                                 continue;
 534                         }
 535                         if (nodemap->nodes[i].flags == 0) {
 536                                 struct ctdb_control_get_ifaces *ifaces;
 537
 538                                 ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
 539                                                            nodemap->nodes[i].pnn,
 540                                                            ctdb, &ifaces);
 541                                 if (ret == 0) {
 542                                         for (j=0; j < ifaces->num; j++) {
 543                                                 if (ifaces->ifaces[j].link_state != 0) {
 544                                                         continue;
 545                                                 }
 546                                                 partially_online = 1;
 547                                                 break;
 548                                         }
 549                                         talloc_free(ifaces);
 550                                 }
 551                         }
 552                         printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
 553                                 ctdb_addr_to_str(&nodemap->nodes[i].addr),
 554                                !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
 555                                !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
 556                                !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 557                                !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
 558                                !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED),
 559                                !!(nodemap->nodes[i].flags&NODE_FLAGS_INACTIVE),
 560                                partially_online);
 561                 }
 562                 return 0;
 563         }
 564
 565         printf("Number of nodes:%d\n", nodemap->num);
 566         for(i=0;i<nodemap->num;i++){
 567                 static const struct {
 568                         uint32_t flag;
 569                         const char *name;
 570                 } flag_names[] = {
 571                         { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 572                         { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 573                         { NODE_FLAGS_BANNED,                "BANNED" },
 574                         { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 575                         { NODE_FLAGS_DELETED,               "DELETED" },
 576                         { NODE_FLAGS_STOPPED,               "STOPPED" },
 577                         { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 578                 };
 579                 char *flags_str = NULL;
 580                 int j;
 581
 582                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 583                         continue;
 584                 }
 585                 if (nodemap->nodes[i].flags == 0) {
 586                         struct ctdb_control_get_ifaces *ifaces;
 587
 588                         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
 589                                                    nodemap->nodes[i].pnn,
 590                                                    ctdb, &ifaces);
 591                         if (ret == 0) {
 592                                 for (j=0; j < ifaces->num; j++) {
 593                                         if (ifaces->ifaces[j].link_state != 0) {
 594                                                 continue;
 595                                         }
 596                                         flags_str = talloc_strdup(ctdb, "PARTIALLYONLINE");
 597                                         break;
 598                                 }
 599                                 talloc_free(ifaces);
 600                         }
 601                 }
 602                 for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 603                         if (nodemap->nodes[i].flags & flag_names[j].flag) {
 604                                 if (flags_str == NULL) {
 605                                         flags_str = talloc_strdup(ctdb, flag_names[j].name);
 606                                 } else {
 607                                         flags_str = talloc_asprintf_append(flags_str, "|%s",
 608                                                                            flag_names[j].name);
 609                                 }
 610                                 CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
 611                         }
 612                 }
 613                 if (flags_str == NULL) {
 614                         flags_str = talloc_strdup(ctdb, "OK");
 615                         CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
 616                 }
 617                 printf("pnn:%d %-16s %s%s\n", nodemap->nodes[i].pnn,
 618                        ctdb_addr_to_str(&nodemap->nodes[i].addr),
 619                        flags_str,
 620                        nodemap->nodes[i].pnn == mypnn?" (THIS NODE)":"");
 621                 talloc_free(flags_str);
 622         }
 623
 624         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &vnnmap);
 625         if (ret != 0) {
 626                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
 627                 return ret;
 628         }
 629         if (vnnmap->generation == INVALID_GENERATION) {
 630                 printf("Generation:INVALID\n");
 631         } else {
 632                 printf("Generation:%d\n",vnnmap->generation);
 633         }
 634         printf("Size:%d\n",vnnmap->size);
 635         for(i=0;i<vnnmap->size;i++){
 636                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
 637         }
 638
 639         ret = ctdb_ctrl_getrecmode(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmode);
 640         if (ret != 0) {
 641                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
 642                 return ret;
 643         }
 644         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
 645
 646         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
 647         if (ret != 0) {
 648                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
 649                 return ret;
 650         }
 651         printf("Recovery master:%d\n",recmaster);
 652
 653         return 0;
 654 }
 655
 656
 657 struct natgw_node {
 658         struct natgw_node *next;
 659         const char *addr;
 660 };
 661
 662 /*
 663   display the list of nodes belonging to this natgw configuration
 664  */
 665 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
 666 {
 667         int i, ret;
 668         const char *natgw_list;
 669         int nlines;
 670         char **lines;
 671         struct natgw_node *natgw_nodes = NULL;
 672         struct natgw_node *natgw_node;
 673         struct ctdb_node_map *nodemap=NULL;
 674
 675
 676         /* read the natgw nodes file into a linked list */
 677         natgw_list = getenv("NATGW_NODES");
 678         if (natgw_list == NULL) {
 679                 natgw_list = "/etc/ctdb/natgw_nodes";
 680         }
 681         lines = file_lines_load(natgw_list, &nlines, ctdb);
 682         if (lines == NULL) {
 683                 ctdb_set_error(ctdb, "Failed to load natgw node list '%s'\n", natgw_list);
 684                 return -1;
 685         }
 686         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 687                 nlines--;
 688         }
 689         for (i=0;i<nlines;i++) {
 690                 char *node;
 691
 692                 node = lines[i];
 693                 /* strip leading spaces */
 694                 while((*node == ' ') || (*node == '\t')) {
 695                         node++;
 696                 }
 697                 if (*node == '#') {
 698                         continue;
 699                 }
 700                 if (strcmp(node, "") == 0) {
 701                         continue;
 702                 }
 703                 natgw_node = talloc(ctdb, struct natgw_node);
 704                 natgw_node->addr = talloc_strdup(natgw_node, node);
 705                 CTDB_NO_MEMORY(ctdb, natgw_node->addr);
 706                 natgw_node->next = natgw_nodes;
 707                 natgw_nodes = natgw_node;
 708         }
 709
 710         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
 711         if (ret != 0) {
 712                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
 713                 return ret;
 714         }
 715
 716         i=0;
 717         while(i<nodemap->num) {
 718                 for(natgw_node=natgw_nodes;natgw_node;natgw_node=natgw_node->next) {
 719                         if (!strcmp(natgw_node->addr, ctdb_addr_to_str(&nodemap->nodes[i].addr))) {
 720                                 break;
 721                         }
 722                 }
 723
 724                 /* this node was not in the natgw so we just remove it from
 725                  * the list
 726                  */
 727                 if ((natgw_node == NULL)
 728                 ||  (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) ) {
 729                         int j;
 730
 731                         for (j=i+1; j<nodemap->num; j++) {
 732                                 nodemap->nodes[j-1] = nodemap->nodes[j];
 733                         }
 734                         nodemap->num--;
 735                         continue;
 736                 }
 737
 738                 i++;
 739         }
 740
 741         /* pick a node to be natgwmaster
 742          * we dont allow STOPPED, DELETED, BANNED or UNHEALTHY nodes to become the natgwmaster
 743          */
 744         for(i=0;i<nodemap->num;i++){
 745                 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY))) {
 746                         printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
 747                         break;
 748                 }
 749         }
 750         /* we couldnt find any healthy node, try unhealthy ones */
 751         if (i == nodemap->num) {
 752                 for(i=0;i<nodemap->num;i++){
 753                         if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
 754                                 printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
 755                                 break;
 756                         }
 757                 }
 758         }
 759         /* unless all nodes are STOPPED, when we pick one anyway */
 760         if (i == nodemap->num) {
 761                 for(i=0;i<nodemap->num;i++){
 762                         if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED))) {
 763                                 printf("%d %s\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
 764                                 break;
 765                         }
 766                 }
 767                 /* or if we still can not find any */
 768                 if (i == nodemap->num) {
 769                         printf("-1 0.0.0.0\n");
 770                 }
 771         }
 772
 773         /* print the pruned list of nodes belonging to this natgw list */
 774         for(i=0;i<nodemap->num;i++){
 775                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 776                         continue;
 777                 }
 778                 printf(":%d:%s:%d:%d:%d:%d:%d\n", nodemap->nodes[i].pnn,
 779                         ctdb_addr_to_str(&nodemap->nodes[i].addr),
 780                        !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
 781                        !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
 782                        !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 783                        !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
 784                        !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED));
 785         }
 786
 787         return 0;
 788 }
 789
 790 /*
 791   display the status of the scripts for monitoring (or other events)
 792  */
 793 static int control_one_scriptstatus(struct ctdb_context *ctdb,
 794                                     enum ctdb_eventscript_call type)
 795 {
 796         struct ctdb_scripts_wire *script_status;
 797         int ret, i;
 798
 799         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
 800         if (ret != 0) {
 801                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
 802                 return ret;
 803         }
 804
 805         if (script_status == NULL) {
 806                 if (!options.machinereadable) {
 807                         printf("%s cycle never run\n",
 808                                ctdb_eventscript_call_names[type]);
 809                 }
 810                 return 0;
 811         }
 812
 813         if (!options.machinereadable) {
 814                 printf("%d scripts were executed last %s cycle\n",
 815                        script_status->num_scripts,
 816                        ctdb_eventscript_call_names[type]);
 817         }
 818         for (i=0; i<script_status->num_scripts; i++) {
 819                 const char *status = NULL;
 820
 821                 switch (script_status->scripts[i].status) {
 822                 case -ETIME:
 823                         status = "TIMEDOUT";
 824                         break;
 825                 case -ENOEXEC:
 826                         status = "DISABLED";
 827                         break;
 828                 case 0:
 829                         status = "OK";
 830                         break;
 831                 default:
 832                         if (script_status->scripts[i].status > 0)
 833                                 status = "ERROR";
 834                         break;
 835                 }
 836                 if (options.machinereadable) {
 837                         printf("%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
 838                                ctdb_eventscript_call_names[type],
 839                                script_status->scripts[i].name,
 840                                script_status->scripts[i].status,
 841                                status,
 842                                (long)script_status->scripts[i].start.tv_sec,
 843                                (long)script_status->scripts[i].start.tv_usec,
 844                                (long)script_status->scripts[i].finished.tv_sec,
 845                                (long)script_status->scripts[i].finished.tv_usec,
 846                                script_status->scripts[i].output);
 847                         continue;
 848                 }
 849                 if (status)
 850                         printf("%-20s Status:%s    ",
 851                                script_status->scripts[i].name, status);
 852                 else
 853                         /* Some other error, eg from stat. */
 854                         printf("%-20s Status:CANNOT RUN (%s)",
 855                                script_status->scripts[i].name,
 856                                strerror(-script_status->scripts[i].status));
 857
 858                 if (script_status->scripts[i].status >= 0) {
 859                         printf("Duration:%.3lf ",
 860                         timeval_delta(&script_status->scripts[i].finished,
 861                               &script_status->scripts[i].start));
 862                 }
 863                 if (script_status->scripts[i].status != -ENOEXEC) {
 864                         printf("%s",
 865                                ctime(&script_status->scripts[i].start.tv_sec));
 866                         if (script_status->scripts[i].status != 0) {
 867                                 printf("   OUTPUT:%s\n",
 868                                        script_status->scripts[i].output);
 869                         }
 870                 } else {
 871                         printf("\n");
 872                 }
 873         }
 874         return 0;
 875 }
 876
 877
 878 static int control_scriptstatus(struct ctdb_context *ctdb,
 879                                 int argc, const char **argv)
 880 {
 881         int ret;
 882         enum ctdb_eventscript_call type, min, max;
 883         const char *arg;
 884
 885         if (argc > 1) {
 886                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
 887                 return -1;
 888         }
 889
 890         if (argc == 0)
 891                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
 892         else
 893                 arg = argv[0];
 894
 895         for (type = 0; type < CTDB_EVENT_MAX; type++) {
 896                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
 897                         min = type;
 898                         max = type+1;
 899                         break;
 900                 }
 901         }
 902         if (type == CTDB_EVENT_MAX) {
 903                 if (strcmp(arg, "all") == 0) {
 904                         min = 0;
 905                         max = CTDB_EVENT_MAX;
 906                 } else {
 907                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
 908                         return -1;
 909                 }
 910         }
 911
 912         if (options.machinereadable) {
 913                 printf(":Type:Name:Code:Status:Start:End:Error Output...:\n");
 914         }
 915
 916         for (type = min; type < max; type++) {
 917                 ret = control_one_scriptstatus(ctdb, type);
 918                 if (ret != 0) {
 919                         return ret;
 920                 }
 921         }
 922
 923         return 0;
 924 }
 925
 926 /*
 927   enable an eventscript
 928  */
 929 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
 930 {
 931         int ret;
 932
 933         if (argc < 1) {
 934                 usage();
 935         }
 936
 937         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
 938         if (ret != 0) {
 939           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
 940                 return ret;
 941         }
 942
 943         return 0;
 944 }
 945
 946 /*
 947   disable an eventscript
 948  */
 949 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
 950 {
 951         int ret;
 952
 953         if (argc < 1) {
 954                 usage();
 955         }
 956
 957         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
 958         if (ret != 0) {
 959           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
 960                 return ret;
 961         }
 962
 963         return 0;
 964 }
 965
 966 /*
 967   display the pnn of the recovery master
 968  */
 969 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
 970 {
 971         int ret;
 972         uint32_t recmaster;
 973
 974         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
 975         if (ret != 0) {
 976                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
 977                 return ret;
 978         }
 979         printf("%d\n",recmaster);
 980
 981         return 0;
 982 }
 983
 984 /*
 985   add a tickle to a public address
 986  */
 987 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
 988 {
 989         struct ctdb_tcp_connection t;
 990         TDB_DATA data;
 991         int ret;
 992
 993         if (argc < 2) {
 994                 usage();
 995         }
 996
 997         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
 998                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
 999                 return -1;
1000         }
1001         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1002                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1003                 return -1;
1004         }
1005
1006         data.dptr = (uint8_t *)&t;
1007         data.dsize = sizeof(t);
1008
1009         /* tell all nodes about this tcp connection */
1010         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1011                            0, data, ctdb, NULL, NULL, NULL, NULL);
1012         if (ret != 0) {
1013                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1014                 return -1;
1015         }
1016
1017         return 0;
1018 }
1019
1020
1021 /*
1022   delete a tickle from a node
1023  */
1024 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1025 {
1026         struct ctdb_tcp_connection t;
1027         TDB_DATA data;
1028         int ret;
1029
1030         if (argc < 2) {
1031                 usage();
1032         }
1033
1034         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1035                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1036                 return -1;
1037         }
1038         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1039                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1040                 return -1;
1041         }
1042
1043         data.dptr = (uint8_t *)&t;
1044         data.dsize = sizeof(t);
1045
1046         /* tell all nodes about this tcp connection */
1047         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1048                            0, data, ctdb, NULL, NULL, NULL, NULL);
1049         if (ret != 0) {
1050                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1051                 return -1;
1052         }
1053
1054         return 0;
1055 }
1056
1057
1058 /*
1059   get a list of all tickles for this pnn
1060  */
1061 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1062 {
1063         struct ctdb_control_tcp_tickle_list *list;
1064         ctdb_sock_addr addr;
1065         int i, ret;
1066         unsigned port = 0;
1067
1068         if (argc < 1) {
1069                 usage();
1070         }
1071
1072         if (argc == 2) {
1073                 port = atoi(argv[1]);
1074         }
1075
1076         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1077                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1078                 return -1;
1079         }
1080
1081         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1082         if (ret == -1) {
1083                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1084                 return -1;
1085         }
1086
1087         if (options.machinereadable){
1088                 printf(":source ip:port:destination ip:port:\n");
1089                 for (i=0;i<list->tickles.num;i++) {
1090                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1091                                 continue;
1092                         }
1093                         printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1094                         printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1095                 }
1096         } else {
1097                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1098                 printf("Num tickles:%u\n", list->tickles.num);
1099                 for (i=0;i<list->tickles.num;i++) {
1100                         if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
1101                                 continue;
1102                         }
1103                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1104                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1105                 }
1106         }
1107
1108         talloc_free(list);
1109
1110         return 0;
1111 }
1112
1113
1114 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1115 {
1116         struct ctdb_all_public_ips *ips;
1117         struct ctdb_public_ip ip;
1118         int i, ret;
1119         uint32_t *nodes;
1120         uint32_t disable_time;
1121         TDB_DATA data;
1122         struct ctdb_node_map *nodemap=NULL;
1123         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1124
1125         disable_time = 30;
1126         data.dptr  = (uint8_t*)&disable_time;
1127         data.dsize = sizeof(disable_time);
1128         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1129         if (ret != 0) {
1130                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1131                 return -1;
1132         }
1133
1134
1135
1136         /* read the public ip list from the node */
1137         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1138         if (ret != 0) {
1139                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1140                 talloc_free(tmp_ctx);
1141                 return -1;
1142         }
1143
1144         for (i=0;i<ips->num;i++) {
1145                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1146                         break;
1147                 }
1148         }
1149         if (i==ips->num) {
1150                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1151                         pnn, ctdb_addr_to_str(addr)));
1152                 talloc_free(tmp_ctx);
1153                 return -1;
1154         }
1155
1156         ip.pnn  = pnn;
1157         ip.addr = *addr;
1158
1159         data.dptr  = (uint8_t *)&ip;
1160         data.dsize = sizeof(ip);
1161
1162         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1163         if (ret != 0) {
1164                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1165                 talloc_free(tmp_ctx);
1166                 return ret;
1167         }
1168
1169         nodes = list_of_active_nodes_except_pnn(ctdb, nodemap, tmp_ctx, pnn);
1170         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1171                                         nodes, 0,
1172                                         LONGTIMELIMIT(),
1173                                         false, data,
1174                                         NULL, NULL,
1175                                         NULL);
1176         if (ret != 0) {
1177                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1178                 talloc_free(tmp_ctx);
1179                 return -1;
1180         }
1181
1182         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1183         if (ret != 0) {
1184                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1185                 talloc_free(tmp_ctx);
1186                 return -1;
1187         }
1188
1189         /* update the recovery daemon so it now knows to expect the new
1190            node assignment for this ip.
1191         */
1192         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1193         if (ret != 0) {
1194                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1195                 return -1;
1196         }
1197
1198         talloc_free(tmp_ctx);
1199         return 0;
1200 }
1201
1202 /*
1203   move/failover an ip address to a specific node
1204  */
1205 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1206 {
1207         uint32_t pnn;
1208         ctdb_sock_addr addr;
1209
1210         if (argc < 2) {
1211                 usage();
1212                 return -1;
1213         }
1214
1215         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1216                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1217                 return -1;
1218         }
1219
1220
1221         if (sscanf(argv[1], "%u", &pnn) != 1) {
1222                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1223                 return -1;
1224         }
1225
1226         if (move_ip(ctdb, &addr, pnn) != 0) {
1227                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
1228                 return -1;
1229         }
1230
1231         return 0;
1232 }
1233
1234 void getips_store_callback(void *param, void *data)
1235 {
1236         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
1237         struct ctdb_all_public_ips *ips = param;
1238         int i;
1239
1240         i = ips->num++;
1241         ips->ips[i].pnn  = node_ip->pnn;
1242         ips->ips[i].addr = node_ip->addr;
1243 }
1244
1245 void getips_count_callback(void *param, void *data)
1246 {
1247         uint32_t *count = param;
1248
1249         (*count)++;
1250 }
1251
1252 #define IP_KEYLEN       4
1253 static uint32_t *ip_key(ctdb_sock_addr *ip)
1254 {
1255         static uint32_t key[IP_KEYLEN];
1256
1257         bzero(key, sizeof(key));
1258
1259         switch (ip->sa.sa_family) {
1260         case AF_INET:
1261                 key[0]  = ip->ip.sin_addr.s_addr;
1262                 break;
1263         case AF_INET6:
1264                 key[0]  = ip->ip6.sin6_addr.s6_addr32[3];
1265                 key[1]  = ip->ip6.sin6_addr.s6_addr32[2];
1266                 key[2]  = ip->ip6.sin6_addr.s6_addr32[1];
1267                 key[3]  = ip->ip6.sin6_addr.s6_addr32[0];
1268                 break;
1269         default:
1270                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1271                 return key;
1272         }
1273
1274         return key;
1275 }
1276
1277 static void *add_ip_callback(void *parm, void *data)
1278 {
1279         return parm;
1280 }
1281
1282 static int
1283 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
1284 {
1285         struct ctdb_all_public_ips *tmp_ips;
1286         struct ctdb_node_map *nodemap=NULL;
1287         trbt_tree_t *ip_tree;
1288         int i, j, len, ret;
1289         uint32_t count;
1290
1291         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1292         if (ret != 0) {
1293                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1294                 return ret;
1295         }
1296
1297         ip_tree = trbt_create(tmp_ctx, 0);
1298
1299         for(i=0;i<nodemap->num;i++){
1300                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1301                         continue;
1302                 }
1303                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1304                         continue;
1305                 }
1306
1307                 /* read the public ip list from this node */
1308                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
1309                 if (ret != 0) {
1310                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1311                         return -1;
1312                 }
1313
1314                 for (j=0; j<tmp_ips->num;j++) {
1315                         struct ctdb_public_ip *node_ip;
1316
1317                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
1318                         node_ip->pnn  = tmp_ips->ips[j].pnn;
1319                         node_ip->addr = tmp_ips->ips[j].addr;
1320
1321                         trbt_insertarray32_callback(ip_tree,
1322                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
1323                                 add_ip_callback,
1324                                 node_ip);
1325                 }
1326                 talloc_free(tmp_ips);
1327         }
1328
1329         /* traverse */
1330         count = 0;
1331         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
1332
1333         len = offsetof(struct ctdb_all_public_ips, ips) +
1334                 count*sizeof(struct ctdb_public_ip);
1335         tmp_ips = talloc_zero_size(tmp_ctx, len);
1336         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
1337
1338         *ips = tmp_ips;
1339
1340         return 0;
1341 }
1342
1343
1344 /*
1345  * scans all other nodes and returns a pnn for another node that can host this
1346  * ip address or -1
1347  */
1348 static int
1349 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1350 {
1351         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1352         struct ctdb_all_public_ips *ips;
1353         struct ctdb_node_map *nodemap=NULL;
1354         int i, j, ret;
1355
1356         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1357         if (ret != 0) {
1358                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1359                 talloc_free(tmp_ctx);
1360                 return ret;
1361         }
1362
1363         for(i=0;i<nodemap->num;i++){
1364                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1365                         continue;
1366                 }
1367                 if (nodemap->nodes[i].pnn == options.pnn) {
1368                         continue;
1369                 }
1370
1371                 /* read the public ip list from this node */
1372                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1373                 if (ret != 0) {
1374                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1375                         return -1;
1376                 }
1377
1378                 for (j=0;j<ips->num;j++) {
1379                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1380                                 talloc_free(tmp_ctx);
1381                                 return nodemap->nodes[i].pnn;
1382                         }
1383                 }
1384                 talloc_free(ips);
1385         }
1386
1387         talloc_free(tmp_ctx);
1388         return -1;
1389 }
1390
1391 /*
1392   add a public ip address to a node
1393  */
1394 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
1395 {
1396         int i, ret;
1397         int len;
1398         uint32_t pnn;
1399         unsigned mask;
1400         ctdb_sock_addr addr;
1401         struct ctdb_control_ip_iface *pub;
1402         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1403         struct ctdb_all_public_ips *ips;
1404
1405
1406         if (argc != 2) {
1407                 talloc_free(tmp_ctx);
1408                 usage();
1409         }
1410
1411         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
1412                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
1413                 talloc_free(tmp_ctx);
1414                 return -1;
1415         }
1416
1417         ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
1418         if (ret != 0) {
1419                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
1420                 talloc_free(tmp_ctx);
1421                 return ret;
1422         }
1423
1424
1425         /* check if some other node is already serving this ip, if not,
1426          * we will claim it
1427          */
1428         for (i=0;i<ips->num;i++) {
1429                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
1430                         break;
1431                 }
1432         }
1433
1434         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
1435         pub = talloc_size(tmp_ctx, len);
1436         CTDB_NO_MEMORY(ctdb, pub);
1437
1438         pub->addr  = addr;
1439         pub->mask  = mask;
1440         pub->len   = strlen(argv[1])+1;
1441         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
1442
1443         ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
1444         if (ret != 0) {
1445                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u\n", options.pnn));
1446                 talloc_free(tmp_ctx);
1447                 return ret;
1448         }
1449
1450         if (i == ips->num) {
1451                 /* no one has this ip so we claim it */
1452                 pnn  = options.pnn;
1453         } else {
1454                 pnn  = ips->ips[i].pnn;
1455         }
1456
1457         if (move_ip(ctdb, &addr, pnn) != 0) {
1458                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
1459                 return -1;
1460         }
1461
1462         talloc_free(tmp_ctx);
1463         return 0;
1464 }
1465
1466 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
1467
1468 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
1469 {
1470         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1471         struct ctdb_node_map *nodemap=NULL;
1472         struct ctdb_all_public_ips *ips;
1473         int ret, i, j;
1474
1475         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1476         if (ret != 0) {
1477                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
1478                 return ret;
1479         }
1480
1481         /* remove it from the nodes that are not hosting the ip currently */
1482         for(i=0;i<nodemap->num;i++){
1483                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1484                         continue;
1485                 }
1486                 if (ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips) != 0) {
1487                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
1488                         continue;
1489                 }
1490
1491                 for (j=0;j<ips->num;j++) {
1492                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1493                                 break;
1494                         }
1495                 }
1496                 if (j==ips->num) {
1497                         continue;
1498                 }
1499
1500                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
1501                         continue;
1502                 }
1503
1504                 options.pnn = nodemap->nodes[i].pnn;
1505                 control_delip(ctdb, argc, argv);
1506         }
1507
1508
1509         /* remove it from every node (also the one hosting it) */
1510         for(i=0;i<nodemap->num;i++){
1511                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1512                         continue;
1513                 }
1514                 if (ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips) != 0) {
1515                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
1516                         continue;
1517                 }
1518
1519                 for (j=0;j<ips->num;j++) {
1520                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1521                                 break;
1522                         }
1523                 }
1524                 if (j==ips->num) {
1525                         continue;
1526                 }
1527
1528                 options.pnn = nodemap->nodes[i].pnn;
1529                 control_delip(ctdb, argc, argv);
1530         }
1531
1532         talloc_free(tmp_ctx);
1533         return 0;
1534 }
1535
1536 /*
1537   delete a public ip address from a node
1538  */
1539 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
1540 {
1541         int i, ret;
1542         ctdb_sock_addr addr;
1543         struct ctdb_control_ip_iface pub;
1544         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1545         struct ctdb_all_public_ips *ips;
1546
1547         if (argc != 1) {
1548                 talloc_free(tmp_ctx);
1549                 usage();
1550         }
1551
1552         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1553                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1554                 return -1;
1555         }
1556
1557         if (options.pnn == CTDB_BROADCAST_ALL) {
1558                 return control_delip_all(ctdb, argc, argv, &addr);
1559         }
1560
1561         pub.addr  = addr;
1562         pub.mask  = 0;
1563         pub.len   = 0;
1564
1565         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
1566         if (ret != 0) {
1567                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
1568                 talloc_free(tmp_ctx);
1569                 return ret;
1570         }
1571
1572         for (i=0;i<ips->num;i++) {
1573                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
1574                         break;
1575                 }
1576         }
1577
1578         if (i==ips->num) {
1579                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
1580                         ctdb_addr_to_str(&addr)));
1581                 talloc_free(tmp_ctx);
1582                 return -1;
1583         }
1584
1585         if (ips->ips[i].pnn == options.pnn) {
1586                 ret = find_other_host_for_public_ip(ctdb, &addr);
1587                 if (ret != -1) {
1588                         if (move_ip(ctdb, &addr, ret) != 0) {
1589                                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", ret));
1590                                 return -1;
1591                         }
1592                 }
1593         }
1594
1595         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
1596         if (ret != 0) {
1597                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
1598                 talloc_free(tmp_ctx);
1599                 return ret;
1600         }
1601
1602         talloc_free(tmp_ctx);
1603         return 0;
1604 }
1605
1606 /*
1607   kill a tcp connection
1608  */
1609 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
1610 {
1611         int ret;
1612         struct ctdb_control_killtcp killtcp;
1613
1614         if (argc < 2) {
1615                 usage();
1616         }
1617
1618         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
1619                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
1620                 return -1;
1621         }
1622
1623         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
1624                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
1625                 return -1;
1626         }
1627
1628         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
1629         if (ret != 0) {
1630                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
1631                 return ret;
1632         }
1633
1634         return 0;
1635 }
1636
1637
1638 /*
1639   send a gratious arp
1640  */
1641 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
1642 {
1643         int ret;
1644         ctdb_sock_addr addr;
1645
1646         if (argc < 2) {
1647                 usage();
1648         }
1649
1650         if (!parse_ip(argv[0], NULL, 0, &addr)) {
1651                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
1652                 return -1;
1653         }
1654
1655         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
1656         if (ret != 0) {
1657                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
1658                 return ret;
1659         }
1660
1661         return 0;
1662 }
1663
1664 /*
1665   register a server id
1666  */
1667 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1668 {
1669         int ret;
1670         struct ctdb_server_id server_id;
1671
1672         if (argc < 3) {
1673                 usage();
1674         }
1675
1676         server_id.pnn       = strtoul(argv[0], NULL, 0);
1677         server_id.type      = strtoul(argv[1], NULL, 0);
1678         server_id.server_id = strtoul(argv[2], NULL, 0);
1679
1680         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
1681         if (ret != 0) {
1682                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
1683                 return ret;
1684         }
1685         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
1686         sleep(999);
1687         return -1;
1688 }
1689
1690 /*
1691   unregister a server id
1692  */
1693 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1694 {
1695         int ret;
1696         struct ctdb_server_id server_id;
1697
1698         if (argc < 3) {
1699                 usage();
1700         }
1701
1702         server_id.pnn       = strtoul(argv[0], NULL, 0);
1703         server_id.type      = strtoul(argv[1], NULL, 0);
1704         server_id.server_id = strtoul(argv[2], NULL, 0);
1705
1706         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
1707         if (ret != 0) {
1708                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
1709                 return ret;
1710         }
1711         return -1;
1712 }
1713
1714 /*
1715   check if a server id exists
1716  */
1717 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1718 {
1719         uint32_t status;
1720         int ret;
1721         struct ctdb_server_id server_id;
1722
1723         if (argc < 3) {
1724                 usage();
1725         }
1726
1727         server_id.pnn       = strtoul(argv[0], NULL, 0);
1728         server_id.type      = strtoul(argv[1], NULL, 0);
1729         server_id.server_id = strtoul(argv[2], NULL, 0);
1730
1731         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
1732         if (ret != 0) {
1733                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
1734                 return ret;
1735         }
1736
1737         if (status) {
1738                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
1739         } else {
1740                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
1741         }
1742         return 0;
1743 }
1744
1745 /*
1746   get a list of all server ids that are registered on a node
1747  */
1748 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
1749 {
1750         int i, ret;
1751         struct ctdb_server_id_list *server_ids;
1752
1753         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
1754         if (ret != 0) {
1755                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
1756                 return ret;
1757         }
1758
1759         for (i=0; i<server_ids->num; i++) {
1760                 printf("Server id %d:%d:%d\n",
1761                         server_ids->server_ids[i].pnn,
1762                         server_ids->server_ids[i].type,
1763                         server_ids->server_ids[i].server_id);
1764         }
1765
1766         return -1;
1767 }
1768
1769 /*
1770   send a tcp tickle ack
1771  */
1772 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
1773 {
1774         int ret;
1775         ctdb_sock_addr  src, dst;
1776
1777         if (argc < 2) {
1778                 usage();
1779         }
1780
1781         if (!parse_ip_port(argv[0], &src)) {
1782                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
1783                 return -1;
1784         }
1785
1786         if (!parse_ip_port(argv[1], &dst)) {
1787                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
1788                 return -1;
1789         }
1790
1791         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
1792         if (ret==0) {
1793                 return 0;
1794         }
1795         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
1796
1797         return -1;
1798 }
1799
1800
1801 /*
1802   display public ip status
1803  */
1804 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
1805 {
1806         int i, ret;
1807         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1808         struct ctdb_all_public_ips *ips;
1809
1810         if (options.pnn == CTDB_BROADCAST_ALL) {
1811                 /* read the list of public ips from all nodes */
1812                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
1813         } else {
1814                 /* read the public ip list from this node */
1815                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
1816         }
1817         if (ret != 0) {
1818                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
1819                 talloc_free(tmp_ctx);
1820                 return ret;
1821         }
1822
1823         if (options.machinereadable){
1824                 printf(":Public IP:Node:");
1825                 if (options.verbose){
1826                         printf("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
1827                 }
1828                 printf("\n");
1829         } else {
1830                 if (options.pnn == CTDB_BROADCAST_ALL) {
1831                         printf("Public IPs on ALL nodes\n");
1832                 } else {
1833                         printf("Public IPs on node %u\n", options.pnn);
1834                 }
1835         }
1836
1837         for (i=1;i<=ips->num;i++) {
1838                 struct ctdb_control_public_ip_info *info = NULL;
1839                 int32_t pnn;
1840                 char *aciface = NULL;
1841                 char *avifaces = NULL;
1842                 char *cifaces = NULL;
1843
1844                 if (options.pnn == CTDB_BROADCAST_ALL) {
1845                         pnn = ips->ips[ips->num-i].pnn;
1846                 } else {
1847                         pnn = options.pnn;
1848                 }
1849
1850                 if (pnn != -1) {
1851                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
1852                                                    &ips->ips[ips->num-i].addr, &info);
1853                 } else {
1854                         ret = -1;
1855                 }
1856
1857                 if (ret == 0) {
1858                         int j;
1859                         for (j=0; j < info->num; j++) {
1860                                 if (cifaces == NULL) {
1861                                         cifaces = talloc_strdup(info,
1862                                                                 info->ifaces[j].name);
1863                                 } else {
1864                                         cifaces = talloc_asprintf_append(cifaces,
1865                                                                          ",%s",
1866                                                                          info->ifaces[j].name);
1867                                 }
1868
1869                                 if (info->active_idx == j) {
1870                                         aciface = info->ifaces[j].name;
1871                                 }
1872
1873                                 if (info->ifaces[j].link_state == 0) {
1874                                         continue;
1875                                 }
1876
1877                                 if (avifaces == NULL) {
1878                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
1879                                 } else {
1880                                         avifaces = talloc_asprintf_append(avifaces,
1881                                                                           ",%s",
1882                                                                           info->ifaces[j].name);
1883                                 }
1884                         }
1885                 }
1886
1887                 if (options.machinereadable){
1888                         printf(":%s:%d:",
1889                                 ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
1890                                 ips->ips[ips->num-i].pnn);
1891                         if (options.verbose){
1892                                 printf("%s:%s:%s:",
1893                                         aciface?aciface:"",
1894                                         avifaces?avifaces:"",
1895                                         cifaces?cifaces:"");
1896                         }
1897                         printf("\n");
1898                 } else {
1899                         if (options.verbose) {
1900                                 printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
1901                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
1902                                         ips->ips[ips->num-i].pnn,
1903                                         aciface?aciface:"",
1904                                         avifaces?avifaces:"",
1905                                         cifaces?cifaces:"");
1906                         } else {
1907                                 printf("%s %d\n",
1908                                         ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
1909                                         ips->ips[ips->num-i].pnn);
1910                         }
1911                 }
1912                 talloc_free(info);
1913         }
1914
1915         talloc_free(tmp_ctx);
1916         return 0;
1917 }
1918
1919 /*
1920   public ip info
1921  */
1922 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
1923 {
1924         int i, ret;
1925         ctdb_sock_addr addr;
1926         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1927         struct ctdb_control_public_ip_info *info;
1928
1929         if (argc != 1) {
1930                 talloc_free(tmp_ctx);
1931                 usage();
1932         }
1933
1934         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1935                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1936                 return -1;
1937         }
1938
1939         /* read the public ip info from this node */
1940         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
1941                                            tmp_ctx, &addr, &info);
1942         if (ret != 0) {
1943                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
1944                                   argv[0], options.pnn));
1945                 talloc_free(tmp_ctx);
1946                 return ret;
1947         }
1948
1949         printf("Public IP[%s] info on node %u\n",
1950                ctdb_addr_to_str(&info->ip.addr),
1951                options.pnn);
1952
1953         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
1954                ctdb_addr_to_str(&info->ip.addr),
1955                info->ip.pnn, info->num);
1956
1957         for (i=0; i<info->num; i++) {
1958                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
1959
1960                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
1961                        i+1, info->ifaces[i].name,
1962                        info->ifaces[i].link_state?"up":"down",
1963                        (unsigned int)info->ifaces[i].references,
1964                        (i==info->active_idx)?" (active)":"");
1965         }
1966
1967         talloc_free(tmp_ctx);
1968         return 0;
1969 }
1970
1971 /*
1972   display interfaces status
1973  */
1974 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
1975 {
1976         int i, ret;
1977         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1978         struct ctdb_control_get_ifaces *ifaces;
1979
1980         /* read the public ip list from this node */
1981         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn,
1982                                    tmp_ctx, &ifaces);
1983         if (ret != 0) {
1984                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
1985                                   options.pnn));
1986                 talloc_free(tmp_ctx);
1987                 return ret;
1988         }
1989
1990         if (options.machinereadable){
1991                 printf(":Name:LinkStatus:References:\n");
1992         } else {
1993                 printf("Interfaces on node %u\n", options.pnn);
1994         }
1995
1996         for (i=0; i<ifaces->num; i++) {
1997                 if (options.machinereadable){
1998                         printf(":%s:%s:%u\n",
1999                                ifaces->ifaces[i].name,
2000                                ifaces->ifaces[i].link_state?"1":"0",
2001                                (unsigned int)ifaces->ifaces[i].references);
2002                 } else {
2003                         printf("name:%s link:%s references:%u\n",
2004                                ifaces->ifaces[i].name,
2005                                ifaces->ifaces[i].link_state?"up":"down",
2006                                (unsigned int)ifaces->ifaces[i].references);
2007                 }
2008         }
2009
2010         talloc_free(tmp_ctx);
2011         return 0;
2012 }
2013
2014
2015 /*
2016   set link status of an interface
2017  */
2018 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
2019 {
2020         int ret;
2021         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2022         struct ctdb_control_iface_info info;
2023
2024         ZERO_STRUCT(info);
2025
2026         if (argc != 2) {
2027                 usage();
2028         }
2029
2030         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
2031                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
2032                                   argv[0]));
2033                 talloc_free(tmp_ctx);
2034                 return -1;
2035         }
2036         strcpy(info.name, argv[0]);
2037
2038         if (strcmp(argv[1], "up") == 0) {
2039                 info.link_state = 1;
2040         } else if (strcmp(argv[1], "down") == 0) {
2041                 info.link_state = 0;
2042         } else {
2043                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
2044                                   argv[1]));
2045                 talloc_free(tmp_ctx);
2046                 return -1;
2047         }
2048
2049         /* read the public ip list from this node */
2050         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
2051                                    tmp_ctx, &info);
2052         if (ret != 0) {
2053                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
2054                                   argv[0], options.pnn));
2055                 talloc_free(tmp_ctx);
2056                 return ret;
2057         }
2058
2059         talloc_free(tmp_ctx);
2060         return 0;
2061 }
2062
2063 /*
2064   display pid of a ctdb daemon
2065  */
2066 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
2067 {
2068         uint32_t pid;
2069         int ret;
2070
2071         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
2072         if (ret != 0) {
2073                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
2074                 return ret;
2075         }
2076         printf("Pid:%d\n", pid);
2077
2078         return 0;
2079 }
2080
2081 static uint32_t ipreallocate_finished;
2082
2083 /*
2084   handler for receiving the response to ipreallocate
2085 */
2086 static void ip_reallocate_handler(struct ctdb_context *ctdb, uint64_t srvid,
2087                              TDB_DATA data, void *private_data)
2088 {
2089         ipreallocate_finished = 1;
2090 }
2091
2092 static void ctdb_every_second(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
2093 {
2094         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2095
2096         event_add_timed(ctdb->ev, ctdb,
2097                                 timeval_current_ofs(1, 0),
2098                                 ctdb_every_second, ctdb);
2099 }
2100
2101 /*
2102   ask the recovery daemon on the recovery master to perform a ip reallocation
2103  */
2104 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2105 {
2106         int i, ret;
2107         TDB_DATA data;
2108         struct takeover_run_reply rd;
2109         uint32_t recmaster;
2110         struct ctdb_node_map *nodemap=NULL;
2111         int retries=0;
2112         struct timeval tv = timeval_current();
2113
2114         /* we need some events to trigger so we can timeout and restart
2115            the loop
2116         */
2117         event_add_timed(ctdb->ev, ctdb,
2118                                 timeval_current_ofs(1, 0),
2119                                 ctdb_every_second, ctdb);
2120
2121         rd.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
2122         if (rd.pnn == -1) {
2123                 DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
2124                 return -1;
2125         }
2126         rd.srvid = getpid();
2127
2128         /* register a message port for receiveing the reply so that we
2129            can receive the reply
2130         */
2131         ctdb_client_set_message_handler(ctdb, rd.srvid, ip_reallocate_handler, NULL);
2132
2133         data.dptr = (uint8_t *)&rd;
2134         data.dsize = sizeof(rd);
2135
2136 again:
2137         /* check that there are valid nodes available */
2138         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap) != 0) {
2139                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2140                 return -1;
2141         }
2142         for (i=0; i<nodemap->num;i++) {
2143                 if ((nodemap->nodes[i].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) == 0) {
2144                         break;
2145                 }
2146         }
2147         if (i==nodemap->num) {
2148                 DEBUG(DEBUG_ERR,("No recmaster available, no need to wait for cluster convergence\n"));
2149                 return 0;
2150         }
2151
2152
2153         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
2154         if (ret != 0) {
2155                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
2156                 return ret;
2157         }
2158
2159         /* verify the node exists */
2160         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), recmaster, ctdb, &nodemap) != 0) {
2161                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2162                 return -1;
2163         }
2164
2165
2166         /* check tha there are nodes available that can act as a recmaster */
2167         for (i=0; i<nodemap->num; i++) {
2168                 if (nodemap->nodes[i].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2169                         continue;
2170                 }
2171                 break;
2172         }
2173         if (i == nodemap->num) {
2174                 DEBUG(DEBUG_ERR,("No possible nodes to host addresses.\n"));
2175                 return 0;
2176         }
2177
2178         /* verify the recovery master is not STOPPED, nor BANNED */
2179         if (nodemap->nodes[recmaster].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2180                 DEBUG(DEBUG_ERR,("No suitable recmaster found. Try again\n"));
2181                 retries++;
2182                 sleep(1);
2183                 goto again;
2184         }
2185
2186
2187         /* verify the recovery master is not STOPPED, nor BANNED */
2188         if (nodemap->nodes[recmaster].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2189                 DEBUG(DEBUG_ERR,("No suitable recmaster found. Try again\n"));
2190                 retries++;
2191                 sleep(1);
2192                 goto again;
2193         }
2194
2195         ipreallocate_finished = 0;
2196         ret = ctdb_client_send_message(ctdb, recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
2197         if (ret != 0) {
2198                 DEBUG(DEBUG_ERR,("Failed to send ip takeover run request message to %u\n", options.pnn));
2199                 return -1;
2200         }
2201
2202         tv = timeval_current();
2203         /* this loop will terminate when we have received the reply */
2204         while (timeval_elapsed(&tv) < 3.0) {
2205                 event_loop_once(ctdb->ev);
2206         }
2207         if (ipreallocate_finished == 1) {
2208                 return 0;
2209         }
2210
2211         DEBUG(DEBUG_ERR,("Timed out waiting for recmaster ipreallocate. Trying again\n"));
2212         retries++;
2213         sleep(1);
2214         goto again;
2215
2216         return 0;
2217 }
2218
2219
2220 /*
2221   disable a remote node
2222  */
2223 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
2224 {
2225         int ret;
2226         struct ctdb_node_map *nodemap=NULL;
2227
2228         /* check if the node is already disabled */
2229         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2230                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2231                 exit(10);
2232         }
2233         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2234                 DEBUG(DEBUG_ERR,("Node %d is already disabled.\n", options.pnn));
2235                 return 0;
2236         }
2237
2238         do {
2239                 ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn, NODE_FLAGS_PERMANENTLY_DISABLED, 0);
2240                 if (ret != 0) {
2241                         DEBUG(DEBUG_ERR, ("Unable to disable node %u\n", options.pnn));
2242                         return ret;
2243                 }
2244
2245                 sleep(1);
2246
2247                 /* read the nodemap and verify the change took effect */
2248                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2249                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2250                         exit(10);
2251                 }
2252
2253         } while (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED));
2254         ret = control_ipreallocate(ctdb, argc, argv);
2255         if (ret != 0) {
2256                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2257                 return ret;
2258         }
2259
2260         return 0;
2261 }
2262
2263 /*
2264   enable a disabled remote node
2265  */
2266 static int control_enable(struct ctdb_context *ctdb, int argc, const char **argv)
2267 {
2268         int ret;
2269
2270         struct ctdb_node_map *nodemap=NULL;
2271
2272
2273         /* check if the node is already enabled */
2274         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2275                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2276                 exit(10);
2277         }
2278         if (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED)) {
2279                 DEBUG(DEBUG_ERR,("Node %d is already enabled.\n", options.pnn));
2280                 return 0;
2281         }
2282
2283         do {
2284                 ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn, 0, NODE_FLAGS_PERMANENTLY_DISABLED);
2285                 if (ret != 0) {
2286                         DEBUG(DEBUG_ERR, ("Unable to enable node %u\n", options.pnn));
2287                         return ret;
2288                 }
2289
2290                 sleep(1);
2291
2292                 /* read the nodemap and verify the change took effect */
2293                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2294                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2295                         exit(10);
2296                 }
2297
2298         } while (nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED);
2299
2300         ret = control_ipreallocate(ctdb, argc, argv);
2301         if (ret != 0) {
2302                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2303                 return ret;
2304         }
2305
2306         return 0;
2307 }
2308
2309 /*
2310   stop a remote node
2311  */
2312 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
2313 {
2314         int ret;
2315         struct ctdb_node_map *nodemap=NULL;
2316
2317         do {
2318                 ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
2319                 if (ret != 0) {
2320                         DEBUG(DEBUG_ERR, ("Unable to stop node %u   try again\n", options.pnn));
2321                 }
2322
2323                 sleep(1);
2324
2325                 /* read the nodemap and verify the change took effect */
2326                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2327                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2328                         exit(10);
2329                 }
2330
2331         } while (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_STOPPED));
2332         ret = control_ipreallocate(ctdb, argc, argv);
2333         if (ret != 0) {
2334                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2335                 return ret;
2336         }
2337
2338         return 0;
2339 }
2340
2341 /*
2342   restart a stopped remote node
2343  */
2344 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
2345 {
2346         int ret;
2347
2348         struct ctdb_node_map *nodemap=NULL;
2349
2350         do {
2351                 ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
2352                 if (ret != 0) {
2353                         DEBUG(DEBUG_ERR, ("Unable to continue node %u\n", options.pnn));
2354                         return ret;
2355                 }
2356
2357                 sleep(1);
2358
2359                 /* read the nodemap and verify the change took effect */
2360                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2361                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2362                         exit(10);
2363                 }
2364
2365         } while (nodemap->nodes[options.pnn].flags & NODE_FLAGS_STOPPED);
2366         ret = control_ipreallocate(ctdb, argc, argv);
2367         if (ret != 0) {
2368                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2369                 return ret;
2370         }
2371
2372         return 0;
2373 }
2374
2375 static uint32_t get_generation(struct ctdb_context *ctdb)
2376 {
2377         struct ctdb_vnn_map *vnnmap=NULL;
2378         int ret;
2379
2380         /* wait until the recmaster is not in recovery mode */
2381         while (1) {
2382                 uint32_t recmode, recmaster;
2383
2384                 if (vnnmap != NULL) {
2385                         talloc_free(vnnmap);
2386                         vnnmap = NULL;
2387                 }
2388
2389                 /* get the recmaster */
2390                 ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
2391                 if (ret != 0) {
2392                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
2393                         exit(10);
2394                 }
2395
2396                 /* get recovery mode */
2397                 ret = ctdb_ctrl_getrecmode(ctdb, ctdb, TIMELIMIT(), recmaster, &recmode);
2398                 if (ret != 0) {
2399                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
2400                         exit(10);
2401                 }
2402
2403                 /* get the current generation number */
2404                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, ctdb, &vnnmap);
2405                 if (ret != 0) {
2406                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
2407                         exit(10);
2408                 }
2409
2410                 if ((recmode == CTDB_RECOVERY_NORMAL)
2411                 &&  (vnnmap->generation != 1)){
2412                         return vnnmap->generation;
2413                 }
2414                 sleep(1);
2415         }
2416 }
2417
2418 /*
2419   ban a node from the cluster
2420  */
2421 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
2422 {
2423         int ret;
2424         struct ctdb_node_map *nodemap=NULL;
2425         struct ctdb_ban_time bantime;
2426
2427         if (argc < 1) {
2428                 usage();
2429         }
2430
2431         /* verify the node exists */
2432         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2433         if (ret != 0) {
2434                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2435                 return ret;
2436         }
2437
2438         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_BANNED) {
2439                 DEBUG(DEBUG_ERR,("Node %u is already banned.\n", options.pnn));
2440                 return -1;
2441         }
2442
2443         bantime.pnn  = options.pnn;
2444         bantime.time = strtoul(argv[0], NULL, 0);
2445
2446         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, &bantime);
2447         if (ret != 0) {
2448                 DEBUG(DEBUG_ERR,("Banning node %d for %d seconds failed.\n", bantime.pnn, bantime.time));
2449                 return -1;
2450         }
2451
2452         ret = control_ipreallocate(ctdb, argc, argv);
2453         if (ret != 0) {
2454                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2455                 return ret;
2456         }
2457
2458         return 0;
2459 }
2460
2461
2462 /*
2463   unban a node from the cluster
2464  */
2465 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
2466 {
2467         int ret;
2468         struct ctdb_node_map *nodemap=NULL;
2469         struct ctdb_ban_time bantime;
2470
2471         /* verify the node exists */
2472         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2473         if (ret != 0) {
2474                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2475                 return ret;
2476         }
2477
2478         if (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_BANNED)) {
2479                 DEBUG(DEBUG_ERR,("Node %u is not banned.\n", options.pnn));
2480                 return -1;
2481         }
2482
2483         bantime.pnn  = options.pnn;
2484         bantime.time = 0;
2485
2486         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, &bantime);
2487         if (ret != 0) {
2488                 DEBUG(DEBUG_ERR,("Unbanning node %d failed.\n", bantime.pnn));
2489                 return -1;
2490         }
2491
2492         ret = control_ipreallocate(ctdb, argc, argv);
2493         if (ret != 0) {
2494                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2495                 return ret;
2496         }
2497
2498         return 0;
2499 }
2500
2501
2502 /*
2503   show ban information for a node
2504  */
2505 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
2506 {
2507         int ret;
2508         struct ctdb_node_map *nodemap=NULL;
2509         struct ctdb_ban_time *bantime;
2510
2511         /* verify the node exists */
2512         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2513         if (ret != 0) {
2514                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2515                 return ret;
2516         }
2517
2518         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
2519         if (ret != 0) {
2520                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
2521                 return -1;
2522         }
2523
2524         if (bantime->time == 0) {
2525                 printf("Node %u is not banned\n", bantime->pnn);
2526         } else {
2527                 printf("Node %u is banned banned for %d seconds\n", bantime->pnn, bantime->time);
2528         }
2529
2530         return 0;
2531 }
2532
2533 /*
2534   shutdown a daemon
2535  */
2536 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
2537 {
2538         int ret;
2539
2540         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
2541         if (ret != 0) {
2542                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
2543                 return ret;
2544         }
2545
2546         return 0;
2547 }
2548
2549 /*
2550   trigger a recovery
2551  */
2552 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
2553 {
2554         int ret;
2555         uint32_t generation, next_generation;
2556
2557         /* record the current generation number */
2558         generation = get_generation(ctdb);
2559
2560         ret = ctdb_ctrl_freeze_priority(ctdb, TIMELIMIT(), options.pnn, 1);
2561         if (ret != 0) {
2562                 DEBUG(DEBUG_ERR, ("Unable to freeze node\n"));
2563                 return ret;
2564         }
2565
2566         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
2567         if (ret != 0) {
2568                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
2569                 return ret;
2570         }
2571
2572         /* wait until we are in a new generation */
2573         while (1) {
2574                 next_generation = get_generation(ctdb);
2575                 if (next_generation != generation) {
2576                         return 0;
2577                 }
2578                 sleep(1);
2579         }
2580
2581         return 0;
2582 }
2583
2584
2585 /*
2586   display monitoring mode of a remote node
2587  */
2588 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
2589 {
2590         uint32_t monmode;
2591         int ret;
2592
2593         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
2594         if (ret != 0) {
2595                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
2596                 return ret;
2597         }
2598         if (!options.machinereadable){
2599                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
2600         } else {
2601                 printf(":mode:\n");
2602                 printf(":%d:\n",monmode);
2603         }
2604         return 0;
2605 }
2606
2607
2608 /*
2609   display capabilities of a remote node
2610  */
2611 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
2612 {
2613         uint32_t capabilities;
2614         int ret;
2615
2616         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
2617         if (ret != 0) {
2618                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
2619                 return ret;
2620         }
2621
2622         if (!options.machinereadable){
2623                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
2624                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
2625                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
2626                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
2627         } else {
2628                 printf(":RECMASTER:LMASTER:LVS:NATGW:\n");
2629                 printf(":%d:%d:%d:%d:\n",
2630                         !!(capabilities&CTDB_CAP_RECMASTER),
2631                         !!(capabilities&CTDB_CAP_LMASTER),
2632                         !!(capabilities&CTDB_CAP_LVS),
2633                         !!(capabilities&CTDB_CAP_NATGW));
2634         }
2635         return 0;
2636 }
2637
2638 /*
2639   display lvs configuration
2640  */
2641 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
2642 {
2643         uint32_t *capabilities;
2644         struct ctdb_node_map *nodemap=NULL;
2645         int i, ret;
2646         int healthy_count = 0;
2647
2648         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
2649         if (ret != 0) {
2650                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2651                 return ret;
2652         }
2653
2654         capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
2655         CTDB_NO_MEMORY(ctdb, capabilities);
2656
2657         /* collect capabilities for all connected nodes */
2658         for (i=0; i<nodemap->num; i++) {
2659                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2660                         continue;
2661                 }
2662                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2663                         continue;
2664                 }
2665
2666                 ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
2667                 if (ret != 0) {
2668                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
2669                         return ret;
2670                 }
2671
2672                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2673                         continue;
2674                 }
2675
2676                 if (!(nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY)) {
2677                         healthy_count++;
2678                 }
2679         }
2680
2681         /* Print all LVS nodes */
2682         for (i=0; i<nodemap->num; i++) {
2683                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2684                         continue;
2685                 }
2686                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2687                         continue;
2688                 }
2689                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2690                         continue;
2691                 }
2692
2693                 if (healthy_count != 0) {
2694                         if (nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY) {
2695                                 continue;
2696                         }
2697                 }
2698
2699                 printf("%d:%s\n", i,
2700                         ctdb_addr_to_str(&nodemap->nodes[i].addr));
2701         }
2702
2703         return 0;
2704 }
2705
2706 /*
2707   display who is the lvs master
2708  */
2709 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
2710 {
2711         uint32_t *capabilities;
2712         struct ctdb_node_map *nodemap=NULL;
2713         int i, ret;
2714         int healthy_count = 0;
2715
2716         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
2717         if (ret != 0) {
2718                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2719                 return ret;
2720         }
2721
2722         capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
2723         CTDB_NO_MEMORY(ctdb, capabilities);
2724
2725         /* collect capabilities for all connected nodes */
2726         for (i=0; i<nodemap->num; i++) {
2727                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2728                         continue;
2729                 }
2730                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2731                         continue;
2732                 }
2733
2734                 ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
2735                 if (ret != 0) {
2736                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
2737                         return ret;
2738                 }
2739
2740                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2741                         continue;
2742                 }
2743
2744                 if (!(nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY)) {
2745                         healthy_count++;
2746                 }
2747         }
2748
2749         /* find and show the lvsmaster */
2750         for (i=0; i<nodemap->num; i++) {
2751                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2752                         continue;
2753                 }
2754                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2755                         continue;
2756                 }
2757                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2758                         continue;
2759                 }
2760
2761                 if (healthy_count != 0) {
2762                         if (nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY) {
2763                                 continue;
2764                         }
2765                 }
2766
2767                 if (options.machinereadable){
2768                         printf("%d\n", i);
2769                 } else {
2770                         printf("Node %d is LVS master\n", i);
2771                 }
2772                 return 0;
2773         }
2774
2775         printf("There is no LVS master\n");
2776         return -1;
2777 }
2778
2779 /*
2780   disable monitoring on a  node
2781  */
2782 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
2783 {
2784
2785         int ret;
2786
2787         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
2788         if (ret != 0) {
2789                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
2790                 return ret;
2791         }
2792         printf("Monitoring mode:%s\n","DISABLED");
2793
2794         return 0;
2795 }
2796
2797 /*
2798   enable monitoring on a  node
2799  */
2800 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
2801 {
2802
2803         int ret;
2804
2805         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
2806         if (ret != 0) {
2807                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
2808                 return ret;
2809         }
2810         printf("Monitoring mode:%s\n","ACTIVE");
2811
2812         return 0;
2813 }
2814
2815 /*
2816   display remote list of keys/data for a db
2817  */
2818 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
2819 {
2820         const char *db_name;
2821         struct ctdb_db_context *ctdb_db;
2822         int ret;
2823
2824         if (argc < 1) {
2825                 usage();
2826         }
2827
2828         db_name = argv[0];
2829
2830
2831         if (db_exists(ctdb, db_name)) {
2832                 DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
2833                 return -1;
2834         }
2835
2836         ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
2837
2838         if (ctdb_db == NULL) {
2839                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
2840                 return -1;
2841         }
2842
2843         /* traverse and dump the cluster tdb */
2844         ret = ctdb_dump_db(ctdb_db, stdout);
2845         if (ret == -1) {
2846                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
2847                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
2848                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
2849                                   db_name));
2850                 return -1;
2851         }
2852         talloc_free(ctdb_db);
2853
2854         printf("Dumped %d records\n", ret);
2855         return 0;
2856 }
2857
2858
2859 /*
2860   fetch a record from a persistent database
2861  */
2862 static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
2863 {
2864         const char *db_name;
2865         struct ctdb_db_context *ctdb_db;
2866         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2867         struct ctdb_transaction_handle *h;
2868         TDB_DATA key, data;
2869         int fd, ret;
2870
2871         if (argc < 2) {
2872                 talloc_free(tmp_ctx);
2873                 usage();
2874         }
2875
2876         db_name = argv[0];
2877
2878
2879         if (db_exists(ctdb, db_name)) {
2880                 DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
2881                 talloc_free(tmp_ctx);
2882                 return -1;
2883         }
2884
2885         ctdb_db = ctdb_attach(ctdb, db_name, true, 0);
2886
2887         if (ctdb_db == NULL) {
2888                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
2889                 talloc_free(tmp_ctx);
2890                 return -1;
2891         }
2892
2893         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
2894         if (h == NULL) {
2895                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
2896                 talloc_free(tmp_ctx);
2897                 return -1;
2898         }
2899
2900         key.dptr  = discard_const(argv[1]);
2901         key.dsize = strlen(argv[1]);
2902         ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
2903         if (ret != 0) {
2904                 DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
2905                 talloc_free(tmp_ctx);
2906                 return -1;
2907         }
2908
2909         if (data.dsize == 0 || data.dptr == NULL) {
2910                 DEBUG(DEBUG_ERR,("Record is empty\n"));
2911                 talloc_free(tmp_ctx);
2912                 return -1;
2913         }
2914
2915         if (argc == 3) {
2916           fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
2917                 if (fd == -1) {
2918                         DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
2919                         talloc_free(tmp_ctx);
2920                         return -1;
2921                 }
2922                 write(fd, data.dptr, data.dsize);
2923                 close(fd);
2924         } else {
2925                 write(1, data.dptr, data.dsize);
2926         }
2927
2928         /* abort the transaction */
2929         talloc_free(h);
2930
2931
2932         talloc_free(tmp_ctx);
2933         return 0;
2934 }
2935
2936 /*
2937   write a record to a persistent database
2938  */
2939 static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
2940 {
2941         const char *db_name;
2942         struct ctdb_db_context *ctdb_db;
2943         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2944         struct ctdb_transaction_handle *h;
2945         struct stat st;
2946         TDB_DATA key, data;
2947         int fd, ret;
2948
2949         if (argc < 3) {
2950                 talloc_free(tmp_ctx);
2951                 usage();
2952         }
2953
2954         fd = open(argv[2], O_RDONLY);
2955         if (fd == -1) {
2956                 DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
2957                 talloc_free(tmp_ctx);
2958                 return -1;
2959         }
2960
2961         ret = fstat(fd, &st);
2962         if (ret == -1) {
2963                 DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
2964                 close(fd);
2965                 talloc_free(tmp_ctx);
2966                 return -1;
2967         }
2968
2969         if (!S_ISREG(st.st_mode)) {
2970                 DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
2971                 close(fd);
2972                 talloc_free(tmp_ctx);
2973                 return -1;
2974         }
2975
2976         data.dsize = st.st_size;
2977         if (data.dsize == 0) {
2978                 data.dptr  = NULL;
2979         } else {
2980                 data.dptr = talloc_size(tmp_ctx, data.dsize);
2981                 if (data.dptr == NULL) {
2982                         DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
2983                         close(fd);
2984                         talloc_free(tmp_ctx);
2985                         return -1;
2986                 }
2987                 ret = read(fd, data.dptr, data.dsize);
2988                 if (ret != data.dsize) {
2989                         DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
2990                         close(fd);
2991                         talloc_free(tmp_ctx);
2992                         return -1;
2993                 }
2994         }
2995         close(fd);
2996
2997
2998         db_name = argv[0];
2999
3000         ctdb_db = ctdb_attach(ctdb, db_name, true, 0);
3001
3002         if (ctdb_db == NULL) {
3003                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3004                 talloc_free(tmp_ctx);
3005                 return -1;
3006         }
3007
3008         h = ctdb_transaction_start(ctdb_db, tmp_ctx);
3009         if (h == NULL) {
3010                 DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
3011                 talloc_free(tmp_ctx);
3012                 return -1;
3013         }
3014
3015         key.dptr  = discard_const(argv[1]);
3016         key.dsize = strlen(argv[1]);
3017         ret = ctdb_transaction_store(h, key, data);
3018         if (ret != 0) {
3019                 DEBUG(DEBUG_ERR,("Failed to store record\n"));
3020                 talloc_free(tmp_ctx);
3021                 return -1;
3022         }
3023
3024         ret = ctdb_transaction_commit(h);
3025         if (ret != 0) {
3026                 DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
3027                 talloc_free(tmp_ctx);
3028                 return -1;
3029         }
3030
3031
3032         talloc_free(tmp_ctx);
3033         return 0;
3034 }
3035
3036 static void log_handler(struct ctdb_context *ctdb, uint64_t srvid,
3037                              TDB_DATA data, void *private_data)
3038 {
3039         DEBUG(DEBUG_ERR,("Log data received\n"));
3040         if (data.dsize > 0) {
3041                 printf("%s", data.dptr);
3042         }
3043
3044         exit(0);
3045 }
3046
3047 /*
3048   display a list of log messages from the in memory ringbuffer
3049  */
3050 static int control_getlog(struct ctdb_context *ctdb, int argc, const char **argv)
3051 {
3052         int ret;
3053         int32_t res;
3054         struct ctdb_get_log_addr log_addr;
3055         TDB_DATA data;
3056         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3057         char *errmsg;
3058         struct timeval tv;
3059
3060         if (argc != 1) {
3061                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3062                 talloc_free(tmp_ctx);
3063                 return -1;
3064         }
3065
3066         log_addr.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
3067         log_addr.srvid = getpid();
3068         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
3069                 log_addr.level = get_debug_by_desc(argv[0]);
3070         } else {
3071                 log_addr.level = strtol(argv[0], NULL, 0);
3072         }
3073
3074
3075         data.dptr = (unsigned char *)&log_addr;
3076         data.dsize = sizeof(log_addr);
3077
3078         DEBUG(DEBUG_ERR, ("Pulling logs from node %u\n", options.pnn));
3079
3080         ctdb_client_set_message_handler(ctdb, log_addr.srvid, log_handler, NULL);
3081         sleep(1);
3082
3083         DEBUG(DEBUG_ERR,("Listen for response on %d\n", (int)log_addr.srvid));
3084
3085         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_GET_LOG,
3086                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
3087         if (ret != 0 || res != 0) {
3088                 DEBUG(DEBUG_ERR,("Failed to get logs - %s\n", errmsg));
3089                 talloc_free(tmp_ctx);
3090                 return -1;
3091         }
3092
3093
3094         tv = timeval_current();
3095         /* this loop will terminate when we have received the reply */
3096         while (timeval_elapsed(&tv) < 3.0) {
3097                 event_loop_once(ctdb->ev);
3098         }
3099
3100         DEBUG(DEBUG_INFO,("Timed out waiting for log data.\n"));
3101
3102         talloc_free(tmp_ctx);
3103         return 0;
3104 }
3105
3106 /*
3107   clear the in memory log area
3108  */
3109 static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **argv)
3110 {
3111         int ret;
3112         int32_t res;
3113         char *errmsg;
3114         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3115
3116         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_CLEAR_LOG,
3117                            0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
3118         if (ret != 0 || res != 0) {
3119                 DEBUG(DEBUG_ERR,("Failed to clear logs\n"));
3120                 talloc_free(tmp_ctx);
3121                 return -1;
3122         }
3123
3124         talloc_free(tmp_ctx);
3125         return 0;
3126 }
3127
3128
3129
3130 /*
3131   display a list of the databases on a remote ctdb
3132  */
3133 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
3134 {
3135         int i, ret;
3136         struct ctdb_dbid_map *dbmap=NULL;
3137
3138         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
3139         if (ret != 0) {
3140                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
3141                 return ret;
3142         }
3143
3144         if(options.machinereadable){
3145                 printf(":ID:Name:Path:Persistent:Unhealthy:\n");
3146                 for(i=0;i<dbmap->num;i++){
3147                         const char *path;
3148                         const char *name;
3149                         const char *health;
3150                         bool persistent;
3151
3152                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
3153                                             dbmap->dbs[i].dbid, ctdb, &path);
3154                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
3155                                             dbmap->dbs[i].dbid, ctdb, &name);
3156                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
3157                                               dbmap->dbs[i].dbid, ctdb, &health);
3158                         persistent = dbmap->dbs[i].persistent;
3159                         printf(":0x%08X:%s:%s:%d:%d:\n",
3160                                dbmap->dbs[i].dbid, name, path,
3161                                !!(persistent), !!(health));
3162                 }
3163                 return 0;
3164         }
3165
3166         printf("Number of databases:%d\n", dbmap->num);
3167         for(i=0;i<dbmap->num;i++){
3168                 const char *path;
3169                 const char *name;
3170                 const char *health;
3171                 bool persistent;
3172
3173                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
3174                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
3175                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
3176                 persistent = dbmap->dbs[i].persistent;
3177                 printf("dbid:0x%08x name:%s path:%s%s%s\n",
3178                        dbmap->dbs[i].dbid, name, path,
3179                        persistent?" PERSISTENT":"",
3180                        health?" UNHEALTHY":"");
3181         }
3182
3183         return 0;
3184 }
3185
3186 /*
3187   display the status of a database on a remote ctdb
3188  */
3189 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
3190 {
3191         int i, ret;
3192         struct ctdb_dbid_map *dbmap=NULL;
3193         const char *db_name;
3194
3195         if (argc < 1) {
3196                 usage();
3197         }
3198
3199         db_name = argv[0];
3200
3201         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
3202         if (ret != 0) {
3203                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
3204                 return ret;
3205         }
3206
3207         for(i=0;i<dbmap->num;i++){
3208                 const char *path;
3209                 const char *name;
3210                 const char *health;
3211                 bool persistent;
3212
3213                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
3214                 if (strcmp(name, db_name) != 0) {
3215                         continue;
3216                 }
3217
3218                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
3219                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
3220                 persistent = dbmap->dbs[i].persistent;
3221                 printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nHEALTH: %s\n",
3222                        dbmap->dbs[i].dbid, name, path,
3223                        persistent?"yes":"no",
3224                        health?health:"OK");
3225                 return 0;
3226         }
3227
3228         DEBUG(DEBUG_ERR, ("db %s doesn't exist on node %u\n", db_name, options.pnn));
3229         return 0;
3230 }
3231
3232 /*
3233   check if the local node is recmaster or not
3234   it will return 1 if this node is the recmaster and 0 if it is not
3235   or if the local ctdb daemon could not be contacted
3236  */
3237 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3238 {
3239         uint32_t mypnn, recmaster;
3240         int ret;
3241
3242         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
3243         if (mypnn == -1) {
3244                 printf("Failed to get pnn of node\n");
3245                 return 1;
3246         }
3247
3248         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
3249         if (ret != 0) {
3250                 printf("Failed to get the recmaster\n");
3251                 return 1;
3252         }
3253
3254         if (recmaster != mypnn) {
3255                 printf("this node is not the recmaster\n");
3256                 return 1;
3257         }
3258
3259         printf("this node is the recmaster\n");
3260         return 0;
3261 }
3262
3263 /*
3264   ping a node
3265  */
3266 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
3267 {
3268         int ret;
3269         struct timeval tv = timeval_current();
3270         ret = ctdb_ctrl_ping(ctdb, options.pnn);
3271         if (ret == -1) {
3272                 printf("Unable to get ping response from node %u\n", options.pnn);
3273                 return -1;
3274         } else {
3275                 printf("response from %u time=%.6f sec  (%d clients)\n",
3276                        options.pnn, timeval_elapsed(&tv), ret);
3277         }
3278         return 0;
3279 }
3280
3281
3282 /*
3283   get a tunable
3284  */
3285 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
3286 {
3287         const char *name;
3288         uint32_t value;
3289         int ret;
3290
3291         if (argc < 1) {
3292                 usage();
3293         }
3294
3295         name = argv[0];
3296         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
3297         if (ret == -1) {
3298                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
3299                 return -1;
3300         }
3301
3302         printf("%-19s = %u\n", name, value);
3303         return 0;
3304 }
3305
3306 /*
3307   set a tunable
3308  */
3309 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
3310 {
3311         const char *name;
3312         uint32_t value;
3313         int ret;
3314
3315         if (argc < 2) {
3316                 usage();
3317         }
3318
3319         name = argv[0];
3320         value = strtoul(argv[1], NULL, 0);
3321
3322         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
3323         if (ret == -1) {
3324                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
3325                 return -1;
3326         }
3327         return 0;
3328 }
3329
3330 /*
3331   list all tunables
3332  */
3333 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
3334 {
3335         uint32_t count;
3336         const char **list;
3337         int ret, i;
3338
3339         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
3340         if (ret == -1) {
3341                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
3342                 return -1;
3343         }
3344
3345         for (i=0;i<count;i++) {
3346                 control_getvar(ctdb, 1, &list[i]);
3347         }
3348
3349         talloc_free(list);
3350
3351         return 0;
3352 }
3353
3354 /*
3355   display debug level on a node
3356  */
3357 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
3358 {
3359         int ret;
3360         int32_t level;
3361
3362         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
3363         if (ret != 0) {
3364                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
3365                 return ret;
3366         } else {
3367                 if (options.machinereadable){
3368                         printf(":Name:Level:\n");
3369                         printf(":%s:%d:\n",get_debug_by_level(level),level);
3370                 } else {
3371                         printf("Node %u is at debug level %s (%d)\n", options.pnn, get_debug_by_level(level), level);
3372                 }
3373         }
3374         return 0;
3375 }
3376
3377 /*
3378   display reclock file of a node
3379  */
3380 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
3381 {
3382         int ret;
3383         const char *reclock;
3384
3385         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
3386         if (ret != 0) {
3387                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
3388                 return ret;
3389         } else {
3390                 if (options.machinereadable){
3391                         if (reclock != NULL) {
3392                                 printf("%s", reclock);
3393                         }
3394                 } else {
3395                         if (reclock == NULL) {
3396                                 printf("No reclock file used.\n");
3397                         } else {
3398                                 printf("Reclock file:%s\n", reclock);
3399                         }
3400                 }
3401         }
3402         return 0;
3403 }
3404
3405 /*
3406   set the reclock file of a node
3407  */
3408 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
3409 {
3410         int ret;
3411         const char *reclock;
3412
3413         if (argc == 0) {
3414                 reclock = NULL;
3415         } else if (argc == 1) {
3416                 reclock = argv[0];
3417         } else {
3418                 usage();
3419         }
3420
3421         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
3422         if (ret != 0) {
3423                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
3424                 return ret;
3425         }
3426         return 0;
3427 }
3428
3429 /*
3430   set the natgw state on/off
3431  */
3432 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
3433 {
3434         int ret;
3435         uint32_t natgwstate;
3436
3437         if (argc == 0) {
3438                 usage();
3439         }
3440
3441         if (!strcmp(argv[0], "on")) {
3442                 natgwstate = 1;
3443         } else if (!strcmp(argv[0], "off")) {
3444                 natgwstate = 0;
3445         } else {
3446                 usage();
3447         }
3448
3449         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
3450         if (ret != 0) {
3451                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
3452                 return ret;
3453         }
3454
3455         return 0;
3456 }
3457
3458 /*
3459   set the lmaster role on/off
3460  */
3461 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
3462 {
3463         int ret;
3464         uint32_t lmasterrole;
3465
3466         if (argc == 0) {
3467                 usage();
3468         }
3469
3470         if (!strcmp(argv[0], "on")) {
3471                 lmasterrole = 1;
3472         } else if (!strcmp(argv[0], "off")) {
3473                 lmasterrole = 0;
3474         } else {
3475                 usage();
3476         }
3477
3478         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
3479         if (ret != 0) {
3480                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
3481                 return ret;
3482         }
3483
3484         return 0;
3485 }
3486
3487 /*
3488   set the recmaster role on/off
3489  */
3490 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
3491 {
3492         int ret;
3493         uint32_t recmasterrole;
3494
3495         if (argc == 0) {
3496                 usage();
3497         }
3498
3499         if (!strcmp(argv[0], "on")) {
3500                 recmasterrole = 1;
3501         } else if (!strcmp(argv[0], "off")) {
3502                 recmasterrole = 0;
3503         } else {
3504                 usage();
3505         }
3506
3507         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
3508         if (ret != 0) {
3509                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
3510                 return ret;
3511         }
3512
3513         return 0;
3514 }
3515
3516 /*
3517   set debug level on a node or all nodes
3518  */
3519 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
3520 {
3521         int i, ret;
3522         int32_t level;
3523
3524         if (argc == 0) {
3525                 printf("You must specify the debug level. Valid levels are:\n");
3526                 for (i=0; debug_levels[i].description != NULL; i++) {
3527                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
3528                 }
3529
3530                 return 0;
3531         }
3532
3533         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
3534                 level = get_debug_by_desc(argv[0]);
3535         } else {
3536                 level = strtol(argv[0], NULL, 0);
3537         }
3538
3539         for (i=0; debug_levels[i].description != NULL; i++) {
3540                 if (level == debug_levels[i].level) {
3541                         break;
3542                 }
3543         }
3544         if (debug_levels[i].description == NULL) {
3545                 printf("Invalid debug level, must be one of\n");
3546                 for (i=0; debug_levels[i].description != NULL; i++) {
3547                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
3548                 }
3549                 return -1;
3550         }
3551
3552         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
3553         if (ret != 0) {
3554                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
3555         }
3556         return 0;
3557 }
3558
3559
3560 /*
3561   thaw a node
3562  */
3563 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
3564 {
3565         int ret;
3566         uint32_t priority;
3567
3568         if (argc == 1) {
3569                 priority = strtol(argv[0], NULL, 0);
3570         } else {
3571                 priority = 0;
3572         }
3573         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
3574
3575         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
3576         if (ret != 0) {
3577                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
3578         }
3579         return 0;
3580 }
3581
3582
3583 /*
3584   attach to a database
3585  */
3586 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
3587 {
3588         const char *db_name;
3589         struct ctdb_db_context *ctdb_db;
3590
3591         if (argc < 1) {
3592                 usage();
3593         }
3594         db_name = argv[0];
3595
3596         ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
3597         if (ctdb_db == NULL) {
3598                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3599                 return -1;
3600         }
3601
3602         return 0;
3603 }
3604
3605 /*
3606   set db priority
3607  */
3608 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
3609 {
3610         struct ctdb_db_priority db_prio;
3611         int ret;
3612
3613         if (argc < 2) {
3614                 usage();
3615         }
3616
3617         db_prio.db_id    = strtoul(argv[0], NULL, 0);
3618         db_prio.priority = strtoul(argv[1], NULL, 0);
3619
3620         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
3621         if (ret != 0) {
3622                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
3623                 return -1;
3624         }
3625
3626         return 0;
3627 }
3628
3629 /*
3630   get db priority
3631  */
3632 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
3633 {
3634         uint32_t db_id, priority;
3635         int ret;
3636
3637         if (argc < 1) {
3638                 usage();
3639         }
3640
3641         db_id = strtoul(argv[0], NULL, 0);
3642
3643         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
3644         if (ret != 0) {
3645                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
3646                 return -1;
3647         }
3648
3649         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
3650
3651         return 0;
3652 }
3653
3654 /*
3655   run an eventscript on a node
3656  */
3657 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
3658 {
3659         TDB_DATA data;
3660         int ret;
3661         int32_t res;
3662         char *errmsg;
3663         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3664
3665         if (argc != 1) {
3666                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3667                 return -1;
3668         }
3669
3670         data.dptr = (unsigned char *)discard_const(argv[0]);
3671         data.dsize = strlen((char *)data.dptr) + 1;
3672
3673         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
3674
3675         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
3676                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
3677         if (ret != 0 || res != 0) {
3678                 DEBUG(DEBUG_ERR,("Failed to run eventscripts - %s\n", errmsg));
3679                 talloc_free(tmp_ctx);
3680                 return -1;
3681         }
3682         talloc_free(tmp_ctx);
3683         return 0;
3684 }
3685
3686 #define DB_VERSION 1
3687 #define MAX_DB_NAME 64
3688 struct db_file_header {
3689         unsigned long version;
3690         time_t timestamp;
3691         unsigned long persistent;
3692         unsigned long size;
3693         const char name[MAX_DB_NAME];
3694 };
3695
3696 struct backup_data {
3697         struct ctdb_marshall_buffer *records;
3698         uint32_t len;
3699         uint32_t total;
3700         bool traverse_error;
3701 };
3702
3703 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
3704 {
3705         struct backup_data *bd = talloc_get_type(private, struct backup_data);
3706         struct ctdb_rec_data *rec;
3707
3708         /* add the record */
3709         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
3710         if (rec == NULL) {
3711                 bd->traverse_error = true;
3712                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
3713                 return -1;
3714         }
3715         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
3716         if (bd->records == NULL) {
3717                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
3718                 bd->traverse_error = true;
3719                 return -1;
3720         }
3721         bd->records->count++;
3722         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
3723         bd->len += rec->length;
3724         talloc_free(rec);
3725
3726         bd->total++;
3727         return 0;
3728 }
3729
3730 /*
3731  * backup a database to a file
3732  */
3733 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
3734 {
3735         int i, ret;
3736         struct ctdb_dbid_map *dbmap=NULL;
3737         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3738         struct db_file_header dbhdr;
3739         struct ctdb_db_context *ctdb_db;
3740         struct backup_data *bd;
3741         int fh = -1;
3742         int status = -1;
3743         const char *reason = NULL;
3744
3745         if (argc != 2) {
3746                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3747                 return -1;
3748         }
3749
3750         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
3751         if (ret != 0) {
3752                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
3753                 return ret;
3754         }
3755
3756         for(i=0;i<dbmap->num;i++){
3757                 const char *name;
3758
3759                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
3760                 if(!strcmp(argv[0], name)){
3761                         talloc_free(discard_const(name));
3762                         break;
3763                 }
3764                 talloc_free(discard_const(name));
3765         }
3766         if (i == dbmap->num) {
3767                 DEBUG(DEBUG_ERR,("No database with name '%s' found\n", argv[0]));
3768                 talloc_free(tmp_ctx);
3769                 return -1;
3770         }
3771
3772         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
3773                                     dbmap->dbs[i].dbid, tmp_ctx, &reason);
3774         if (ret != 0) {
3775                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
3776                                  argv[0]));
3777                 talloc_free(tmp_ctx);
3778                 return -1;
3779         }
3780         if (reason) {
3781                 uint32_t allow_unhealthy = 0;
3782
3783                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
3784                                       "AllowUnhealthyDBRead",
3785                                       &allow_unhealthy);
3786
3787                 if (allow_unhealthy != 1) {
3788                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
3789                                          argv[0], reason));
3790
3791                         DEBUG(DEBUG_ERR,("disallow backup : tunnable AllowUnhealthyDBRead = %u\n",
3792                                          allow_unhealthy));
3793                         talloc_free(tmp_ctx);
3794                         return -1;
3795                 }
3796
3797                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
3798                                      argv[0], argv[0]));
3799                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
3800                                      "tunnable AllowUnhealthyDBRead = %u\n",
3801                                      allow_unhealthy));
3802         }
3803
3804         ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
3805         if (ctdb_db == NULL) {
3806                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
3807                 talloc_free(tmp_ctx);
3808                 return -1;
3809         }
3810
3811
3812         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
3813         if (ret == -1) {
3814                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
3815                 talloc_free(tmp_ctx);
3816                 return -1;
3817         }
3818
3819
3820         bd = talloc_zero(tmp_ctx, struct backup_data);
3821         if (bd == NULL) {
3822                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
3823                 talloc_free(tmp_ctx);
3824                 return -1;
3825         }
3826
3827         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
3828         if (bd->records == NULL) {
3829                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
3830                 talloc_free(tmp_ctx);
3831                 return -1;
3832         }
3833
3834         bd->len = offsetof(struct ctdb_marshall_buffer, data);
3835         bd->records->db_id = ctdb_db->db_id;
3836         /* traverse the database collecting all records */
3837         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
3838             bd->traverse_error) {
3839                 DEBUG(DEBUG_ERR,("Traverse error\n"));
3840                 talloc_free(tmp_ctx);
3841                 return -1;
3842         }
3843
3844         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
3845
3846
3847         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
3848         if (fh == -1) {
3849                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
3850                 talloc_free(tmp_ctx);
3851                 return -1;
3852         }
3853
3854         dbhdr.version = DB_VERSION;
3855         dbhdr.timestamp = time(NULL);
3856         dbhdr.persistent = dbmap->dbs[i].persistent;
3857         dbhdr.size = bd->len;
3858         if (strlen(argv[0]) >= MAX_DB_NAME) {
3859                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
3860                 goto done;
3861         }
3862         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME);
3863         ret = write(fh, &dbhdr, sizeof(dbhdr));
3864         if (ret == -1) {
3865                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
3866                 goto done;
3867         }
3868         ret = write(fh, bd->records, bd->len);
3869         if (ret == -1) {
3870                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
3871                 goto done;
3872         }
3873
3874         status = 0;
3875 done:
3876         if (fh != -1) {
3877                 ret = close(fh);
3878                 if (ret == -1) {
3879                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
3880                 }
3881         }
3882         talloc_free(tmp_ctx);
3883         return status;
3884 }
3885
3886 /*
3887  * restore a database from a file
3888  */
3889 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
3890 {
3891         int ret;
3892         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3893         TDB_DATA outdata;
3894         TDB_DATA data;
3895         struct db_file_header dbhdr;
3896         struct ctdb_db_context *ctdb_db;
3897         struct ctdb_node_map *nodemap=NULL;
3898         struct ctdb_vnn_map *vnnmap=NULL;
3899         int i, fh;
3900         struct ctdb_control_wipe_database w;
3901         uint32_t *nodes;
3902         uint32_t generation;
3903         struct tm *tm;
3904         char tbuf[100];
3905         char *dbname;
3906
3907         if (argc < 1 || argc > 2) {
3908                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3909                 return -1;
3910         }
3911
3912         fh = open(argv[0], O_RDONLY);
3913         if (fh == -1) {
3914                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
3915                 talloc_free(tmp_ctx);
3916                 return -1;
3917         }
3918
3919         read(fh, &dbhdr, sizeof(dbhdr));
3920         if (dbhdr.version != DB_VERSION) {
3921                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
3922                 talloc_free(tmp_ctx);
3923                 return -1;
3924         }
3925
3926         dbname = discard_const(dbhdr.name);
3927         if (argc == 2) {
3928                 dbname = discard_const(argv[1]);
3929         }
3930
3931         outdata.dsize = dbhdr.size;
3932         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
3933         if (outdata.dptr == NULL) {
3934                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
3935                 close(fh);
3936                 talloc_free(tmp_ctx);
3937                 return -1;
3938         }
3939         read(fh, outdata.dptr, outdata.dsize);
3940         close(fh);
3941
3942         tm = localtime(&dbhdr.timestamp);
3943         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
3944         printf("Restoring database '%s' from backup @ %s\n",
3945                 dbname, tbuf);
3946
3947
3948         ctdb_db = ctdb_attach(ctdb, dbname, dbhdr.persistent, 0);
3949         if (ctdb_db == NULL) {
3950                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
3951                 talloc_free(tmp_ctx);
3952                 return -1;
3953         }
3954
3955         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
3956         if (ret != 0) {
3957                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3958                 talloc_free(tmp_ctx);
3959                 return ret;
3960         }
3961
3962
3963         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
3964         if (ret != 0) {
3965                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
3966                 talloc_free(tmp_ctx);
3967                 return ret;
3968         }
3969
3970         /* freeze all nodes */
3971         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3972         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
3973                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
3974                                         nodes, i,
3975                                         TIMELIMIT(),
3976                                         false, tdb_null,
3977                                         NULL, NULL,
3978                                         NULL) != 0) {
3979                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
3980                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3981                         talloc_free(tmp_ctx);
3982                         return -1;
3983                 }
3984         }
3985
3986         generation = vnnmap->generation;
3987         data.dptr = (void *)&generation;
3988         data.dsize = sizeof(generation);
3989
3990         /* start a cluster wide transaction */
3991         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3992         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
3993                                         nodes, 0,
3994                                         TIMELIMIT(), false, data,
3995                                         NULL, NULL,
3996                                         NULL) != 0) {
3997                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
3998                 return -1;
3999         }
4000
4001
4002         w.db_id = ctdb_db->db_id;
4003         w.transaction_id = generation;
4004
4005         data.dptr = (void *)&w;
4006         data.dsize = sizeof(w);
4007
4008         /* wipe all the remote databases. */
4009         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4010         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
4011                                         nodes, 0,
4012                                         TIMELIMIT(), false, data,
4013                                         NULL, NULL,
4014                                         NULL) != 0) {
4015                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
4016                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4017                 talloc_free(tmp_ctx);
4018                 return -1;
4019         }
4020
4021         /* push the database */
4022         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4023         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
4024                                         nodes, 0,
4025                                         TIMELIMIT(), false, outdata,
4026                                         NULL, NULL,
4027                                         NULL) != 0) {
4028                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
4029                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4030                 talloc_free(tmp_ctx);
4031                 return -1;
4032         }
4033
4034         data.dptr = (void *)&ctdb_db->db_id;
4035         data.dsize = sizeof(ctdb_db->db_id);
4036
4037         /* mark the database as healthy */
4038         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4039         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
4040                                         nodes, 0,
4041                                         TIMELIMIT(), false, data,
4042                                         NULL, NULL,
4043                                         NULL) != 0) {
4044                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
4045                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4046                 talloc_free(tmp_ctx);
4047                 return -1;
4048         }
4049
4050         data.dptr = (void *)&generation;
4051         data.dsize = sizeof(generation);
4052
4053         /* commit all the changes */
4054         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
4055                                         nodes, 0,
4056                                         TIMELIMIT(), false, data,
4057                                         NULL, NULL,
4058                                         NULL) != 0) {
4059                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
4060                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4061                 talloc_free(tmp_ctx);
4062                 return -1;
4063         }
4064
4065
4066         /* thaw all nodes */
4067         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4068         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
4069                                         nodes, 0,
4070                                         TIMELIMIT(),
4071                                         false, tdb_null,
4072                                         NULL, NULL,
4073                                         NULL) != 0) {
4074                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
4075                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4076                 talloc_free(tmp_ctx);
4077                 return -1;
4078         }
4079
4080
4081         talloc_free(tmp_ctx);
4082         return 0;
4083 }
4084
4085 /*
4086  * dump a database backup from a file
4087  */
4088 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
4089 {
4090         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4091         TDB_DATA outdata;
4092         struct db_file_header dbhdr;
4093         int i, fh;
4094         struct tm *tm;
4095         char tbuf[100];
4096         struct ctdb_rec_data *rec = NULL;
4097         struct ctdb_marshall_buffer *m;
4098
4099         if (argc != 1) {
4100                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
4101                 return -1;
4102         }
4103
4104         fh = open(argv[0], O_RDONLY);
4105         if (fh == -1) {
4106                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
4107                 talloc_free(tmp_ctx);
4108                 return -1;
4109         }
4110
4111         read(fh, &dbhdr, sizeof(dbhdr));
4112         if (dbhdr.version != DB_VERSION) {
4113                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
4114                 talloc_free(tmp_ctx);
4115                 return -1;
4116         }
4117
4118         outdata.dsize = dbhdr.size;
4119         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
4120         if (outdata.dptr == NULL) {
4121                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
4122                 close(fh);
4123                 talloc_free(tmp_ctx);
4124                 return -1;
4125         }
4126         read(fh, outdata.dptr, outdata.dsize);
4127         close(fh);
4128         m = (struct ctdb_marshall_buffer *)outdata.dptr;
4129
4130         tm = localtime(&dbhdr.timestamp);
4131         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
4132         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
4133                 dbhdr.name, m->db_id, tbuf);
4134
4135         for (i=0; i < m->count; i++) {
4136                 uint32_t reqid = 0;
4137                 TDB_DATA key, data;
4138
4139                 /* we do not want the header splitted, so we pass NULL*/
4140                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
4141                                               NULL, &key, &data);
4142
4143                 ctdb_dumpdb_record(ctdb, key, data, stdout);
4144         }
4145
4146         printf("Dumped %d records\n", i);
4147         talloc_free(tmp_ctx);
4148         return 0;
4149 }
4150
4151 /*
4152  * wipe a database from a file
4153  */
4154 static int control_wipedb(struct ctdb_context *ctdb, int argc,
4155                           const char **argv)
4156 {
4157         int ret;
4158         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4159         TDB_DATA data;
4160         struct ctdb_db_context *ctdb_db;
4161         struct ctdb_node_map *nodemap = NULL;
4162         struct ctdb_vnn_map *vnnmap = NULL;
4163         int i;
4164         struct ctdb_control_wipe_database w;
4165         uint32_t *nodes;
4166         uint32_t generation;
4167         struct ctdb_dbid_map *dbmap = NULL;
4168
4169         if (argc != 1) {
4170                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
4171                 return -1;
4172         }
4173
4174         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
4175                                  &dbmap);
4176         if (ret != 0) {
4177                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n",
4178                                   options.pnn));
4179                 return ret;
4180         }
4181
4182         for(i=0;i<dbmap->num;i++){
4183                 const char *name;
4184
4185                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
4186                                     dbmap->dbs[i].dbid, tmp_ctx, &name);
4187                 if(!strcmp(argv[0], name)){
4188                         talloc_free(discard_const(name));
4189                         break;
4190                 }
4191                 talloc_free(discard_const(name));
4192         }
4193         if (i == dbmap->num) {
4194                 DEBUG(DEBUG_ERR, ("No database with name '%s' found\n",
4195                                   argv[0]));
4196                 talloc_free(tmp_ctx);
4197                 return -1;
4198         }
4199
4200         ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
4201         if (ctdb_db == NULL) {
4202                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
4203                                   argv[0]));
4204                 talloc_free(tmp_ctx);
4205                 return -1;
4206         }
4207
4208         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
4209                                    &nodemap);
4210         if (ret != 0) {
4211                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
4212                                   options.pnn));
4213                 talloc_free(tmp_ctx);
4214                 return ret;
4215         }
4216
4217         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
4218                                   &vnnmap);
4219         if (ret != 0) {
4220                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
4221                                   options.pnn));
4222                 talloc_free(tmp_ctx);
4223                 return ret;
4224         }
4225
4226         /* freeze all nodes */
4227         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4228         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
4229                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
4230                                                 nodes, i,
4231                                                 TIMELIMIT(),
4232                                                 false, tdb_null,
4233                                                 NULL, NULL,
4234                                                 NULL);
4235                 if (ret != 0) {
4236                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
4237                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
4238                                              CTDB_RECOVERY_ACTIVE);
4239                         talloc_free(tmp_ctx);
4240                         return -1;
4241                 }
4242         }
4243
4244         generation = vnnmap->generation;
4245         data.dptr = (void *)&generation;
4246         data.dsize = sizeof(generation);
4247
4248         /* start a cluster wide transaction */
4249         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4250         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
4251                                         nodes, 0,
4252                                         TIMELIMIT(), false, data,
4253                                         NULL, NULL,
4254                                         NULL);
4255         if (ret!= 0) {
4256                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
4257                                   "transactions.\n"));
4258                 return -1;
4259         }
4260
4261         w.db_id = ctdb_db->db_id;
4262         w.transaction_id = generation;
4263
4264         data.dptr = (void *)&w;
4265         data.dsize = sizeof(w);
4266
4267         /* wipe all the remote databases. */
4268         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4269         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
4270                                         nodes, 0,
4271                                         TIMELIMIT(), false, data,
4272                                         NULL, NULL,
4273                                         NULL) != 0) {
4274                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
4275                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4276                 talloc_free(tmp_ctx);
4277                 return -1;
4278         }
4279
4280         data.dptr = (void *)&ctdb_db->db_id;
4281         data.dsize = sizeof(ctdb_db->db_id);
4282
4283         /* mark the database as healthy */
4284         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4285         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
4286                                         nodes, 0,
4287                                         TIMELIMIT(), false, data,
4288                                         NULL, NULL,
4289                                         NULL) != 0) {
4290                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
4291                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4292                 talloc_free(tmp_ctx);
4293                 return -1;
4294         }
4295
4296         data.dptr = (void *)&generation;
4297         data.dsize = sizeof(generation);
4298
4299         /* commit all the changes */
4300         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
4301                                         nodes, 0,
4302                                         TIMELIMIT(), false, data,
4303                                         NULL, NULL,
4304                                         NULL) != 0) {
4305                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
4306                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4307                 talloc_free(tmp_ctx);
4308                 return -1;
4309         }
4310
4311         /* thaw all nodes */
4312         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4313         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
4314                                         nodes, 0,
4315                                         TIMELIMIT(),
4316                                         false, tdb_null,
4317                                         NULL, NULL,
4318                                         NULL) != 0) {
4319                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
4320                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4321                 talloc_free(tmp_ctx);
4322                 return -1;
4323         }
4324
4325         talloc_free(tmp_ctx);
4326         return 0;
4327 }
4328
4329 /*
4330  * set flags of a node in the nodemap
4331  */
4332 static int control_setflags(struct ctdb_context *ctdb, int argc, const char **argv)
4333 {
4334         int ret;
4335         int32_t status;
4336         int node;
4337         int flags;
4338         TDB_DATA data;
4339         struct ctdb_node_flag_change c;
4340
4341         if (argc != 2) {
4342                 usage();
4343                 return -1;
4344         }
4345
4346         if (sscanf(argv[0], "%d", &node) != 1) {
4347                 DEBUG(DEBUG_ERR, ("Badly formed node\n"));
4348                 usage();
4349                 return -1;
4350         }
4351         if (sscanf(argv[1], "0x%x", &flags) != 1) {
4352                 DEBUG(DEBUG_ERR, ("Badly formed flags\n"));
4353                 usage();
4354                 return -1;
4355         }
4356
4357         c.pnn       = node;
4358         c.old_flags = 0;
4359         c.new_flags = flags;
4360
4361         data.dsize = sizeof(c);
4362         data.dptr = (unsigned char *)&c;
4363
4364         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_MODIFY_FLAGS, 0,
4365                            data, NULL, NULL, &status, NULL, NULL);
4366         if (ret != 0 || status != 0) {
4367                 DEBUG(DEBUG_ERR,("Failed to modify flags\n"));
4368                 return -1;
4369         }
4370         return 0;
4371 }
4372
4373 /*
4374   dump memory usage
4375  */
4376 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
4377 {
4378         TDB_DATA data;
4379         int ret;
4380         int32_t res;
4381         char *errmsg;
4382         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4383         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
4384                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
4385         if (ret != 0 || res != 0) {
4386                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
4387                 talloc_free(tmp_ctx);
4388                 return -1;
4389         }
4390         write(1, data.dptr, data.dsize);
4391         talloc_free(tmp_ctx);
4392         return 0;
4393 }
4394
4395 /*
4396   handler for memory dumps
4397 */
4398 static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
4399                              TDB_DATA data, void *private_data)
4400 {
4401         write(1, data.dptr, data.dsize);
4402         exit(0);
4403 }
4404
4405 /*
4406   dump memory usage on the recovery daemon
4407  */
4408 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
4409 {
4410         int ret;
4411         TDB_DATA data;
4412         struct rd_memdump_reply rd;
4413
4414         rd.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
4415         if (rd.pnn == -1) {
4416                 DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
4417                 return -1;
4418         }
4419         rd.srvid = getpid();
4420
4421         /* register a message port for receiveing the reply so that we
4422            can receive the reply
4423         */
4424         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
4425
4426
4427         data.dptr = (uint8_t *)&rd;
4428         data.dsize = sizeof(rd);
4429
4430         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
4431         if (ret != 0) {
4432                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
4433                 return -1;
4434         }
4435
4436         /* this loop will terminate when we have received the reply */
4437         while (1) {
4438                 event_loop_once(ctdb->ev);
4439         }
4440
4441         return 0;
4442 }
4443
4444 /*
4445   send a message to a srvid
4446  */
4447 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
4448 {
4449         unsigned long srvid;
4450         int ret;
4451         TDB_DATA data;
4452
4453         if (argc < 2) {
4454                 usage();
4455         }
4456
4457         srvid      = strtoul(argv[0], NULL, 0);
4458
4459         data.dptr = (uint8_t *)discard_const(argv[1]);
4460         data.dsize= strlen(argv[1]);
4461
4462         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
4463         if (ret != 0) {
4464                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
4465                 return -1;
4466         }
4467
4468         return 0;
4469 }
4470
4471 /*
4472   handler for msglisten
4473 */
4474 static void msglisten_handler(struct ctdb_context *ctdb, uint64_t srvid,
4475                              TDB_DATA data, void *private_data)
4476 {
4477         int i;
4478
4479         printf("Message received: ");
4480         for (i=0;i<data.dsize;i++) {
4481                 printf("%c", data.dptr[i]);
4482         }
4483         printf("\n");
4484 }
4485
4486 /*
4487   listen for messages on a messageport
4488  */
4489 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
4490 {
4491         uint64_t srvid;
4492
4493         srvid = getpid();
4494
4495         /* register a message port and listen for messages
4496         */
4497         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
4498         printf("Listening for messages on srvid:%d\n", (int)srvid);
4499
4500         while (1) {
4501                 event_loop_once(ctdb->ev);
4502         }
4503
4504         return 0;
4505 }
4506
4507 /*
4508   list all nodes in the cluster
4509   if the daemon is running, we read the data from the daemon.
4510   if the daemon is not running we parse the nodes file directly
4511  */
4512 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
4513 {
4514         int i, ret;
4515         struct ctdb_node_map *nodemap=NULL;
4516
4517         if (ctdb != NULL) {
4518                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
4519                 if (ret != 0) {
4520                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
4521                         return ret;
4522                 }
4523
4524                 for(i=0;i<nodemap->num;i++){
4525                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
4526                                 continue;
4527                         }
4528                         if (options.machinereadable){
4529                                 printf(":%d:%s:\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
4530                         } else {
4531                                 printf("%s\n", ctdb_addr_to_str(&nodemap->nodes[i].addr));
4532                         }
4533                 }
4534         } else {
4535                 TALLOC_CTX *mem_ctx = talloc_new(NULL);
4536                 struct pnn_node *pnn_nodes;
4537                 struct pnn_node *pnn_node;
4538
4539                 pnn_nodes = read_nodes_file(mem_ctx);
4540                 if (pnn_nodes == NULL) {
4541                         DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
4542                         talloc_free(mem_ctx);
4543                         return -1;
4544                 }
4545
4546                 for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
4547                         ctdb_sock_addr addr;
4548
4549                         if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
4550                                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
4551                                 talloc_free(mem_ctx);
4552                                 return -1;
4553                         }
4554
4555                         if (options.machinereadable){
4556                                 printf(":%d:%s:\n", pnn_node->pnn, pnn_node->addr);
4557                         } else {
4558                                 printf("%s\n", pnn_node->addr);
4559                         }
4560                 }
4561                 talloc_free(mem_ctx);
4562         }
4563
4564         return 0;
4565 }
4566
4567 /*
4568   reload the nodes file on the local node
4569  */
4570 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
4571 {
4572         int i, ret;
4573         int mypnn;
4574         struct ctdb_node_map *nodemap=NULL;
4575
4576         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
4577         if (mypnn == -1) {
4578                 DEBUG(DEBUG_ERR, ("Failed to read pnn of local node\n"));
4579                 return -1;
4580         }
4581
4582         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
4583         if (ret != 0) {
4584                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
4585                 return ret;
4586         }
4587
4588         /* reload the nodes file on all remote nodes */
4589         for (i=0;i<nodemap->num;i++) {
4590                 if (nodemap->nodes[i].pnn == mypnn) {
4591                         continue;
4592                 }
4593                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", nodemap->nodes[i].pnn));
4594                 ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(),
4595                         nodemap->nodes[i].pnn);
4596                 if (ret != 0) {
4597                         DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", nodemap->nodes[i].pnn));
4598                 }
4599         }
4600
4601         /* reload the nodes file on the local node */
4602         DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", mypnn));
4603         ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(), mypnn);
4604         if (ret != 0) {
4605                 DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", mypnn));
4606         }
4607
4608         /* initiate a recovery */
4609         control_recover(ctdb, argc, argv);
4610
4611         return 0;
4612 }
4613
4614
4615 static const struct {
4616         const char *name;
4617         int (*fn)(struct ctdb_context *, int, const char **);
4618         bool auto_all;
4619         bool without_daemon; /* can be run without daemon running ? */
4620         const char *msg;
4621         const char *args;
4622 } ctdb_commands[] = {
4623 #ifdef CTDB_VERS
4624         { "version",         control_version,           true,   false,  "show version of ctdb" },
4625 #endif
4626         { "status",          control_status,            true,   false,  "show node status" },
4627         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
4628         { "ping",            control_ping,              true,   false,  "ping all nodes" },
4629         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
4630         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
4631         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
4632         { "statistics",      control_statistics,        false,  false, "show statistics" },
4633         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
4634         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
4635         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
4636         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
4637         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
4638         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
4639         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
4640         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname>" },
4641         { "catdb",           control_catdb,             true,   false,  "dump a database" ,                     "<dbname>"},
4642         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
4643         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
4644         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
4645         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
4646         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
4647         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
4648         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
4649         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
4650         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
4651         { "getlog",          control_getlog,            true,   false,  "get the log data from the in memory ringbuffer", "<level>" },
4652         { "clearlog",          control_clearlog,        true,   false,  "clear the log data from the in memory ringbuffer" },
4653         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname>" },
4654         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
4655         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
4656         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
4657         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
4658         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
4659         { "stop",            control_stop,              true,   false,  "stop a node" },
4660         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
4661         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime|0>"},
4662         { "unban",           control_unban,             true,   false,  "unban a node" },
4663         { "showban",         control_showban,           true,   false,  "show ban information"},
4664         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
4665         { "recover",         control_recover,           true,   false,  "force recovery" },
4666         { "sync",            control_ipreallocate,      true,   false,  "wait until ctdbd has synced all state changes" },
4667         { "ipreallocate",    control_ipreallocate,      true,   false,  "force the recovery daemon to perform a ip reallocation procedure" },
4668         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
4669         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
4670         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "<srcip:port> <dstip:port>" },
4671         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
4672         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
4673         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
4674         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
4675
4676         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
4677
4678         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
4679         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
4680         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
4681         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
4682         { "vacuum",          ctdb_vacuum,               false,  false, "vacuum the databases of empty records", "[max_records]"},
4683         { "repack",          ctdb_repack,               false,  false, "repack all databases", "[max_freelist]"},
4684         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
4685         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
4686         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
4687         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
4688         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
4689         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
4690         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<database> <file>"},
4691         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
4692         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
4693         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname>"},
4694         { "recmaster",        control_recmaster,        false,  false, "show the pnn for the recovery master."},
4695         { "setflags",        control_setflags,          false,  false, "set flags for a node in the nodemap.", "<node> <flags>"},
4696         { "scriptstatus",    control_scriptstatus,  false,      false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
4697         { "enablescript",     control_enablescript,  false,     false, "enable an eventscript", "<script>"},
4698         { "disablescript",    control_disablescript,  false,    false, "disable an eventscript", "<script>"},
4699         { "natgwlist",        control_natgwlist,        false,  false, "show the nodes belonging to this natgw configuration"},
4700         { "xpnn",             control_xpnn,             true,   true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
4701         { "getreclock",       control_getreclock,       false,  false, "Show the reclock file of a node"},
4702         { "setreclock",       control_setreclock,       false,  false, "Set/clear the reclock file of a node", "[filename]"},
4703         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
4704         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
4705         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
4706         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbid> <prio:1-3>"},
4707         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbid>"},
4708         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
4709         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
4710         { "sync",            control_ipreallocate,      true,   false,  "wait until ctdbd has synced all state changes" },
4711         { "pfetch",          control_pfetch,            true,   false,  "fetch a record from a persistent database", "<db> <key> [<file>]" },
4712         { "pstore",          control_pstore,            true,   false,  "write a record to a persistent database", "<db> <key> <file containing record>" },
4713 };
4714
4715 /*
4716   show usage message
4717  */
4718 static void usage(void)
4719 {
4720         int i;
4721         printf(
4722 "Usage: ctdb [options] <control>\n" \
4723 "Options:\n" \
4724 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
4725 "   -Y                 generate machinereadable output\n"
4726 "   -v                 generate verbose output\n"
4727 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
4728         printf("Controls:\n");
4729         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
4730                 printf("  %-15s %-27s  %s\n",
4731                        ctdb_commands[i].name,
4732                        ctdb_commands[i].args?ctdb_commands[i].args:"",
4733                        ctdb_commands[i].msg);
4734         }
4735         exit(1);
4736 }
4737
4738
4739 static void ctdb_alarm(int sig)
4740 {
4741         printf("Maximum runtime exceeded - exiting\n");
4742         _exit(ERR_TIMEOUT);
4743 }
4744
4745 /*
4746   main program
4747 */
4748 int main(int argc, const char *argv[])
4749 {
4750         struct ctdb_context *ctdb;
4751         char *nodestring = NULL;
4752         struct poptOption popt_options[] = {
4753                 POPT_AUTOHELP
4754                 POPT_CTDB_CMDLINE
4755                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
4756                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
4757                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
4758                 { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
4759                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
4760                 POPT_TABLEEND
4761         };
4762         int opt;
4763         const char **extra_argv;
4764         int extra_argc = 0;
4765         int ret=-1, i;
4766         poptContext pc;
4767         struct event_context *ev;
4768         const char *control;
4769
4770         setlinebuf(stdout);
4771
4772         /* set some defaults */
4773         options.maxruntime = 0;
4774         options.timelimit = 3;
4775         options.pnn = CTDB_CURRENT_NODE;
4776
4777         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
4778
4779         while ((opt = poptGetNextOpt(pc)) != -1) {
4780                 switch (opt) {
4781                 default:
4782                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
4783                                 poptBadOption(pc, 0), poptStrerror(opt)));
4784                         exit(1);
4785                 }
4786         }
4787
4788         /* setup the remaining options for the main program to use */
4789         extra_argv = poptGetArgs(pc);
4790         if (extra_argv) {
4791                 extra_argv++;
4792                 while (extra_argv[extra_argc]) extra_argc++;
4793         }
4794
4795         if (extra_argc < 1) {
4796                 usage();
4797         }
4798
4799         if (options.maxruntime == 0) {
4800                 const char *ctdb_timeout;
4801                 ctdb_timeout = getenv("CTDB_TIMEOUT");
4802                 if (ctdb_timeout != NULL) {
4803                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
4804                 } else {
4805                         /* default timeout is 120 seconds */
4806                         options.maxruntime = 120;
4807                 }
4808         }
4809
4810         signal(SIGALRM, ctdb_alarm);
4811         alarm(options.maxruntime);
4812
4813         /* setup the node number to contact */
4814         if (nodestring != NULL) {
4815                 if (strcmp(nodestring, "all") == 0) {
4816                         options.pnn = CTDB_BROADCAST_ALL;
4817                 } else {
4818                         options.pnn = strtoul(nodestring, NULL, 0);
4819                 }
4820         }
4821
4822         control = extra_argv[0];
4823
4824         ev = event_context_init(NULL);
4825         if (!ev) {
4826                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
4827                 exit(1);
4828         }
4829         tevent_loop_allow_nesting(ev);
4830
4831         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
4832                 if (strcmp(control, ctdb_commands[i].name) == 0) {
4833                         int j;
4834
4835                         if (ctdb_commands[i].without_daemon == true) {
4836                                 close(2);
4837                         }
4838
4839                         /* initialise ctdb */
4840                         ctdb = ctdb_cmdline_client(ev);
4841
4842                         if (ctdb_commands[i].without_daemon == false) {
4843                                 const char *socket_name;
4844
4845                                 if (ctdb == NULL) {
4846                                         DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
4847                                         exit(1);
4848                                 }
4849
4850                                 /* initialize a libctdb connection as well */
4851                                 socket_name = ctdb_get_socketname(ctdb);
4852                                 ctdb_connection = ctdb_connect(socket_name,
4853                                                        ctdb_log_file, stderr);
4854                                 if (ctdb_connection == NULL) {
4855                                         fprintf(stderr, "Failed to connect to daemon from libctdb\n");
4856                                         exit(1);
4857                                 }
4858
4859                                 /* verify the node exists */
4860                                 verify_node(ctdb);
4861
4862                                 if (options.pnn == CTDB_CURRENT_NODE) {
4863                                         int pnn;
4864                                         pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
4865                                         if (pnn == -1) {
4866                                                 return -1;
4867                                         }
4868                                         options.pnn = pnn;
4869                                 }
4870                         }
4871
4872                         if (ctdb_commands[i].auto_all &&
4873                             options.pnn == CTDB_BROADCAST_ALL) {
4874                                 uint32_t *nodes;
4875                                 uint32_t num_nodes;
4876                                 ret = 0;
4877
4878                                 nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
4879                                 CTDB_NO_MEMORY(ctdb, nodes);
4880
4881                                 for (j=0;j<num_nodes;j++) {
4882                                         options.pnn = nodes[j];
4883                                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
4884                                 }
4885                                 talloc_free(nodes);
4886                         } else {
4887                                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
4888                         }
4889                         break;
4890                 }
4891         }
4892
4893         if (i == ARRAY_SIZE(ctdb_commands)) {
4894                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
4895                 exit(1);
4896         }
4897
4898         return ret;
4899 }