tools/ctdb.c

   1 /*
   2    ctdb control tool
   3
   4    Copyright (C) Andrew Tridgell  2007
   5    Copyright (C) Ronnie Sahlberg  2007
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "lib/tevent/tevent.h"
  23 #include "system/time.h"
  24 #include "system/filesys.h"
  25 #include "system/network.h"
  26 #include "system/locale.h"
  27 #include "popt.h"
  28 #include "cmdline.h"
  29 #include "../include/ctdb.h"
  30 #include "../include/ctdb_client.h"
  31 #include "../include/ctdb_private.h"
  32 #include "../common/rb_tree.h"
  33 #include "db_wrap.h"
  34
  35 #define ERR_TIMEOUT     20      /* timed out trying to reach node */
  36 #define ERR_NONODE      21      /* node does not exist */
  37 #define ERR_DISNODE     22      /* node is disconnected */
  38
  39 struct ctdb_connection *ctdb_connection;
  40
  41 static void usage(void);
  42
  43 static struct {
  44         int timelimit;
  45         uint32_t pnn;
  46         int machinereadable;
  47         int maxruntime;
  48 } options;
  49
  50 #define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
  51 #define LONGTIMELIMIT() timeval_current_ofs(options.timelimit*10, 0)
  52
  53 #ifdef CTDB_VERS
  54 static int control_version(struct ctdb_context *ctdb, int argc, const char **argv)
  55 {
  56 #define STR(x) #x
  57 #define XSTR(x) STR(x)
  58         printf("CTDB version: %s\n", XSTR(CTDB_VERS));
  59         return 0;
  60 }
  61 #endif
  62
  63
  64 /*
  65   verify that a node exists and is reachable
  66  */
  67 static void verify_node(struct ctdb_context *ctdb)
  68 {
  69         int ret;
  70         struct ctdb_node_map *nodemap=NULL;
  71
  72         if (options.pnn == CTDB_CURRENT_NODE) {
  73                 return;
  74         }
  75         if (options.pnn == CTDB_BROADCAST_ALL) {
  76                 return;
  77         }
  78
  79         /* verify the node exists */
  80         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
  81                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
  82                 exit(10);
  83         }
  84         if (options.pnn >= nodemap->num) {
  85                 DEBUG(DEBUG_ERR, ("Node %u does not exist\n", options.pnn));
  86                 exit(ERR_NONODE);
  87         }
  88         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_DELETED) {
  89                 DEBUG(DEBUG_ERR, ("Node %u is DELETED\n", options.pnn));
  90                 exit(ERR_DISNODE);
  91         }
  92         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_DISCONNECTED) {
  93                 DEBUG(DEBUG_ERR, ("Node %u is DISCONNECTED\n", options.pnn));
  94                 exit(ERR_DISNODE);
  95         }
  96
  97         /* verify we can access the node */
  98         ret = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
  99         if (ret == -1) {
 100                 DEBUG(DEBUG_ERR,("Can not access node. Node is not operational.\n"));
 101                 exit(10);
 102         }
 103 }
 104
 105 /*
 106  check if a database exists
 107 */
 108 static int db_exists(struct ctdb_context *ctdb, const char *db_name)
 109 {
 110         int i, ret;
 111         struct ctdb_dbid_map *dbmap=NULL;
 112
 113         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
 114         if (ret != 0) {
 115                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
 116                 return -1;
 117         }
 118
 119         for(i=0;i<dbmap->num;i++){
 120                 const char *name;
 121
 122                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
 123                 if (!strcmp(name, db_name)) {
 124                         return 0;
 125                 }
 126         }
 127
 128         return -1;
 129 }
 130
 131 /*
 132   see if a process exists
 133  */
 134 static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
 135 {
 136         uint32_t pnn, pid;
 137         int ret;
 138         if (argc < 1) {
 139                 usage();
 140         }
 141
 142         if (sscanf(argv[0], "%u:%u", &pnn, &pid) != 2) {
 143                 DEBUG(DEBUG_ERR, ("Badly formed pnn:pid\n"));
 144                 return -1;
 145         }
 146
 147         ret = ctdb_ctrl_process_exists(ctdb, pnn, pid);
 148         if (ret == 0) {
 149                 printf("%u:%u exists\n", pnn, pid);
 150         } else {
 151                 printf("%u:%u does not exist\n", pnn, pid);
 152         }
 153         return ret;
 154 }
 155
 156 /*
 157   display statistics structure
 158  */
 159 static void show_statistics(struct ctdb_statistics *s)
 160 {
 161         TALLOC_CTX *tmp_ctx = talloc_new(NULL);
 162         int i;
 163         const char *prefix=NULL;
 164         int preflen=0;
 165         int tmp, days, hours, minutes, seconds;
 166         const struct {
 167                 const char *name;
 168                 uint32_t offset;
 169         } fields[] = {
 170 #define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
 171                 STATISTICS_FIELD(num_clients),
 172                 STATISTICS_FIELD(frozen),
 173                 STATISTICS_FIELD(recovering),
 174                 STATISTICS_FIELD(num_recoveries),
 175                 STATISTICS_FIELD(client_packets_sent),
 176                 STATISTICS_FIELD(client_packets_recv),
 177                 STATISTICS_FIELD(node_packets_sent),
 178                 STATISTICS_FIELD(node_packets_recv),
 179                 STATISTICS_FIELD(keepalive_packets_sent),
 180                 STATISTICS_FIELD(keepalive_packets_recv),
 181                 STATISTICS_FIELD(node.req_call),
 182                 STATISTICS_FIELD(node.reply_call),
 183                 STATISTICS_FIELD(node.req_dmaster),
 184                 STATISTICS_FIELD(node.reply_dmaster),
 185                 STATISTICS_FIELD(node.reply_error),
 186                 STATISTICS_FIELD(node.req_message),
 187                 STATISTICS_FIELD(node.req_control),
 188                 STATISTICS_FIELD(node.reply_control),
 189                 STATISTICS_FIELD(client.req_call),
 190                 STATISTICS_FIELD(client.req_message),
 191                 STATISTICS_FIELD(client.req_control),
 192                 STATISTICS_FIELD(timeouts.call),
 193                 STATISTICS_FIELD(timeouts.control),
 194                 STATISTICS_FIELD(timeouts.traverse),
 195                 STATISTICS_FIELD(total_calls),
 196                 STATISTICS_FIELD(pending_calls),
 197                 STATISTICS_FIELD(lockwait_calls),
 198                 STATISTICS_FIELD(pending_lockwait_calls),
 199                 STATISTICS_FIELD(childwrite_calls),
 200                 STATISTICS_FIELD(pending_childwrite_calls),
 201                 STATISTICS_FIELD(memory_used),
 202                 STATISTICS_FIELD(max_hop_count),
 203         };
 204         tmp = s->statistics_current_time.tv_sec - s->statistics_start_time.tv_sec;
 205         seconds = tmp%60;
 206         tmp    /= 60;
 207         minutes = tmp%60;
 208         tmp    /= 60;
 209         hours   = tmp%24;
 210         tmp    /= 24;
 211         days    = tmp;
 212
 213         printf("CTDB version %u\n", CTDB_VERSION);
 214         printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
 215         printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
 216
 217         for (i=0;i<ARRAY_SIZE(fields);i++) {
 218                 if (strchr(fields[i].name, '.')) {
 219                         preflen = strcspn(fields[i].name, ".")+1;
 220                         if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
 221                                 prefix = fields[i].name;
 222                                 printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
 223                         }
 224                 } else {
 225                         preflen = 0;
 226                 }
 227                 printf(" %*s%-22s%*s%10u\n",
 228                        preflen?4:0, "",
 229                        fields[i].name+preflen,
 230                        preflen?0:4, "",
 231                        *(uint32_t *)(fields[i].offset+(uint8_t *)s));
 232         }
 233         printf(" %-30s     %.6f sec\n", "max_reclock_ctdbd", s->reclock.ctdbd);
 234         printf(" %-30s     %.6f sec\n", "max_reclock_recd", s->reclock.recd);
 235
 236         printf(" %-30s     %.6f sec\n", "max_call_latency", s->max_call_latency);
 237         printf(" %-30s     %.6f sec\n", "max_lockwait_latency", s->max_lockwait_latency);
 238         printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
 239         printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
 240
 241         talloc_free(tmp_ctx);
 242 }
 243
 244 /*
 245   display remote ctdb statistics combined from all nodes
 246  */
 247 static int control_statistics_all(struct ctdb_context *ctdb)
 248 {
 249         int ret, i;
 250         struct ctdb_statistics statistics;
 251         uint32_t *nodes;
 252         uint32_t num_nodes;
 253
 254         nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
 255         CTDB_NO_MEMORY(ctdb, nodes);
 256
 257         ZERO_STRUCT(statistics);
 258
 259         for (i=0;i<num_nodes;i++) {
 260                 struct ctdb_statistics s1;
 261                 int j;
 262                 uint32_t *v1 = (uint32_t *)&s1;
 263                 uint32_t *v2 = (uint32_t *)&statistics;
 264                 uint32_t num_ints =
 265                         offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
 266                 ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
 267                 if (ret != 0) {
 268                         DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", nodes[i]));
 269                         return ret;
 270                 }
 271                 for (j=0;j<num_ints;j++) {
 272                         v2[j] += v1[j];
 273                 }
 274                 statistics.max_hop_count =
 275                         MAX(statistics.max_hop_count, s1.max_hop_count);
 276                 statistics.max_call_latency =
 277                         MAX(statistics.max_call_latency, s1.max_call_latency);
 278                 statistics.max_lockwait_latency =
 279                         MAX(statistics.max_lockwait_latency, s1.max_lockwait_latency);
 280         }
 281         talloc_free(nodes);
 282         printf("Gathered statistics for %u nodes\n", num_nodes);
 283         show_statistics(&statistics);
 284         return 0;
 285 }
 286
 287 /*
 288   display remote ctdb statistics
 289  */
 290 static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
 291 {
 292         int ret;
 293         struct ctdb_statistics statistics;
 294
 295         if (options.pnn == CTDB_BROADCAST_ALL) {
 296                 return control_statistics_all(ctdb);
 297         }
 298
 299         ret = ctdb_ctrl_statistics(ctdb, options.pnn, &statistics);
 300         if (ret != 0) {
 301                 DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
 302                 return ret;
 303         }
 304         show_statistics(&statistics);
 305         return 0;
 306 }
 307
 308
 309 /*
 310   reset remote ctdb statistics
 311  */
 312 static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
 313 {
 314         int ret;
 315
 316         ret = ctdb_statistics_reset(ctdb, options.pnn);
 317         if (ret != 0) {
 318                 DEBUG(DEBUG_ERR, ("Unable to reset statistics on node %u\n", options.pnn));
 319                 return ret;
 320         }
 321         return 0;
 322 }
 323
 324
 325 /*
 326   display uptime of remote node
 327  */
 328 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
 329 {
 330         int ret;
 331         struct ctdb_uptime *uptime = NULL;
 332         int tmp, days, hours, minutes, seconds;
 333
 334         ret = ctdb_ctrl_uptime(ctdb, ctdb, TIMELIMIT(), options.pnn, &uptime);
 335         if (ret != 0) {
 336                 DEBUG(DEBUG_ERR, ("Unable to get uptime from node %u\n", options.pnn));
 337                 return ret;
 338         }
 339
 340         if (options.machinereadable){
 341                 printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n");
 342                 printf(":%u:%u:%u:%lf\n",
 343                         (unsigned int)uptime->current_time.tv_sec,
 344                         (unsigned int)uptime->ctdbd_start_time.tv_sec,
 345                         (unsigned int)uptime->last_recovery_finished.tv_sec,
 346                         timeval_delta(&uptime->last_recovery_finished,
 347                                       &uptime->last_recovery_started)
 348                 );
 349                 return 0;
 350         }
 351
 352         printf("Current time of node          :                %s", ctime(&uptime->current_time.tv_sec));
 353
 354         tmp = uptime->current_time.tv_sec - uptime->ctdbd_start_time.tv_sec;
 355         seconds = tmp%60;
 356         tmp    /= 60;
 357         minutes = tmp%60;
 358         tmp    /= 60;
 359         hours   = tmp%24;
 360         tmp    /= 24;
 361         days    = tmp;
 362         printf("Ctdbd start time              : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
 363
 364         tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
 365         seconds = tmp%60;
 366         tmp    /= 60;
 367         minutes = tmp%60;
 368         tmp    /= 60;
 369         hours   = tmp%24;
 370         tmp    /= 24;
 371         days    = tmp;
 372         printf("Time of last recovery/failover: (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
 373
 374         printf("Duration of last recovery/failover: %lf seconds\n",
 375                 timeval_delta(&uptime->last_recovery_finished,
 376                               &uptime->last_recovery_started));
 377
 378         return 0;
 379 }
 380
 381 /*
 382   show the PNN of the current node
 383  */
 384 static int control_pnn(struct ctdb_context *ctdb, int argc, const char **argv)
 385 {
 386         uint32_t mypnn;
 387         bool ret;
 388
 389         ret = ctdb_getpnn(ctdb_connection, options.pnn, &mypnn);
 390         if (!ret) {
 391                 DEBUG(DEBUG_ERR, ("Unable to get pnn from node."));
 392                 return -1;
 393         }
 394
 395         printf("PNN:%d\n", mypnn);
 396         return 0;
 397 }
 398
 399
 400 struct pnn_node {
 401         struct pnn_node *next;
 402         const char *addr;
 403         int pnn;
 404 };
 405
 406 static struct pnn_node *read_nodes_file(TALLOC_CTX *mem_ctx)
 407 {
 408         const char *nodes_list;
 409         int nlines;
 410         char **lines;
 411         int i, pnn;
 412         struct pnn_node *pnn_nodes = NULL;
 413         struct pnn_node *pnn_node;
 414         struct pnn_node *tmp_node;
 415
 416         /* read the nodes file */
 417         nodes_list = getenv("CTDB_NODES");
 418         if (nodes_list == NULL) {
 419                 nodes_list = "/etc/ctdb/nodes";
 420         }
 421         lines = file_lines_load(nodes_list, &nlines, mem_ctx);
 422         if (lines == NULL) {
 423                 return NULL;
 424         }
 425         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 426                 nlines--;
 427         }
 428         for (i=0, pnn=0; i<nlines; i++) {
 429                 char *node;
 430
 431                 node = lines[i];
 432                 /* strip leading spaces */
 433                 while((*node == ' ') || (*node == '\t')) {
 434                         node++;
 435                 }
 436                 if (*node == '#') {
 437                         pnn++;
 438                         continue;
 439                 }
 440                 if (strcmp(node, "") == 0) {
 441                         continue;
 442                 }
 443                 pnn_node = talloc(mem_ctx, struct pnn_node);
 444                 pnn_node->pnn = pnn++;
 445                 pnn_node->addr = talloc_strdup(pnn_node, node);
 446                 pnn_node->next = pnn_nodes;
 447                 pnn_nodes = pnn_node;
 448         }
 449
 450         /* swap them around so we return them in incrementing order */
 451         pnn_node = pnn_nodes;
 452         pnn_nodes = NULL;
 453         while (pnn_node) {
 454                 tmp_node = pnn_node;
 455                 pnn_node = pnn_node->next;
 456
 457                 tmp_node->next = pnn_nodes;
 458                 pnn_nodes = tmp_node;
 459         }
 460
 461         return pnn_nodes;
 462 }
 463
 464 /*
 465   show the PNN of the current node
 466   discover the pnn by loading the nodes file and try to bind to all
 467   addresses one at a time until the ip address is found.
 468  */
 469 static int control_xpnn(struct ctdb_context *ctdb, int argc, const char **argv)
 470 {
 471         TALLOC_CTX *mem_ctx = talloc_new(NULL);
 472         struct pnn_node *pnn_nodes;
 473         struct pnn_node *pnn_node;
 474
 475         pnn_nodes = read_nodes_file(mem_ctx);
 476         if (pnn_nodes == NULL) {
 477                 DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
 478                 talloc_free(mem_ctx);
 479                 return -1;
 480         }
 481
 482         for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
 483                 ctdb_sock_addr addr;
 484
 485                 if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
 486                         DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
 487                         talloc_free(mem_ctx);
 488                         return -1;
 489                 }
 490
 491                 if (ctdb_sys_have_ip(&addr)) {
 492                         printf("PNN:%d\n", pnn_node->pnn);
 493                         talloc_free(mem_ctx);
 494                         return 0;
 495                 }
 496         }
 497
 498         printf("Failed to detect which PNN this node is\n");
 499         talloc_free(mem_ctx);
 500         return -1;
 501 }
 502
 503 /*
 504   display remote ctdb status
 505  */
 506 static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
 507 {
 508         int i, ret;
 509         struct ctdb_vnn_map *vnnmap=NULL;
 510         struct ctdb_node_map *nodemap=NULL;
 511         uint32_t recmode, recmaster;
 512         int mypnn;
 513
 514         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
 515         if (mypnn == -1) {
 516                 return -1;
 517         }
 518
 519         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
 520         if (ret != 0) {
 521                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
 522                 return ret;
 523         }
 524
 525         if(options.machinereadable){
 526                 printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:\n");
 527                 for(i=0;i<nodemap->num;i++){
 528                         int partially_online = 0;
 529                         int j;
 530
 531                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 532                                 continue;
 533                         }
 534                         if (nodemap->nodes[i].flags == 0) {
 535                                 struct ctdb_control_get_ifaces *ifaces;
 536
 537                                 ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
 538                                                            nodemap->nodes[i].pnn,
 539                                                            ctdb, &ifaces);
 540                                 if (ret == 0) {
 541                                         for (j=0; j < ifaces->num; j++) {
 542                                                 if (ifaces->ifaces[j].link_state != 0) {
 543                                                         continue;
 544                                                 }
 545                                                 partially_online = 1;
 546                                                 break;
 547                                         }
 548                                         talloc_free(ifaces);
 549                                 }
 550                         }
 551                         printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
 552                                 ctdb_addr_to_str(&nodemap->nodes[i].addr),
 553                                !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
 554                                !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
 555                                !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 556                                !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
 557                                !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED),
 558                                !!(nodemap->nodes[i].flags&NODE_FLAGS_INACTIVE),
 559                                partially_online);
 560                 }
 561                 return 0;
 562         }
 563
 564         printf("Number of nodes:%d\n", nodemap->num);
 565         for(i=0;i<nodemap->num;i++){
 566                 static const struct {
 567                         uint32_t flag;
 568                         const char *name;
 569                 } flag_names[] = {
 570                         { NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
 571                         { NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
 572                         { NODE_FLAGS_BANNED,                "BANNED" },
 573                         { NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
 574                         { NODE_FLAGS_DELETED,               "DELETED" },
 575                         { NODE_FLAGS_STOPPED,               "STOPPED" },
 576                         { NODE_FLAGS_INACTIVE,              "INACTIVE" },
 577                 };
 578                 char *flags_str = NULL;
 579                 int j;
 580
 581                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 582                         continue;
 583                 }
 584                 if (nodemap->nodes[i].flags == 0) {
 585                         struct ctdb_control_get_ifaces *ifaces;
 586
 587                         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
 588                                                    nodemap->nodes[i].pnn,
 589                                                    ctdb, &ifaces);
 590                         if (ret == 0) {
 591                                 for (j=0; j < ifaces->num; j++) {
 592                                         if (ifaces->ifaces[j].link_state != 0) {
 593                                                 continue;
 594                                         }
 595                                         flags_str = talloc_strdup(ctdb, "PARTIALLYONLINE");
 596                                         break;
 597                                 }
 598                                 talloc_free(ifaces);
 599                         }
 600                 }
 601                 for (j=0;j<ARRAY_SIZE(flag_names);j++) {
 602                         if (nodemap->nodes[i].flags & flag_names[j].flag) {
 603                                 if (flags_str == NULL) {
 604                                         flags_str = talloc_strdup(ctdb, flag_names[j].name);
 605                                 } else {
 606                                         flags_str = talloc_asprintf_append(flags_str, "|%s",
 607                                                                            flag_names[j].name);
 608                                 }
 609                                 CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
 610                         }
 611                 }
 612                 if (flags_str == NULL) {
 613                         flags_str = talloc_strdup(ctdb, "OK");
 614                         CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
 615                 }
 616                 printf("pnn:%d %-16s %s%s\n", nodemap->nodes[i].pnn,
 617                        ctdb_addr_to_str(&nodemap->nodes[i].addr),
 618                        flags_str,
 619                        nodemap->nodes[i].pnn == mypnn?" (THIS NODE)":"");
 620                 talloc_free(flags_str);
 621         }
 622
 623         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &vnnmap);
 624         if (ret != 0) {
 625                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
 626                 return ret;
 627         }
 628         if (vnnmap->generation == INVALID_GENERATION) {
 629                 printf("Generation:INVALID\n");
 630         } else {
 631                 printf("Generation:%d\n",vnnmap->generation);
 632         }
 633         printf("Size:%d\n",vnnmap->size);
 634         for(i=0;i<vnnmap->size;i++){
 635                 printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
 636         }
 637
 638         ret = ctdb_ctrl_getrecmode(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmode);
 639         if (ret != 0) {
 640                 DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
 641                 return ret;
 642         }
 643         printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
 644
 645         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
 646         if (ret != 0) {
 647                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
 648                 return ret;
 649         }
 650         printf("Recovery master:%d\n",recmaster);
 651
 652         return 0;
 653 }
 654
 655
 656 struct natgw_node {
 657         struct natgw_node *next;
 658         const char *addr;
 659 };
 660
 661 /*
 662   display the list of nodes belonging to this natgw configuration
 663  */
 664 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
 665 {
 666         int i, ret;
 667         const char *natgw_list;
 668         int nlines;
 669         char **lines;
 670         struct natgw_node *natgw_nodes = NULL;
 671         struct natgw_node *natgw_node;
 672         struct ctdb_node_map *nodemap=NULL;
 673
 674
 675         /* read the natgw nodes file into a linked list */
 676         natgw_list = getenv("NATGW_NODES");
 677         if (natgw_list == NULL) {
 678                 natgw_list = "/etc/ctdb/natgw_nodes";
 679         }
 680         lines = file_lines_load(natgw_list, &nlines, ctdb);
 681         if (lines == NULL) {
 682                 ctdb_set_error(ctdb, "Failed to load natgw node list '%s'\n", natgw_list);
 683                 return -1;
 684         }
 685         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 686                 nlines--;
 687         }
 688         for (i=0;i<nlines;i++) {
 689                 char *node;
 690
 691                 node = lines[i];
 692                 /* strip leading spaces */
 693                 while((*node == ' ') || (*node == '\t')) {
 694                         node++;
 695                 }
 696                 if (*node == '#') {
 697                         continue;
 698                 }
 699                 if (strcmp(node, "") == 0) {
 700                         continue;
 701                 }
 702                 natgw_node = talloc(ctdb, struct natgw_node);
 703                 natgw_node->addr = talloc_strdup(natgw_node, node);
 704                 CTDB_NO_MEMORY(ctdb, natgw_node->addr);
 705                 natgw_node->next = natgw_nodes;
 706                 natgw_nodes = natgw_node;
 707         }
 708
 709         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
 710         if (ret != 0) {
 711                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node.\n"));
 712                 return ret;
 713         }
 714
 715         i=0;
 716         while(i<nodemap->num) {
 717                 for(natgw_node=natgw_nodes;natgw_node;natgw_node=natgw_node->next) {
 718                         if (!strcmp(natgw_node->addr, ctdb_addr_to_str(&nodemap->nodes[i].addr))) {
 719                                 break;
 720                         }
 721                 }
 722
 723                 /* this node was not in the natgw so we just remove it from
 724                  * the list
 725                  */
 726                 if ((natgw_node == NULL)
 727                 ||  (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) ) {
 728                         int j;
 729
 730                         for (j=i+1; j<nodemap->num; j++) {
 731                                 nodemap->nodes[j-1] = nodemap->nodes[j];
 732                         }
 733                         nodemap->num--;
 734                         continue;
 735                 }
 736
 737                 i++;
 738         }
 739
 740         /* pick a node to be natgwmaster
 741          * we dont allow STOPPED, DELETED, BANNED or UNHEALTHY nodes to become the natgwmaster
 742          */
 743         for(i=0;i<nodemap->num;i++){
 744                 if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY))) {
 745                         printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
 746                         break;
 747                 }
 748         }
 749         /* we couldnt find any healthy node, try unhealthy ones */
 750         if (i == nodemap->num) {
 751                 for(i=0;i<nodemap->num;i++){
 752                         if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
 753                                 printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
 754                                 break;
 755                         }
 756                 }
 757         }
 758         /* unless all nodes are STOPPED, when we pick one anyway */
 759         if (i == nodemap->num) {
 760                 for(i=0;i<nodemap->num;i++){
 761                         if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED))) {
 762                                 printf("%d %s\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
 763                                 break;
 764                         }
 765                 }
 766                 /* or if we still can not find any */
 767                 if (i == nodemap->num) {
 768                         printf("-1 0.0.0.0\n");
 769                 }
 770         }
 771
 772         /* print the pruned list of nodes belonging to this natgw list */
 773         for(i=0;i<nodemap->num;i++){
 774                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
 775                         continue;
 776                 }
 777                 printf(":%d:%s:%d:%d:%d:%d:%d\n", nodemap->nodes[i].pnn,
 778                         ctdb_addr_to_str(&nodemap->nodes[i].addr),
 779                        !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
 780                        !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
 781                        !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
 782                        !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
 783                        !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED));
 784         }
 785
 786         return 0;
 787 }
 788
 789 /*
 790   display the status of the scripts for monitoring (or other events)
 791  */
 792 static int control_one_scriptstatus(struct ctdb_context *ctdb,
 793                                     enum ctdb_eventscript_call type)
 794 {
 795         struct ctdb_scripts_wire *script_status;
 796         int ret, i;
 797
 798         ret = ctdb_ctrl_getscriptstatus(ctdb, TIMELIMIT(), options.pnn, ctdb, type, &script_status);
 799         if (ret != 0) {
 800                 DEBUG(DEBUG_ERR, ("Unable to get script status from node %u\n", options.pnn));
 801                 return ret;
 802         }
 803
 804         if (script_status == NULL) {
 805                 if (!options.machinereadable) {
 806                         printf("%s cycle never run\n",
 807                                ctdb_eventscript_call_names[type]);
 808                 }
 809                 return 0;
 810         }
 811
 812         if (!options.machinereadable) {
 813                 printf("%d scripts were executed last %s cycle\n",
 814                        script_status->num_scripts,
 815                        ctdb_eventscript_call_names[type]);
 816         }
 817         for (i=0; i<script_status->num_scripts; i++) {
 818                 const char *status = NULL;
 819
 820                 switch (script_status->scripts[i].status) {
 821                 case -ETIME:
 822                         status = "TIMEDOUT";
 823                         break;
 824                 case -ENOEXEC:
 825                         status = "DISABLED";
 826                         break;
 827                 case 0:
 828                         status = "OK";
 829                         break;
 830                 default:
 831                         if (script_status->scripts[i].status > 0)
 832                                 status = "ERROR";
 833                         break;
 834                 }
 835                 if (options.machinereadable) {
 836                         printf("%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n",
 837                                ctdb_eventscript_call_names[type],
 838                                script_status->scripts[i].name,
 839                                script_status->scripts[i].status,
 840                                status,
 841                                (long)script_status->scripts[i].start.tv_sec,
 842                                (long)script_status->scripts[i].start.tv_usec,
 843                                (long)script_status->scripts[i].finished.tv_sec,
 844                                (long)script_status->scripts[i].finished.tv_usec,
 845                                script_status->scripts[i].output);
 846                         continue;
 847                 }
 848                 if (status)
 849                         printf("%-20s Status:%s    ",
 850                                script_status->scripts[i].name, status);
 851                 else
 852                         /* Some other error, eg from stat. */
 853                         printf("%-20s Status:CANNOT RUN (%s)",
 854                                script_status->scripts[i].name,
 855                                strerror(-script_status->scripts[i].status));
 856
 857                 if (script_status->scripts[i].status >= 0) {
 858                         printf("Duration:%.3lf ",
 859                         timeval_delta(&script_status->scripts[i].finished,
 860                               &script_status->scripts[i].start));
 861                 }
 862                 if (script_status->scripts[i].status != -ENOEXEC) {
 863                         printf("%s",
 864                                ctime(&script_status->scripts[i].start.tv_sec));
 865                         if (script_status->scripts[i].status != 0) {
 866                                 printf("   OUTPUT:%s\n",
 867                                        script_status->scripts[i].output);
 868                         }
 869                 } else {
 870                         printf("\n");
 871                 }
 872         }
 873         return 0;
 874 }
 875
 876
 877 static int control_scriptstatus(struct ctdb_context *ctdb,
 878                                 int argc, const char **argv)
 879 {
 880         int ret;
 881         enum ctdb_eventscript_call type, min, max;
 882         const char *arg;
 883
 884         if (argc > 1) {
 885                 DEBUG(DEBUG_ERR, ("Unknown arguments to scriptstatus\n"));
 886                 return -1;
 887         }
 888
 889         if (argc == 0)
 890                 arg = ctdb_eventscript_call_names[CTDB_EVENT_MONITOR];
 891         else
 892                 arg = argv[0];
 893
 894         for (type = 0; type < CTDB_EVENT_MAX; type++) {
 895                 if (strcmp(arg, ctdb_eventscript_call_names[type]) == 0) {
 896                         min = type;
 897                         max = type+1;
 898                         break;
 899                 }
 900         }
 901         if (type == CTDB_EVENT_MAX) {
 902                 if (strcmp(arg, "all") == 0) {
 903                         min = 0;
 904                         max = CTDB_EVENT_MAX;
 905                 } else {
 906                         DEBUG(DEBUG_ERR, ("Unknown event type %s\n", argv[0]));
 907                         return -1;
 908                 }
 909         }
 910
 911         if (options.machinereadable) {
 912                 printf(":Type:Name:Code:Status:Start:End:Error Output...:\n");
 913         }
 914
 915         for (type = min; type < max; type++) {
 916                 ret = control_one_scriptstatus(ctdb, type);
 917                 if (ret != 0) {
 918                         return ret;
 919                 }
 920         }
 921
 922         return 0;
 923 }
 924
 925 /*
 926   enable an eventscript
 927  */
 928 static int control_enablescript(struct ctdb_context *ctdb, int argc, const char **argv)
 929 {
 930         int ret;
 931
 932         if (argc < 1) {
 933                 usage();
 934         }
 935
 936         ret = ctdb_ctrl_enablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
 937         if (ret != 0) {
 938           DEBUG(DEBUG_ERR, ("Unable to enable script %s on node %u\n", argv[0], options.pnn));
 939                 return ret;
 940         }
 941
 942         return 0;
 943 }
 944
 945 /*
 946   disable an eventscript
 947  */
 948 static int control_disablescript(struct ctdb_context *ctdb, int argc, const char **argv)
 949 {
 950         int ret;
 951
 952         if (argc < 1) {
 953                 usage();
 954         }
 955
 956         ret = ctdb_ctrl_disablescript(ctdb, TIMELIMIT(), options.pnn, argv[0]);
 957         if (ret != 0) {
 958           DEBUG(DEBUG_ERR, ("Unable to disable script %s on node %u\n", argv[0], options.pnn));
 959                 return ret;
 960         }
 961
 962         return 0;
 963 }
 964
 965 /*
 966   display the pnn of the recovery master
 967  */
 968 static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **argv)
 969 {
 970         int ret;
 971         uint32_t recmaster;
 972
 973         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
 974         if (ret != 0) {
 975                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
 976                 return ret;
 977         }
 978         printf("%d\n",recmaster);
 979
 980         return 0;
 981 }
 982
 983 /*
 984   add a tickle to a public address
 985  */
 986 static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
 987 {
 988         struct ctdb_tcp_connection t;
 989         TDB_DATA data;
 990         int ret;
 991
 992         if (argc < 2) {
 993                 usage();
 994         }
 995
 996         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
 997                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
 998                 return -1;
 999         }
1000         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1001                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1002                 return -1;
1003         }
1004
1005         data.dptr = (uint8_t *)&t;
1006         data.dsize = sizeof(t);
1007
1008         /* tell all nodes about this tcp connection */
1009         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
1010                            0, data, ctdb, NULL, NULL, NULL, NULL);
1011         if (ret != 0) {
1012                 DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
1013                 return -1;
1014         }
1015
1016         return 0;
1017 }
1018
1019
1020 /*
1021   delete a tickle from a node
1022  */
1023 static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
1024 {
1025         struct ctdb_tcp_connection t;
1026         TDB_DATA data;
1027         int ret;
1028
1029         if (argc < 2) {
1030                 usage();
1031         }
1032
1033         if (parse_ip_port(argv[0], &t.src_addr) == 0) {
1034                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1035                 return -1;
1036         }
1037         if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
1038                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
1039                 return -1;
1040         }
1041
1042         data.dptr = (uint8_t *)&t;
1043         data.dsize = sizeof(t);
1044
1045         /* tell all nodes about this tcp connection */
1046         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
1047                            0, data, ctdb, NULL, NULL, NULL, NULL);
1048         if (ret != 0) {
1049                 DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
1050                 return -1;
1051         }
1052
1053         return 0;
1054 }
1055
1056
1057 /*
1058   get a list of all tickles for this pnn
1059  */
1060 static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char **argv)
1061 {
1062         struct ctdb_control_tcp_tickle_list *list;
1063         ctdb_sock_addr addr;
1064         int i, ret;
1065
1066         if (argc < 1) {
1067                 usage();
1068         }
1069
1070         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1071                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1072                 return -1;
1073         }
1074
1075         ret = ctdb_ctrl_get_tcp_tickles(ctdb, TIMELIMIT(), options.pnn, ctdb, &addr, &list);
1076         if (ret == -1) {
1077                 DEBUG(DEBUG_ERR, ("Unable to list tickles\n"));
1078                 return -1;
1079         }
1080
1081         if (options.machinereadable){
1082                 printf(":source ip:port:destination ip:port:\n");
1083                 for (i=0;i<list->tickles.num;i++) {
1084                         printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1085                         printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1086                 }
1087         } else {
1088                 printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
1089                 printf("Num tickles:%u\n", list->tickles.num);
1090                 for (i=0;i<list->tickles.num;i++) {
1091                         printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
1092                         printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
1093                 }
1094         }
1095
1096         talloc_free(list);
1097
1098         return 0;
1099 }
1100
1101
1102 static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn)
1103 {
1104         struct ctdb_all_public_ips *ips;
1105         struct ctdb_public_ip ip;
1106         int i, ret;
1107         uint32_t *nodes;
1108         uint32_t disable_time;
1109         TDB_DATA data;
1110         struct ctdb_node_map *nodemap=NULL;
1111         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1112
1113         disable_time = 30;
1114         data.dptr  = (uint8_t*)&disable_time;
1115         data.dsize = sizeof(disable_time);
1116         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_DISABLE_IP_CHECK, data);
1117         if (ret != 0) {
1118                 DEBUG(DEBUG_ERR,("Failed to send message to disable ipcheck\n"));
1119                 return -1;
1120         }
1121
1122
1123
1124         /* read the public ip list from the node */
1125         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), pnn, ctdb, &ips);
1126         if (ret != 0) {
1127                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", pnn));
1128                 talloc_free(tmp_ctx);
1129                 return -1;
1130         }
1131
1132         for (i=0;i<ips->num;i++) {
1133                 if (ctdb_same_ip(addr, &ips->ips[i].addr)) {
1134                         break;
1135                 }
1136         }
1137         if (i==ips->num) {
1138                 DEBUG(DEBUG_ERR, ("Node %u can not host ip address '%s'\n",
1139                         pnn, ctdb_addr_to_str(addr)));
1140                 talloc_free(tmp_ctx);
1141                 return -1;
1142         }
1143
1144         ip.pnn  = pnn;
1145         ip.addr = *addr;
1146
1147         data.dptr  = (uint8_t *)&ip;
1148         data.dsize = sizeof(ip);
1149
1150         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &nodemap);
1151         if (ret != 0) {
1152                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1153                 talloc_free(tmp_ctx);
1154                 return ret;
1155         }
1156
1157         nodes = list_of_active_nodes_except_pnn(ctdb, nodemap, tmp_ctx, pnn);
1158         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_RELEASE_IP,
1159                                         nodes, 0,
1160                                         LONGTIMELIMIT(),
1161                                         false, data,
1162                                         NULL, NULL,
1163                                         NULL);
1164         if (ret != 0) {
1165                 DEBUG(DEBUG_ERR,("Failed to release IP on nodes\n"));
1166                 talloc_free(tmp_ctx);
1167                 return -1;
1168         }
1169
1170         ret = ctdb_ctrl_takeover_ip(ctdb, LONGTIMELIMIT(), pnn, &ip);
1171         if (ret != 0) {
1172                 DEBUG(DEBUG_ERR,("Failed to take over IP on node %d\n", pnn));
1173                 talloc_free(tmp_ctx);
1174                 return -1;
1175         }
1176
1177         /* update the recovery daemon so it now knows to expect the new
1178            node assignment for this ip.
1179         */
1180         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, CTDB_SRVID_RECD_UPDATE_IP, data);
1181         if (ret != 0) {
1182                 DEBUG(DEBUG_ERR,("Failed to send message to update the ip on the recovery master.\n"));
1183                 return -1;
1184         }
1185
1186         talloc_free(tmp_ctx);
1187         return 0;
1188 }
1189
1190 /*
1191   move/failover an ip address to a specific node
1192  */
1193 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
1194 {
1195         uint32_t pnn;
1196         ctdb_sock_addr addr;
1197
1198         if (argc < 2) {
1199                 usage();
1200                 return -1;
1201         }
1202
1203         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1204                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1205                 return -1;
1206         }
1207
1208
1209         if (sscanf(argv[1], "%u", &pnn) != 1) {
1210                 DEBUG(DEBUG_ERR, ("Badly formed pnn\n"));
1211                 return -1;
1212         }
1213
1214         if (move_ip(ctdb, &addr, pnn) != 0) {
1215                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
1216                 return -1;
1217         }
1218
1219         return 0;
1220 }
1221
1222 void getips_store_callback(void *param, void *data)
1223 {
1224         struct ctdb_public_ip *node_ip = (struct ctdb_public_ip *)data;
1225         struct ctdb_all_public_ips *ips = param;
1226         int i;
1227
1228         i = ips->num++;
1229         ips->ips[i].pnn  = node_ip->pnn;
1230         ips->ips[i].addr = node_ip->addr;
1231 }
1232
1233 void getips_count_callback(void *param, void *data)
1234 {
1235         uint32_t *count = param;
1236
1237         (*count)++;
1238 }
1239
1240 #define IP_KEYLEN       4
1241 static uint32_t *ip_key(ctdb_sock_addr *ip)
1242 {
1243         static uint32_t key[IP_KEYLEN];
1244
1245         bzero(key, sizeof(key));
1246
1247         switch (ip->sa.sa_family) {
1248         case AF_INET:
1249                 key[0]  = ip->ip.sin_addr.s_addr;
1250                 break;
1251         case AF_INET6:
1252                 key[0]  = ip->ip6.sin6_addr.s6_addr32[3];
1253                 key[1]  = ip->ip6.sin6_addr.s6_addr32[2];
1254                 key[2]  = ip->ip6.sin6_addr.s6_addr32[1];
1255                 key[3]  = ip->ip6.sin6_addr.s6_addr32[0];
1256                 break;
1257         default:
1258                 DEBUG(DEBUG_ERR, (__location__ " ERROR, unknown family passed :%u\n", ip->sa.sa_family));
1259                 return key;
1260         }
1261
1262         return key;
1263 }
1264
1265 static void *add_ip_callback(void *parm, void *data)
1266 {
1267         return parm;
1268 }
1269
1270 static int
1271 control_get_all_public_ips(struct ctdb_context *ctdb, TALLOC_CTX *tmp_ctx, struct ctdb_all_public_ips **ips)
1272 {
1273         struct ctdb_all_public_ips *tmp_ips;
1274         struct ctdb_node_map *nodemap=NULL;
1275         trbt_tree_t *ip_tree;
1276         int i, j, len, ret;
1277         uint32_t count;
1278
1279         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1280         if (ret != 0) {
1281                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1282                 return ret;
1283         }
1284
1285         ip_tree = trbt_create(tmp_ctx, 0);
1286
1287         for(i=0;i<nodemap->num;i++){
1288                 if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
1289                         continue;
1290                 }
1291                 if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
1292                         continue;
1293                 }
1294
1295                 /* read the public ip list from this node */
1296                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &tmp_ips);
1297                 if (ret != 0) {
1298                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1299                         return -1;
1300                 }
1301
1302                 for (j=0; j<tmp_ips->num;j++) {
1303                         struct ctdb_public_ip *node_ip;
1304
1305                         node_ip = talloc(tmp_ctx, struct ctdb_public_ip);
1306                         node_ip->pnn  = tmp_ips->ips[j].pnn;
1307                         node_ip->addr = tmp_ips->ips[j].addr;
1308
1309                         trbt_insertarray32_callback(ip_tree,
1310                                 IP_KEYLEN, ip_key(&tmp_ips->ips[j].addr),
1311                                 add_ip_callback,
1312                                 node_ip);
1313                 }
1314                 talloc_free(tmp_ips);
1315         }
1316
1317         /* traverse */
1318         count = 0;
1319         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_count_callback, &count);
1320
1321         len = offsetof(struct ctdb_all_public_ips, ips) +
1322                 count*sizeof(struct ctdb_public_ip);
1323         tmp_ips = talloc_zero_size(tmp_ctx, len);
1324         trbt_traversearray32(ip_tree, IP_KEYLEN, getips_store_callback, tmp_ips);
1325
1326         *ips = tmp_ips;
1327
1328         return 0;
1329 }
1330
1331
1332 /*
1333  * scans all other nodes and returns a pnn for another node that can host this
1334  * ip address or -1
1335  */
1336 static int
1337 find_other_host_for_public_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr)
1338 {
1339         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1340         struct ctdb_all_public_ips *ips;
1341         struct ctdb_node_map *nodemap=NULL;
1342         int i, j, ret;
1343
1344         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1345         if (ret != 0) {
1346                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
1347                 talloc_free(tmp_ctx);
1348                 return ret;
1349         }
1350
1351         for(i=0;i<nodemap->num;i++){
1352                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1353                         continue;
1354                 }
1355                 if (nodemap->nodes[i].pnn == options.pnn) {
1356                         continue;
1357                 }
1358
1359                 /* read the public ip list from this node */
1360                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips);
1361                 if (ret != 0) {
1362                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %u\n", nodemap->nodes[i].pnn));
1363                         return -1;
1364                 }
1365
1366                 for (j=0;j<ips->num;j++) {
1367                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1368                                 talloc_free(tmp_ctx);
1369                                 return nodemap->nodes[i].pnn;
1370                         }
1371                 }
1372                 talloc_free(ips);
1373         }
1374
1375         talloc_free(tmp_ctx);
1376         return -1;
1377 }
1378
1379 /*
1380   add a public ip address to a node
1381  */
1382 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
1383 {
1384         int i, ret;
1385         int len;
1386         uint32_t pnn;
1387         unsigned mask;
1388         ctdb_sock_addr addr;
1389         struct ctdb_control_ip_iface *pub;
1390         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1391         struct ctdb_all_public_ips *ips;
1392
1393
1394         if (argc != 2) {
1395                 talloc_free(tmp_ctx);
1396                 usage();
1397         }
1398
1399         if (!parse_ip_mask(argv[0], argv[1], &addr, &mask)) {
1400                 DEBUG(DEBUG_ERR, ("Badly formed ip/mask : %s\n", argv[0]));
1401                 talloc_free(tmp_ctx);
1402                 return -1;
1403         }
1404
1405         ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
1406         if (ret != 0) {
1407                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
1408                 talloc_free(tmp_ctx);
1409                 return ret;
1410         }
1411
1412
1413         /* check if some other node is already serving this ip, if not,
1414          * we will claim it
1415          */
1416         for (i=0;i<ips->num;i++) {
1417                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
1418                         break;
1419                 }
1420         }
1421
1422         len = offsetof(struct ctdb_control_ip_iface, iface) + strlen(argv[1]) + 1;
1423         pub = talloc_size(tmp_ctx, len);
1424         CTDB_NO_MEMORY(ctdb, pub);
1425
1426         pub->addr  = addr;
1427         pub->mask  = mask;
1428         pub->len   = strlen(argv[1])+1;
1429         memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
1430
1431         ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
1432         if (ret != 0) {
1433                 DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u\n", options.pnn));
1434                 talloc_free(tmp_ctx);
1435                 return ret;
1436         }
1437
1438         if (i == ips->num) {
1439                 /* no one has this ip so we claim it */
1440                 pnn  = options.pnn;
1441         } else {
1442                 pnn  = ips->ips[i].pnn;
1443         }
1444
1445         if (move_ip(ctdb, &addr, pnn) != 0) {
1446                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
1447                 return -1;
1448         }
1449
1450         talloc_free(tmp_ctx);
1451         return 0;
1452 }
1453
1454 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv);
1455
1456 static int control_delip_all(struct ctdb_context *ctdb, int argc, const char **argv, ctdb_sock_addr *addr)
1457 {
1458         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1459         struct ctdb_node_map *nodemap=NULL;
1460         struct ctdb_all_public_ips *ips;
1461         int ret, i, j;
1462
1463         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
1464         if (ret != 0) {
1465                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from current node\n"));
1466                 return ret;
1467         }
1468
1469         /* remove it from the nodes that are not hosting the ip currently */
1470         for(i=0;i<nodemap->num;i++){
1471                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1472                         continue;
1473                 }
1474                 if (ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips) != 0) {
1475                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
1476                         continue;
1477                 }
1478
1479                 for (j=0;j<ips->num;j++) {
1480                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1481                                 break;
1482                         }
1483                 }
1484                 if (j==ips->num) {
1485                         continue;
1486                 }
1487
1488                 if (ips->ips[j].pnn == nodemap->nodes[i].pnn) {
1489                         continue;
1490                 }
1491
1492                 options.pnn = nodemap->nodes[i].pnn;
1493                 control_delip(ctdb, argc, argv);
1494         }
1495
1496
1497         /* remove it from every node (also the one hosting it) */
1498         for(i=0;i<nodemap->num;i++){
1499                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
1500                         continue;
1501                 }
1502                 if (ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, tmp_ctx, &ips) != 0) {
1503                         DEBUG(DEBUG_ERR, ("Unable to get public ip list from node %d\n", nodemap->nodes[i].pnn));
1504                         continue;
1505                 }
1506
1507                 for (j=0;j<ips->num;j++) {
1508                         if (ctdb_same_ip(addr, &ips->ips[j].addr)) {
1509                                 break;
1510                         }
1511                 }
1512                 if (j==ips->num) {
1513                         continue;
1514                 }
1515
1516                 options.pnn = nodemap->nodes[i].pnn;
1517                 control_delip(ctdb, argc, argv);
1518         }
1519
1520         talloc_free(tmp_ctx);
1521         return 0;
1522 }
1523
1524 /*
1525   delete a public ip address from a node
1526  */
1527 static int control_delip(struct ctdb_context *ctdb, int argc, const char **argv)
1528 {
1529         int i, ret;
1530         ctdb_sock_addr addr;
1531         struct ctdb_control_ip_iface pub;
1532         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1533         struct ctdb_all_public_ips *ips;
1534
1535         if (argc != 1) {
1536                 talloc_free(tmp_ctx);
1537                 usage();
1538         }
1539
1540         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1541                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1542                 return -1;
1543         }
1544
1545         if (options.pnn == CTDB_BROADCAST_ALL) {
1546                 return control_delip_all(ctdb, argc, argv, &addr);
1547         }
1548
1549         pub.addr  = addr;
1550         pub.mask  = 0;
1551         pub.len   = 0;
1552
1553         ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
1554         if (ret != 0) {
1555                 DEBUG(DEBUG_ERR, ("Unable to get public ip list from cluster\n"));
1556                 talloc_free(tmp_ctx);
1557                 return ret;
1558         }
1559
1560         for (i=0;i<ips->num;i++) {
1561                 if (ctdb_same_ip(&addr, &ips->ips[i].addr)) {
1562                         break;
1563                 }
1564         }
1565
1566         if (i==ips->num) {
1567                 DEBUG(DEBUG_ERR, ("This node does not support this public address '%s'\n",
1568                         ctdb_addr_to_str(&addr)));
1569                 talloc_free(tmp_ctx);
1570                 return -1;
1571         }
1572
1573         if (ips->ips[i].pnn == options.pnn) {
1574                 ret = find_other_host_for_public_ip(ctdb, &addr);
1575                 if (ret != -1) {
1576                         if (move_ip(ctdb, &addr, ret) != 0) {
1577                                 DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", ret));
1578                                 return -1;
1579                         }
1580                 }
1581         }
1582
1583         ret = ctdb_ctrl_del_public_ip(ctdb, TIMELIMIT(), options.pnn, &pub);
1584         if (ret != 0) {
1585                 DEBUG(DEBUG_ERR, ("Unable to del public ip from node %u\n", options.pnn));
1586                 talloc_free(tmp_ctx);
1587                 return ret;
1588         }
1589
1590         talloc_free(tmp_ctx);
1591         return 0;
1592 }
1593
1594 /*
1595   kill a tcp connection
1596  */
1597 static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
1598 {
1599         int ret;
1600         struct ctdb_control_killtcp killtcp;
1601
1602         if (argc < 2) {
1603                 usage();
1604         }
1605
1606         if (!parse_ip_port(argv[0], &killtcp.src_addr)) {
1607                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
1608                 return -1;
1609         }
1610
1611         if (!parse_ip_port(argv[1], &killtcp.dst_addr)) {
1612                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
1613                 return -1;
1614         }
1615
1616         ret = ctdb_ctrl_killtcp(ctdb, TIMELIMIT(), options.pnn, &killtcp);
1617         if (ret != 0) {
1618                 DEBUG(DEBUG_ERR, ("Unable to killtcp from node %u\n", options.pnn));
1619                 return ret;
1620         }
1621
1622         return 0;
1623 }
1624
1625
1626 /*
1627   send a gratious arp
1628  */
1629 static int control_gratious_arp(struct ctdb_context *ctdb, int argc, const char **argv)
1630 {
1631         int ret;
1632         ctdb_sock_addr addr;
1633
1634         if (argc < 2) {
1635                 usage();
1636         }
1637
1638         if (!parse_ip(argv[0], NULL, 0, &addr)) {
1639                 DEBUG(DEBUG_ERR, ("Bad IP '%s'\n", argv[0]));
1640                 return -1;
1641         }
1642
1643         ret = ctdb_ctrl_gratious_arp(ctdb, TIMELIMIT(), options.pnn, &addr, argv[1]);
1644         if (ret != 0) {
1645                 DEBUG(DEBUG_ERR, ("Unable to send gratious_arp from node %u\n", options.pnn));
1646                 return ret;
1647         }
1648
1649         return 0;
1650 }
1651
1652 /*
1653   register a server id
1654  */
1655 static int regsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1656 {
1657         int ret;
1658         struct ctdb_server_id server_id;
1659
1660         if (argc < 3) {
1661                 usage();
1662         }
1663
1664         server_id.pnn       = strtoul(argv[0], NULL, 0);
1665         server_id.type      = strtoul(argv[1], NULL, 0);
1666         server_id.server_id = strtoul(argv[2], NULL, 0);
1667
1668         ret = ctdb_ctrl_register_server_id(ctdb, TIMELIMIT(), &server_id);
1669         if (ret != 0) {
1670                 DEBUG(DEBUG_ERR, ("Unable to register server_id from node %u\n", options.pnn));
1671                 return ret;
1672         }
1673         DEBUG(DEBUG_ERR,("Srvid registered. Sleeping for 999 seconds\n"));
1674         sleep(999);
1675         return -1;
1676 }
1677
1678 /*
1679   unregister a server id
1680  */
1681 static int unregsrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1682 {
1683         int ret;
1684         struct ctdb_server_id server_id;
1685
1686         if (argc < 3) {
1687                 usage();
1688         }
1689
1690         server_id.pnn       = strtoul(argv[0], NULL, 0);
1691         server_id.type      = strtoul(argv[1], NULL, 0);
1692         server_id.server_id = strtoul(argv[2], NULL, 0);
1693
1694         ret = ctdb_ctrl_unregister_server_id(ctdb, TIMELIMIT(), &server_id);
1695         if (ret != 0) {
1696                 DEBUG(DEBUG_ERR, ("Unable to unregister server_id from node %u\n", options.pnn));
1697                 return ret;
1698         }
1699         return -1;
1700 }
1701
1702 /*
1703   check if a server id exists
1704  */
1705 static int chksrvid(struct ctdb_context *ctdb, int argc, const char **argv)
1706 {
1707         uint32_t status;
1708         int ret;
1709         struct ctdb_server_id server_id;
1710
1711         if (argc < 3) {
1712                 usage();
1713         }
1714
1715         server_id.pnn       = strtoul(argv[0], NULL, 0);
1716         server_id.type      = strtoul(argv[1], NULL, 0);
1717         server_id.server_id = strtoul(argv[2], NULL, 0);
1718
1719         ret = ctdb_ctrl_check_server_id(ctdb, TIMELIMIT(), options.pnn, &server_id, &status);
1720         if (ret != 0) {
1721                 DEBUG(DEBUG_ERR, ("Unable to check server_id from node %u\n", options.pnn));
1722                 return ret;
1723         }
1724
1725         if (status) {
1726                 printf("Server id %d:%d:%d EXISTS\n", server_id.pnn, server_id.type, server_id.server_id);
1727         } else {
1728                 printf("Server id %d:%d:%d does NOT exist\n", server_id.pnn, server_id.type, server_id.server_id);
1729         }
1730         return 0;
1731 }
1732
1733 /*
1734   get a list of all server ids that are registered on a node
1735  */
1736 static int getsrvids(struct ctdb_context *ctdb, int argc, const char **argv)
1737 {
1738         int i, ret;
1739         struct ctdb_server_id_list *server_ids;
1740
1741         ret = ctdb_ctrl_get_server_id_list(ctdb, ctdb, TIMELIMIT(), options.pnn, &server_ids);
1742         if (ret != 0) {
1743                 DEBUG(DEBUG_ERR, ("Unable to get server_id list from node %u\n", options.pnn));
1744                 return ret;
1745         }
1746
1747         for (i=0; i<server_ids->num; i++) {
1748                 printf("Server id %d:%d:%d\n",
1749                         server_ids->server_ids[i].pnn,
1750                         server_ids->server_ids[i].type,
1751                         server_ids->server_ids[i].server_id);
1752         }
1753
1754         return -1;
1755 }
1756
1757 /*
1758   send a tcp tickle ack
1759  */
1760 static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
1761 {
1762         int ret;
1763         ctdb_sock_addr  src, dst;
1764
1765         if (argc < 2) {
1766                 usage();
1767         }
1768
1769         if (!parse_ip_port(argv[0], &src)) {
1770                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[0]));
1771                 return -1;
1772         }
1773
1774         if (!parse_ip_port(argv[1], &dst)) {
1775                 DEBUG(DEBUG_ERR, ("Bad IP:port '%s'\n", argv[1]));
1776                 return -1;
1777         }
1778
1779         ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
1780         if (ret==0) {
1781                 return 0;
1782         }
1783         DEBUG(DEBUG_ERR, ("Error while sending tickle ack\n"));
1784
1785         return -1;
1786 }
1787
1788
1789 /*
1790   display public ip status
1791  */
1792 static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
1793 {
1794         int i, ret;
1795         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1796         struct ctdb_all_public_ips *ips;
1797
1798         if (options.pnn == CTDB_BROADCAST_ALL) {
1799                 /* read the list of public ips from all nodes */
1800                 ret = control_get_all_public_ips(ctdb, tmp_ctx, &ips);
1801         } else {
1802                 /* read the public ip list from this node */
1803                 ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &ips);
1804         }
1805         if (ret != 0) {
1806                 DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", options.pnn));
1807                 talloc_free(tmp_ctx);
1808                 return ret;
1809         }
1810
1811         if (options.machinereadable){
1812                 printf(":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:\n");
1813         } else {
1814                 if (options.pnn == CTDB_BROADCAST_ALL) {
1815                         printf("Public IPs on ALL nodes\n");
1816                 } else {
1817                         printf("Public IPs on node %u\n", options.pnn);
1818                 }
1819         }
1820
1821         for (i=1;i<=ips->num;i++) {
1822                 struct ctdb_control_public_ip_info *info = NULL;
1823                 int32_t pnn;
1824                 char *aciface = NULL;
1825                 char *avifaces = NULL;
1826                 char *cifaces = NULL;
1827
1828                 if (options.pnn == CTDB_BROADCAST_ALL) {
1829                         pnn = ips->ips[ips->num-i].pnn;
1830                 } else {
1831                         pnn = options.pnn;
1832                 }
1833
1834                 if (pnn != -1) {
1835                         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), pnn, ctdb,
1836                                                    &ips->ips[ips->num-i].addr, &info);
1837                 } else {
1838                         ret = -1;
1839                 }
1840
1841                 if (ret == 0) {
1842                         int j;
1843                         for (j=0; j < info->num; j++) {
1844                                 if (cifaces == NULL) {
1845                                         cifaces = talloc_strdup(info,
1846                                                                 info->ifaces[j].name);
1847                                 } else {
1848                                         cifaces = talloc_asprintf_append(cifaces,
1849                                                                          ",%s",
1850                                                                          info->ifaces[j].name);
1851                                 }
1852
1853                                 if (info->active_idx == j) {
1854                                         aciface = info->ifaces[j].name;
1855                                 }
1856
1857                                 if (info->ifaces[j].link_state == 0) {
1858                                         continue;
1859                                 }
1860
1861                                 if (avifaces == NULL) {
1862                                         avifaces = talloc_strdup(info, info->ifaces[j].name);
1863                                 } else {
1864                                         avifaces = talloc_asprintf_append(avifaces,
1865                                                                           ",%s",
1866                                                                           info->ifaces[j].name);
1867                                 }
1868                         }
1869                 }
1870
1871                 if (options.machinereadable){
1872                         printf(":%s:%d:%s:%s:%s:\n",
1873                                ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
1874                                ips->ips[ips->num-i].pnn,
1875                                aciface?aciface:"",
1876                                avifaces?avifaces:"",
1877                                cifaces?cifaces:"");
1878                 } else {
1879                         printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
1880                                ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
1881                                ips->ips[ips->num-i].pnn,
1882                                aciface?aciface:"",
1883                                avifaces?avifaces:"",
1884                                cifaces?cifaces:"");
1885                 }
1886                 talloc_free(info);
1887         }
1888
1889         talloc_free(tmp_ctx);
1890         return 0;
1891 }
1892
1893 /*
1894   public ip info
1895  */
1896 static int control_ipinfo(struct ctdb_context *ctdb, int argc, const char **argv)
1897 {
1898         int i, ret;
1899         ctdb_sock_addr addr;
1900         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1901         struct ctdb_control_public_ip_info *info;
1902
1903         if (argc != 1) {
1904                 talloc_free(tmp_ctx);
1905                 usage();
1906         }
1907
1908         if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
1909                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
1910                 return -1;
1911         }
1912
1913         /* read the public ip info from this node */
1914         ret = ctdb_ctrl_get_public_ip_info(ctdb, TIMELIMIT(), options.pnn,
1915                                            tmp_ctx, &addr, &info);
1916         if (ret != 0) {
1917                 DEBUG(DEBUG_ERR, ("Unable to get public ip[%s]info from node %u\n",
1918                                   argv[0], options.pnn));
1919                 talloc_free(tmp_ctx);
1920                 return ret;
1921         }
1922
1923         printf("Public IP[%s] info on node %u\n",
1924                ctdb_addr_to_str(&info->ip.addr),
1925                options.pnn);
1926
1927         printf("IP:%s\nCurrentNode:%d\nNumInterfaces:%u\n",
1928                ctdb_addr_to_str(&info->ip.addr),
1929                info->ip.pnn, info->num);
1930
1931         for (i=0; i<info->num; i++) {
1932                 info->ifaces[i].name[CTDB_IFACE_SIZE] = '\0';
1933
1934                 printf("Interface[%u]: Name:%s Link:%s References:%u%s\n",
1935                        i+1, info->ifaces[i].name,
1936                        info->ifaces[i].link_state?"up":"down",
1937                        (unsigned int)info->ifaces[i].references,
1938                        (i==info->active_idx)?" (active)":"");
1939         }
1940
1941         talloc_free(tmp_ctx);
1942         return 0;
1943 }
1944
1945 /*
1946   display interfaces status
1947  */
1948 static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv)
1949 {
1950         int i, ret;
1951         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1952         struct ctdb_control_get_ifaces *ifaces;
1953
1954         /* read the public ip list from this node */
1955         ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(), options.pnn,
1956                                    tmp_ctx, &ifaces);
1957         if (ret != 0) {
1958                 DEBUG(DEBUG_ERR, ("Unable to get interfaces from node %u\n",
1959                                   options.pnn));
1960                 talloc_free(tmp_ctx);
1961                 return ret;
1962         }
1963
1964         if (options.machinereadable){
1965                 printf(":Name:LinkStatus:References:\n");
1966         } else {
1967                 printf("Interfaces on node %u\n", options.pnn);
1968         }
1969
1970         for (i=0; i<ifaces->num; i++) {
1971                 if (options.machinereadable){
1972                         printf(":%s:%s:%u\n",
1973                                ifaces->ifaces[i].name,
1974                                ifaces->ifaces[i].link_state?"1":"0",
1975                                (unsigned int)ifaces->ifaces[i].references);
1976                 } else {
1977                         printf("name:%s link:%s references:%u\n",
1978                                ifaces->ifaces[i].name,
1979                                ifaces->ifaces[i].link_state?"up":"down",
1980                                (unsigned int)ifaces->ifaces[i].references);
1981                 }
1982         }
1983
1984         talloc_free(tmp_ctx);
1985         return 0;
1986 }
1987
1988
1989 /*
1990   set link status of an interface
1991  */
1992 static int control_setifacelink(struct ctdb_context *ctdb, int argc, const char **argv)
1993 {
1994         int ret;
1995         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
1996         struct ctdb_control_iface_info info;
1997
1998         ZERO_STRUCT(info);
1999
2000         if (argc != 2) {
2001                 usage();
2002         }
2003
2004         if (strlen(argv[0]) > CTDB_IFACE_SIZE) {
2005                 DEBUG(DEBUG_ERR, ("interfaces name '%s' too long\n",
2006                                   argv[0]));
2007                 talloc_free(tmp_ctx);
2008                 return -1;
2009         }
2010         strcpy(info.name, argv[0]);
2011
2012         if (strcmp(argv[1], "up") == 0) {
2013                 info.link_state = 1;
2014         } else if (strcmp(argv[1], "down") == 0) {
2015                 info.link_state = 0;
2016         } else {
2017                 DEBUG(DEBUG_ERR, ("link state invalid '%s' should be 'up' or 'down'\n",
2018                                   argv[1]));
2019                 talloc_free(tmp_ctx);
2020                 return -1;
2021         }
2022
2023         /* read the public ip list from this node */
2024         ret = ctdb_ctrl_set_iface_link(ctdb, TIMELIMIT(), options.pnn,
2025                                    tmp_ctx, &info);
2026         if (ret != 0) {
2027                 DEBUG(DEBUG_ERR, ("Unable to set link state for interfaces %s node %u\n",
2028                                   argv[0], options.pnn));
2029                 talloc_free(tmp_ctx);
2030                 return ret;
2031         }
2032
2033         talloc_free(tmp_ctx);
2034         return 0;
2035 }
2036
2037 /*
2038   display pid of a ctdb daemon
2039  */
2040 static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
2041 {
2042         uint32_t pid;
2043         int ret;
2044
2045         ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.pnn, &pid);
2046         if (ret != 0) {
2047                 DEBUG(DEBUG_ERR, ("Unable to get daemon pid from node %u\n", options.pnn));
2048                 return ret;
2049         }
2050         printf("Pid:%d\n", pid);
2051
2052         return 0;
2053 }
2054
2055 static uint32_t ipreallocate_finished;
2056
2057 /*
2058   handler for receiving the response to ipreallocate
2059 */
2060 static void ip_reallocate_handler(struct ctdb_context *ctdb, uint64_t srvid,
2061                              TDB_DATA data, void *private_data)
2062 {
2063         ipreallocate_finished = 1;
2064 }
2065
2066 static void ctdb_every_second(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
2067 {
2068         struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
2069
2070         event_add_timed(ctdb->ev, ctdb,
2071                                 timeval_current_ofs(1, 0),
2072                                 ctdb_every_second, ctdb);
2073 }
2074
2075 /*
2076   ask the recovery daemon on the recovery master to perform a ip reallocation
2077  */
2078 static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
2079 {
2080         int i, ret;
2081         TDB_DATA data;
2082         struct takeover_run_reply rd;
2083         uint32_t recmaster;
2084         struct ctdb_node_map *nodemap=NULL;
2085         int retries=0;
2086         struct timeval tv = timeval_current();
2087
2088         /* we need some events to trigger so we can timeout and restart
2089            the loop
2090         */
2091         event_add_timed(ctdb->ev, ctdb,
2092                                 timeval_current_ofs(1, 0),
2093                                 ctdb_every_second, ctdb);
2094
2095         rd.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
2096         if (rd.pnn == -1) {
2097                 DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
2098                 return -1;
2099         }
2100         rd.srvid = getpid();
2101
2102         /* register a message port for receiveing the reply so that we
2103            can receive the reply
2104         */
2105         ctdb_client_set_message_handler(ctdb, rd.srvid, ip_reallocate_handler, NULL);
2106
2107         data.dptr = (uint8_t *)&rd;
2108         data.dsize = sizeof(rd);
2109
2110 again:
2111         /* check that there are valid nodes available */
2112         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap) != 0) {
2113                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2114                 return -1;
2115         }
2116         for (i=0; i<nodemap->num;i++) {
2117                 if ((nodemap->nodes[i].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) == 0) {
2118                         break;
2119                 }
2120         }
2121         if (i==nodemap->num) {
2122                 DEBUG(DEBUG_ERR,("No recmaster available, no need to wait for cluster convergence\n"));
2123                 return 0;
2124         }
2125
2126
2127         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
2128         if (ret != 0) {
2129                 DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
2130                 return ret;
2131         }
2132
2133         /* verify the node exists */
2134         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), recmaster, ctdb, &nodemap) != 0) {
2135                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2136                 return -1;
2137         }
2138
2139
2140         /* check tha there are nodes available that can act as a recmaster */
2141         for (i=0; i<nodemap->num; i++) {
2142                 if (nodemap->nodes[i].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2143                         continue;
2144                 }
2145                 break;
2146         }
2147         if (i == nodemap->num) {
2148                 DEBUG(DEBUG_ERR,("No possible nodes to host addresses.\n"));
2149                 return 0;
2150         }
2151
2152         /* verify the recovery master is not STOPPED, nor BANNED */
2153         if (nodemap->nodes[recmaster].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2154                 DEBUG(DEBUG_ERR,("No suitable recmaster found. Try again\n"));
2155                 retries++;
2156                 sleep(1);
2157                 goto again;
2158         }
2159
2160
2161         /* verify the recovery master is not STOPPED, nor BANNED */
2162         if (nodemap->nodes[recmaster].flags & (NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)) {
2163                 DEBUG(DEBUG_ERR,("No suitable recmaster found. Try again\n"));
2164                 retries++;
2165                 sleep(1);
2166                 goto again;
2167         }
2168
2169         ipreallocate_finished = 0;
2170         ret = ctdb_client_send_message(ctdb, recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
2171         if (ret != 0) {
2172                 DEBUG(DEBUG_ERR,("Failed to send ip takeover run request message to %u\n", options.pnn));
2173                 return -1;
2174         }
2175
2176         tv = timeval_current();
2177         /* this loop will terminate when we have received the reply */
2178         while (timeval_elapsed(&tv) < 3.0) {
2179                 event_loop_once(ctdb->ev);
2180         }
2181         if (ipreallocate_finished == 1) {
2182                 return 0;
2183         }
2184
2185         DEBUG(DEBUG_ERR,("Timed out waiting for recmaster ipreallocate. Trying again\n"));
2186         retries++;
2187         sleep(1);
2188         goto again;
2189
2190         return 0;
2191 }
2192
2193
2194 /*
2195   disable a remote node
2196  */
2197 static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
2198 {
2199         int ret;
2200         struct ctdb_node_map *nodemap=NULL;
2201
2202         /* check if the node is already disabled */
2203         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2204                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2205                 exit(10);
2206         }
2207         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2208                 DEBUG(DEBUG_ERR,("Node %d is already disabled.\n", options.pnn));
2209                 return 0;
2210         }
2211
2212         do {
2213                 ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn, NODE_FLAGS_PERMANENTLY_DISABLED, 0);
2214                 if (ret != 0) {
2215                         DEBUG(DEBUG_ERR, ("Unable to disable node %u\n", options.pnn));
2216                         return ret;
2217                 }
2218
2219                 sleep(1);
2220
2221                 /* read the nodemap and verify the change took effect */
2222                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2223                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2224                         exit(10);
2225                 }
2226
2227         } while (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED));
2228         ret = control_ipreallocate(ctdb, argc, argv);
2229         if (ret != 0) {
2230                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2231                 return ret;
2232         }
2233
2234         return 0;
2235 }
2236
2237 /*
2238   enable a disabled remote node
2239  */
2240 static int control_enable(struct ctdb_context *ctdb, int argc, const char **argv)
2241 {
2242         int ret;
2243
2244         struct ctdb_node_map *nodemap=NULL;
2245
2246
2247         /* check if the node is already enabled */
2248         if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2249                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2250                 exit(10);
2251         }
2252         if (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED)) {
2253                 DEBUG(DEBUG_ERR,("Node %d is already enabled.\n", options.pnn));
2254                 return 0;
2255         }
2256
2257         do {
2258                 ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.pnn, 0, NODE_FLAGS_PERMANENTLY_DISABLED);
2259                 if (ret != 0) {
2260                         DEBUG(DEBUG_ERR, ("Unable to enable node %u\n", options.pnn));
2261                         return ret;
2262                 }
2263
2264                 sleep(1);
2265
2266                 /* read the nodemap and verify the change took effect */
2267                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2268                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2269                         exit(10);
2270                 }
2271
2272         } while (nodemap->nodes[options.pnn].flags & NODE_FLAGS_PERMANENTLY_DISABLED);
2273
2274         ret = control_ipreallocate(ctdb, argc, argv);
2275         if (ret != 0) {
2276                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2277                 return ret;
2278         }
2279
2280         return 0;
2281 }
2282
2283 /*
2284   stop a remote node
2285  */
2286 static int control_stop(struct ctdb_context *ctdb, int argc, const char **argv)
2287 {
2288         int ret;
2289         struct ctdb_node_map *nodemap=NULL;
2290
2291         do {
2292                 ret = ctdb_ctrl_stop_node(ctdb, TIMELIMIT(), options.pnn);
2293                 if (ret != 0) {
2294                         DEBUG(DEBUG_ERR, ("Unable to stop node %u   try again\n", options.pnn));
2295                 }
2296
2297                 sleep(1);
2298
2299                 /* read the nodemap and verify the change took effect */
2300                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2301                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2302                         exit(10);
2303                 }
2304
2305         } while (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_STOPPED));
2306         ret = control_ipreallocate(ctdb, argc, argv);
2307         if (ret != 0) {
2308                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2309                 return ret;
2310         }
2311
2312         return 0;
2313 }
2314
2315 /*
2316   restart a stopped remote node
2317  */
2318 static int control_continue(struct ctdb_context *ctdb, int argc, const char **argv)
2319 {
2320         int ret;
2321
2322         struct ctdb_node_map *nodemap=NULL;
2323
2324         do {
2325                 ret = ctdb_ctrl_continue_node(ctdb, TIMELIMIT(), options.pnn);
2326                 if (ret != 0) {
2327                         DEBUG(DEBUG_ERR, ("Unable to continue node %u\n", options.pnn));
2328                         return ret;
2329                 }
2330
2331                 sleep(1);
2332
2333                 /* read the nodemap and verify the change took effect */
2334                 if (ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap) != 0) {
2335                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2336                         exit(10);
2337                 }
2338
2339         } while (nodemap->nodes[options.pnn].flags & NODE_FLAGS_STOPPED);
2340         ret = control_ipreallocate(ctdb, argc, argv);
2341         if (ret != 0) {
2342                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2343                 return ret;
2344         }
2345
2346         return 0;
2347 }
2348
2349 static uint32_t get_generation(struct ctdb_context *ctdb)
2350 {
2351         struct ctdb_vnn_map *vnnmap=NULL;
2352         int ret;
2353
2354         /* wait until the recmaster is not in recovery mode */
2355         while (1) {
2356                 uint32_t recmode, recmaster;
2357
2358                 if (vnnmap != NULL) {
2359                         talloc_free(vnnmap);
2360                         vnnmap = NULL;
2361                 }
2362
2363                 /* get the recmaster */
2364                 ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, &recmaster);
2365                 if (ret != 0) {
2366                         DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
2367                         exit(10);
2368                 }
2369
2370                 /* get recovery mode */
2371                 ret = ctdb_ctrl_getrecmode(ctdb, ctdb, TIMELIMIT(), recmaster, &recmode);
2372                 if (ret != 0) {
2373                         DEBUG(DEBUG_ERR, ("Unable to get recmode from node %u\n", options.pnn));
2374                         exit(10);
2375                 }
2376
2377                 /* get the current generation number */
2378                 ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), recmaster, ctdb, &vnnmap);
2379                 if (ret != 0) {
2380                         DEBUG(DEBUG_ERR, ("Unable to get vnnmap from recmaster (%u)\n", recmaster));
2381                         exit(10);
2382                 }
2383
2384                 if ((recmode == CTDB_RECOVERY_NORMAL)
2385                 &&  (vnnmap->generation != 1)){
2386                         return vnnmap->generation;
2387                 }
2388                 sleep(1);
2389         }
2390 }
2391
2392 /*
2393   ban a node from the cluster
2394  */
2395 static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
2396 {
2397         int ret;
2398         struct ctdb_node_map *nodemap=NULL;
2399         struct ctdb_ban_time bantime;
2400
2401         if (argc < 1) {
2402                 usage();
2403         }
2404
2405         /* verify the node exists */
2406         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2407         if (ret != 0) {
2408                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2409                 return ret;
2410         }
2411
2412         if (nodemap->nodes[options.pnn].flags & NODE_FLAGS_BANNED) {
2413                 DEBUG(DEBUG_ERR,("Node %u is already banned.\n", options.pnn));
2414                 return -1;
2415         }
2416
2417         bantime.pnn  = options.pnn;
2418         bantime.time = strtoul(argv[0], NULL, 0);
2419
2420         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, &bantime);
2421         if (ret != 0) {
2422                 DEBUG(DEBUG_ERR,("Banning node %d for %d seconds failed.\n", bantime.pnn, bantime.time));
2423                 return -1;
2424         }
2425
2426         ret = control_ipreallocate(ctdb, argc, argv);
2427         if (ret != 0) {
2428                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2429                 return ret;
2430         }
2431
2432         return 0;
2433 }
2434
2435
2436 /*
2437   unban a node from the cluster
2438  */
2439 static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
2440 {
2441         int ret;
2442         struct ctdb_node_map *nodemap=NULL;
2443         struct ctdb_ban_time bantime;
2444
2445         /* verify the node exists */
2446         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2447         if (ret != 0) {
2448                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2449                 return ret;
2450         }
2451
2452         if (!(nodemap->nodes[options.pnn].flags & NODE_FLAGS_BANNED)) {
2453                 DEBUG(DEBUG_ERR,("Node %u is not banned.\n", options.pnn));
2454                 return -1;
2455         }
2456
2457         bantime.pnn  = options.pnn;
2458         bantime.time = 0;
2459
2460         ret = ctdb_ctrl_set_ban(ctdb, TIMELIMIT(), options.pnn, &bantime);
2461         if (ret != 0) {
2462                 DEBUG(DEBUG_ERR,("Unbanning node %d failed.\n", bantime.pnn));
2463                 return -1;
2464         }
2465
2466         ret = control_ipreallocate(ctdb, argc, argv);
2467         if (ret != 0) {
2468                 DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
2469                 return ret;
2470         }
2471
2472         return 0;
2473 }
2474
2475
2476 /*
2477   show ban information for a node
2478  */
2479 static int control_showban(struct ctdb_context *ctdb, int argc, const char **argv)
2480 {
2481         int ret;
2482         struct ctdb_node_map *nodemap=NULL;
2483         struct ctdb_ban_time *bantime;
2484
2485         /* verify the node exists */
2486         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
2487         if (ret != 0) {
2488                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
2489                 return ret;
2490         }
2491
2492         ret = ctdb_ctrl_get_ban(ctdb, TIMELIMIT(), options.pnn, ctdb, &bantime);
2493         if (ret != 0) {
2494                 DEBUG(DEBUG_ERR,("Showing ban info for node %d failed.\n", options.pnn));
2495                 return -1;
2496         }
2497
2498         if (bantime->time == 0) {
2499                 printf("Node %u is not banned\n", bantime->pnn);
2500         } else {
2501                 printf("Node %u is banned banned for %d seconds\n", bantime->pnn, bantime->time);
2502         }
2503
2504         return 0;
2505 }
2506
2507 /*
2508   shutdown a daemon
2509  */
2510 static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
2511 {
2512         int ret;
2513
2514         ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.pnn);
2515         if (ret != 0) {
2516                 DEBUG(DEBUG_ERR, ("Unable to shutdown node %u\n", options.pnn));
2517                 return ret;
2518         }
2519
2520         return 0;
2521 }
2522
2523 /*
2524   trigger a recovery
2525  */
2526 static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
2527 {
2528         int ret;
2529         uint32_t generation, next_generation;
2530
2531         /* record the current generation number */
2532         generation = get_generation(ctdb);
2533
2534         ret = ctdb_ctrl_freeze_priority(ctdb, TIMELIMIT(), options.pnn, 1);
2535         if (ret != 0) {
2536                 DEBUG(DEBUG_ERR, ("Unable to freeze node\n"));
2537                 return ret;
2538         }
2539
2540         ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
2541         if (ret != 0) {
2542                 DEBUG(DEBUG_ERR, ("Unable to set recovery mode\n"));
2543                 return ret;
2544         }
2545
2546         /* wait until we are in a new generation */
2547         while (1) {
2548                 next_generation = get_generation(ctdb);
2549                 if (next_generation != generation) {
2550                         return 0;
2551                 }
2552                 sleep(1);
2553         }
2554
2555         return 0;
2556 }
2557
2558
2559 /*
2560   display monitoring mode of a remote node
2561  */
2562 static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
2563 {
2564         uint32_t monmode;
2565         int ret;
2566
2567         ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.pnn, &monmode);
2568         if (ret != 0) {
2569                 DEBUG(DEBUG_ERR, ("Unable to get monmode from node %u\n", options.pnn));
2570                 return ret;
2571         }
2572         if (!options.machinereadable){
2573                 printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
2574         } else {
2575                 printf(":mode:\n");
2576                 printf(":%d:\n",monmode);
2577         }
2578         return 0;
2579 }
2580
2581
2582 /*
2583   display capabilities of a remote node
2584  */
2585 static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const char **argv)
2586 {
2587         uint32_t capabilities;
2588         int ret;
2589
2590         ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), options.pnn, &capabilities);
2591         if (ret != 0) {
2592                 DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", options.pnn));
2593                 return ret;
2594         }
2595
2596         if (!options.machinereadable){
2597                 printf("RECMASTER: %s\n", (capabilities&CTDB_CAP_RECMASTER)?"YES":"NO");
2598                 printf("LMASTER: %s\n", (capabilities&CTDB_CAP_LMASTER)?"YES":"NO");
2599                 printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO");
2600                 printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO");
2601         } else {
2602                 printf(":RECMASTER:LMASTER:LVS:NATGW:\n");
2603                 printf(":%d:%d:%d:%d:\n",
2604                         !!(capabilities&CTDB_CAP_RECMASTER),
2605                         !!(capabilities&CTDB_CAP_LMASTER),
2606                         !!(capabilities&CTDB_CAP_LVS),
2607                         !!(capabilities&CTDB_CAP_NATGW));
2608         }
2609         return 0;
2610 }
2611
2612 /*
2613   display lvs configuration
2614  */
2615 static int control_lvs(struct ctdb_context *ctdb, int argc, const char **argv)
2616 {
2617         uint32_t *capabilities;
2618         struct ctdb_node_map *nodemap=NULL;
2619         int i, ret;
2620         int healthy_count = 0;
2621
2622         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
2623         if (ret != 0) {
2624                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2625                 return ret;
2626         }
2627
2628         capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
2629         CTDB_NO_MEMORY(ctdb, capabilities);
2630
2631         /* collect capabilities for all connected nodes */
2632         for (i=0; i<nodemap->num; i++) {
2633                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2634                         continue;
2635                 }
2636                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2637                         continue;
2638                 }
2639
2640                 ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
2641                 if (ret != 0) {
2642                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
2643                         return ret;
2644                 }
2645
2646                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2647                         continue;
2648                 }
2649
2650                 if (!(nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY)) {
2651                         healthy_count++;
2652                 }
2653         }
2654
2655         /* Print all LVS nodes */
2656         for (i=0; i<nodemap->num; i++) {
2657                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2658                         continue;
2659                 }
2660                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2661                         continue;
2662                 }
2663                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2664                         continue;
2665                 }
2666
2667                 if (healthy_count != 0) {
2668                         if (nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY) {
2669                                 continue;
2670                         }
2671                 }
2672
2673                 printf("%d:%s\n", i,
2674                         ctdb_addr_to_str(&nodemap->nodes[i].addr));
2675         }
2676
2677         return 0;
2678 }
2679
2680 /*
2681   display who is the lvs master
2682  */
2683 static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **argv)
2684 {
2685         uint32_t *capabilities;
2686         struct ctdb_node_map *nodemap=NULL;
2687         int i, ret;
2688         int healthy_count = 0;
2689
2690         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
2691         if (ret != 0) {
2692                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
2693                 return ret;
2694         }
2695
2696         capabilities = talloc_array(ctdb, uint32_t, nodemap->num);
2697         CTDB_NO_MEMORY(ctdb, capabilities);
2698
2699         /* collect capabilities for all connected nodes */
2700         for (i=0; i<nodemap->num; i++) {
2701                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2702                         continue;
2703                 }
2704                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2705                         continue;
2706                 }
2707
2708                 ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), i, &capabilities[i]);
2709                 if (ret != 0) {
2710                         DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", i));
2711                         return ret;
2712                 }
2713
2714                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2715                         continue;
2716                 }
2717
2718                 if (!(nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY)) {
2719                         healthy_count++;
2720                 }
2721         }
2722
2723         /* find and show the lvsmaster */
2724         for (i=0; i<nodemap->num; i++) {
2725                 if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
2726                         continue;
2727                 }
2728                 if (nodemap->nodes[i].flags & NODE_FLAGS_PERMANENTLY_DISABLED) {
2729                         continue;
2730                 }
2731                 if (!(capabilities[i] & CTDB_CAP_LVS)) {
2732                         continue;
2733                 }
2734
2735                 if (healthy_count != 0) {
2736                         if (nodemap->nodes[i].flags & NODE_FLAGS_UNHEALTHY) {
2737                                 continue;
2738                         }
2739                 }
2740
2741                 if (options.machinereadable){
2742                         printf("%d\n", i);
2743                 } else {
2744                         printf("Node %d is LVS master\n", i);
2745                 }
2746                 return 0;
2747         }
2748
2749         printf("There is no LVS master\n");
2750         return -1;
2751 }
2752
2753 /*
2754   disable monitoring on a  node
2755  */
2756 static int control_disable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
2757 {
2758
2759         int ret;
2760
2761         ret = ctdb_ctrl_disable_monmode(ctdb, TIMELIMIT(), options.pnn);
2762         if (ret != 0) {
2763                 DEBUG(DEBUG_ERR, ("Unable to disable monmode on node %u\n", options.pnn));
2764                 return ret;
2765         }
2766         printf("Monitoring mode:%s\n","DISABLED");
2767
2768         return 0;
2769 }
2770
2771 /*
2772   enable monitoring on a  node
2773  */
2774 static int control_enable_monmode(struct ctdb_context *ctdb, int argc, const char **argv)
2775 {
2776
2777         int ret;
2778
2779         ret = ctdb_ctrl_enable_monmode(ctdb, TIMELIMIT(), options.pnn);
2780         if (ret != 0) {
2781                 DEBUG(DEBUG_ERR, ("Unable to enable monmode on node %u\n", options.pnn));
2782                 return ret;
2783         }
2784         printf("Monitoring mode:%s\n","ACTIVE");
2785
2786         return 0;
2787 }
2788
2789 /*
2790   display remote list of keys/data for a db
2791  */
2792 static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
2793 {
2794         const char *db_name;
2795         struct ctdb_db_context *ctdb_db;
2796         int ret;
2797
2798         if (argc < 1) {
2799                 usage();
2800         }
2801
2802         db_name = argv[0];
2803
2804
2805         if (db_exists(ctdb, db_name)) {
2806                 DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
2807                 return -1;
2808         }
2809
2810         ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
2811
2812         if (ctdb_db == NULL) {
2813                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
2814                 return -1;
2815         }
2816
2817         /* traverse and dump the cluster tdb */
2818         ret = ctdb_dump_db(ctdb_db, stdout);
2819         if (ret == -1) {
2820                 DEBUG(DEBUG_ERR, ("Unable to dump database\n"));
2821                 DEBUG(DEBUG_ERR, ("Maybe try 'ctdb getdbstatus %s'"
2822                                   " and 'ctdb getvar AllowUnhealthyDBRead'\n",
2823                                   db_name));
2824                 return -1;
2825         }
2826         talloc_free(ctdb_db);
2827
2828         printf("Dumped %d records\n", ret);
2829         return 0;
2830 }
2831
2832
2833 static void log_handler(struct ctdb_context *ctdb, uint64_t srvid,
2834                              TDB_DATA data, void *private_data)
2835 {
2836         DEBUG(DEBUG_ERR,("Log data received\n"));
2837         if (data.dsize > 0) {
2838                 printf("%s", data.dptr);
2839         }
2840
2841         exit(0);
2842 }
2843
2844 /*
2845   display a list of log messages from the in memory ringbuffer
2846  */
2847 static int control_getlog(struct ctdb_context *ctdb, int argc, const char **argv)
2848 {
2849         int ret;
2850         int32_t res;
2851         struct ctdb_get_log_addr log_addr;
2852         TDB_DATA data;
2853         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2854         char *errmsg;
2855         struct timeval tv;
2856
2857         if (argc != 1) {
2858                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
2859                 talloc_free(tmp_ctx);
2860                 return -1;
2861         }
2862
2863         log_addr.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
2864         log_addr.srvid = getpid();
2865         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
2866                 log_addr.level = get_debug_by_desc(argv[0]);
2867         } else {
2868                 log_addr.level = strtol(argv[0], NULL, 0);
2869         }
2870
2871
2872         data.dptr = (unsigned char *)&log_addr;
2873         data.dsize = sizeof(log_addr);
2874
2875         DEBUG(DEBUG_ERR, ("Pulling logs from node %u\n", options.pnn));
2876
2877         ctdb_client_set_message_handler(ctdb, log_addr.srvid, log_handler, NULL);
2878         sleep(1);
2879
2880         DEBUG(DEBUG_ERR,("Listen for response on %d\n", (int)log_addr.srvid));
2881
2882         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_GET_LOG,
2883                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
2884         if (ret != 0 || res != 0) {
2885                 DEBUG(DEBUG_ERR,("Failed to get logs - %s\n", errmsg));
2886                 talloc_free(tmp_ctx);
2887                 return -1;
2888         }
2889
2890
2891         tv = timeval_current();
2892         /* this loop will terminate when we have received the reply */
2893         while (timeval_elapsed(&tv) < 3.0) {
2894                 event_loop_once(ctdb->ev);
2895         }
2896
2897         DEBUG(DEBUG_INFO,("Timed out waiting for log data.\n"));
2898
2899         talloc_free(tmp_ctx);
2900         return 0;
2901 }
2902
2903 /*
2904   clear the in memory log area
2905  */
2906 static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **argv)
2907 {
2908         int ret;
2909         int32_t res;
2910         char *errmsg;
2911         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
2912
2913         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_CLEAR_LOG,
2914                            0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
2915         if (ret != 0 || res != 0) {
2916                 DEBUG(DEBUG_ERR,("Failed to clear logs\n"));
2917                 talloc_free(tmp_ctx);
2918                 return -1;
2919         }
2920
2921         talloc_free(tmp_ctx);
2922         return 0;
2923 }
2924
2925
2926
2927 /*
2928   display a list of the databases on a remote ctdb
2929  */
2930 static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
2931 {
2932         int i, ret;
2933         struct ctdb_dbid_map *dbmap=NULL;
2934
2935         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
2936         if (ret != 0) {
2937                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
2938                 return ret;
2939         }
2940
2941         if(options.machinereadable){
2942                 printf(":ID:Name:Path:Persistent:Unhealthy:\n");
2943                 for(i=0;i<dbmap->num;i++){
2944                         const char *path;
2945                         const char *name;
2946                         const char *health;
2947                         bool persistent;
2948
2949                         ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn,
2950                                             dbmap->dbs[i].dbid, ctdb, &path);
2951                         ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
2952                                             dbmap->dbs[i].dbid, ctdb, &name);
2953                         ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
2954                                               dbmap->dbs[i].dbid, ctdb, &health);
2955                         persistent = dbmap->dbs[i].persistent;
2956                         printf(":0x%08X:%s:%s:%d:%d:\n",
2957                                dbmap->dbs[i].dbid, name, path,
2958                                !!(persistent), !!(health));
2959                 }
2960                 return 0;
2961         }
2962
2963         printf("Number of databases:%d\n", dbmap->num);
2964         for(i=0;i<dbmap->num;i++){
2965                 const char *path;
2966                 const char *name;
2967                 const char *health;
2968                 bool persistent;
2969
2970                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
2971                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
2972                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
2973                 persistent = dbmap->dbs[i].persistent;
2974                 printf("dbid:0x%08x name:%s path:%s%s%s\n",
2975                        dbmap->dbs[i].dbid, name, path,
2976                        persistent?" PERSISTENT":"",
2977                        health?" UNHEALTHY":"");
2978         }
2979
2980         return 0;
2981 }
2982
2983 /*
2984   display the status of a database on a remote ctdb
2985  */
2986 static int control_getdbstatus(struct ctdb_context *ctdb, int argc, const char **argv)
2987 {
2988         int i, ret;
2989         struct ctdb_dbid_map *dbmap=NULL;
2990         const char *db_name;
2991
2992         if (argc < 1) {
2993                 usage();
2994         }
2995
2996         db_name = argv[0];
2997
2998         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, ctdb, &dbmap);
2999         if (ret != 0) {
3000                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
3001                 return ret;
3002         }
3003
3004         for(i=0;i<dbmap->num;i++){
3005                 const char *path;
3006                 const char *name;
3007                 const char *health;
3008                 bool persistent;
3009
3010                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &name);
3011                 if (strcmp(name, db_name) != 0) {
3012                         continue;
3013                 }
3014
3015                 ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &path);
3016                 ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, ctdb, &health);
3017                 persistent = dbmap->dbs[i].persistent;
3018                 printf("dbid: 0x%08x\nname: %s\npath: %s\nPERSISTENT: %s\nHEALTH: %s\n",
3019                        dbmap->dbs[i].dbid, name, path,
3020                        persistent?"yes":"no",
3021                        health?health:"OK");
3022                 return 0;
3023         }
3024
3025         DEBUG(DEBUG_ERR, ("db %s doesn't exist on node %u\n", db_name, options.pnn));
3026         return 0;
3027 }
3028
3029 /*
3030   check if the local node is recmaster or not
3031   it will return 1 if this node is the recmaster and 0 if it is not
3032   or if the local ctdb daemon could not be contacted
3033  */
3034 static int control_isnotrecmaster(struct ctdb_context *ctdb, int argc, const char **argv)
3035 {
3036         uint32_t mypnn, recmaster;
3037         int ret;
3038
3039         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
3040         if (mypnn == -1) {
3041                 printf("Failed to get pnn of node\n");
3042                 return 1;
3043         }
3044
3045         ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
3046         if (ret != 0) {
3047                 printf("Failed to get the recmaster\n");
3048                 return 1;
3049         }
3050
3051         if (recmaster != mypnn) {
3052                 printf("this node is not the recmaster\n");
3053                 return 1;
3054         }
3055
3056         printf("this node is the recmaster\n");
3057         return 0;
3058 }
3059
3060 /*
3061   ping a node
3062  */
3063 static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
3064 {
3065         int ret;
3066         struct timeval tv = timeval_current();
3067         ret = ctdb_ctrl_ping(ctdb, options.pnn);
3068         if (ret == -1) {
3069                 printf("Unable to get ping response from node %u\n", options.pnn);
3070                 return -1;
3071         } else {
3072                 printf("response from %u time=%.6f sec  (%d clients)\n",
3073                        options.pnn, timeval_elapsed(&tv), ret);
3074         }
3075         return 0;
3076 }
3077
3078
3079 /*
3080   get a tunable
3081  */
3082 static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
3083 {
3084         const char *name;
3085         uint32_t value;
3086         int ret;
3087
3088         if (argc < 1) {
3089                 usage();
3090         }
3091
3092         name = argv[0];
3093         ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn, name, &value);
3094         if (ret == -1) {
3095                 DEBUG(DEBUG_ERR, ("Unable to get tunable variable '%s'\n", name));
3096                 return -1;
3097         }
3098
3099         printf("%-19s = %u\n", name, value);
3100         return 0;
3101 }
3102
3103 /*
3104   set a tunable
3105  */
3106 static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
3107 {
3108         const char *name;
3109         uint32_t value;
3110         int ret;
3111
3112         if (argc < 2) {
3113                 usage();
3114         }
3115
3116         name = argv[0];
3117         value = strtoul(argv[1], NULL, 0);
3118
3119         ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.pnn, name, value);
3120         if (ret == -1) {
3121                 DEBUG(DEBUG_ERR, ("Unable to set tunable variable '%s'\n", name));
3122                 return -1;
3123         }
3124         return 0;
3125 }
3126
3127 /*
3128   list all tunables
3129  */
3130 static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
3131 {
3132         uint32_t count;
3133         const char **list;
3134         int ret, i;
3135
3136         ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.pnn, ctdb, &list, &count);
3137         if (ret == -1) {
3138                 DEBUG(DEBUG_ERR, ("Unable to list tunable variables\n"));
3139                 return -1;
3140         }
3141
3142         for (i=0;i<count;i++) {
3143                 control_getvar(ctdb, 1, &list[i]);
3144         }
3145
3146         talloc_free(list);
3147
3148         return 0;
3149 }
3150
3151 /*
3152   display debug level on a node
3153  */
3154 static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
3155 {
3156         int ret;
3157         int32_t level;
3158
3159         ret = ctdb_ctrl_get_debuglevel(ctdb, options.pnn, &level);
3160         if (ret != 0) {
3161                 DEBUG(DEBUG_ERR, ("Unable to get debuglevel response from node %u\n", options.pnn));
3162                 return ret;
3163         } else {
3164                 if (options.machinereadable){
3165                         printf(":Name:Level:\n");
3166                         printf(":%s:%d:\n",get_debug_by_level(level),level);
3167                 } else {
3168                         printf("Node %u is at debug level %s (%d)\n", options.pnn, get_debug_by_level(level), level);
3169                 }
3170         }
3171         return 0;
3172 }
3173
3174 /*
3175   display reclock file of a node
3176  */
3177 static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **argv)
3178 {
3179         int ret;
3180         const char *reclock;
3181
3182         ret = ctdb_ctrl_getreclock(ctdb, TIMELIMIT(), options.pnn, ctdb, &reclock);
3183         if (ret != 0) {
3184                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
3185                 return ret;
3186         } else {
3187                 if (options.machinereadable){
3188                         if (reclock != NULL) {
3189                                 printf("%s", reclock);
3190                         }
3191                 } else {
3192                         if (reclock == NULL) {
3193                                 printf("No reclock file used.\n");
3194                         } else {
3195                                 printf("Reclock file:%s\n", reclock);
3196                         }
3197                 }
3198         }
3199         return 0;
3200 }
3201
3202 /*
3203   set the reclock file of a node
3204  */
3205 static int control_setreclock(struct ctdb_context *ctdb, int argc, const char **argv)
3206 {
3207         int ret;
3208         const char *reclock;
3209
3210         if (argc == 0) {
3211                 reclock = NULL;
3212         } else if (argc == 1) {
3213                 reclock = argv[0];
3214         } else {
3215                 usage();
3216         }
3217
3218         ret = ctdb_ctrl_setreclock(ctdb, TIMELIMIT(), options.pnn, reclock);
3219         if (ret != 0) {
3220                 DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
3221                 return ret;
3222         }
3223         return 0;
3224 }
3225
3226 /*
3227   set the natgw state on/off
3228  */
3229 static int control_setnatgwstate(struct ctdb_context *ctdb, int argc, const char **argv)
3230 {
3231         int ret;
3232         uint32_t natgwstate;
3233
3234         if (argc == 0) {
3235                 usage();
3236         }
3237
3238         if (!strcmp(argv[0], "on")) {
3239                 natgwstate = 1;
3240         } else if (!strcmp(argv[0], "off")) {
3241                 natgwstate = 0;
3242         } else {
3243                 usage();
3244         }
3245
3246         ret = ctdb_ctrl_setnatgwstate(ctdb, TIMELIMIT(), options.pnn, natgwstate);
3247         if (ret != 0) {
3248                 DEBUG(DEBUG_ERR, ("Unable to set the natgw state for node %u\n", options.pnn));
3249                 return ret;
3250         }
3251
3252         return 0;
3253 }
3254
3255 /*
3256   set the lmaster role on/off
3257  */
3258 static int control_setlmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
3259 {
3260         int ret;
3261         uint32_t lmasterrole;
3262
3263         if (argc == 0) {
3264                 usage();
3265         }
3266
3267         if (!strcmp(argv[0], "on")) {
3268                 lmasterrole = 1;
3269         } else if (!strcmp(argv[0], "off")) {
3270                 lmasterrole = 0;
3271         } else {
3272                 usage();
3273         }
3274
3275         ret = ctdb_ctrl_setlmasterrole(ctdb, TIMELIMIT(), options.pnn, lmasterrole);
3276         if (ret != 0) {
3277                 DEBUG(DEBUG_ERR, ("Unable to set the lmaster role for node %u\n", options.pnn));
3278                 return ret;
3279         }
3280
3281         return 0;
3282 }
3283
3284 /*
3285   set the recmaster role on/off
3286  */
3287 static int control_setrecmasterrole(struct ctdb_context *ctdb, int argc, const char **argv)
3288 {
3289         int ret;
3290         uint32_t recmasterrole;
3291
3292         if (argc == 0) {
3293                 usage();
3294         }
3295
3296         if (!strcmp(argv[0], "on")) {
3297                 recmasterrole = 1;
3298         } else if (!strcmp(argv[0], "off")) {
3299                 recmasterrole = 0;
3300         } else {
3301                 usage();
3302         }
3303
3304         ret = ctdb_ctrl_setrecmasterrole(ctdb, TIMELIMIT(), options.pnn, recmasterrole);
3305         if (ret != 0) {
3306                 DEBUG(DEBUG_ERR, ("Unable to set the recmaster role for node %u\n", options.pnn));
3307                 return ret;
3308         }
3309
3310         return 0;
3311 }
3312
3313 /*
3314   set debug level on a node or all nodes
3315  */
3316 static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
3317 {
3318         int i, ret;
3319         int32_t level;
3320
3321         if (argc == 0) {
3322                 printf("You must specify the debug level. Valid levels are:\n");
3323                 for (i=0; debug_levels[i].description != NULL; i++) {
3324                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
3325                 }
3326
3327                 return 0;
3328         }
3329
3330         if (isalpha(argv[0][0]) || argv[0][0] == '-') {
3331                 level = get_debug_by_desc(argv[0]);
3332         } else {
3333                 level = strtol(argv[0], NULL, 0);
3334         }
3335
3336         for (i=0; debug_levels[i].description != NULL; i++) {
3337                 if (level == debug_levels[i].level) {
3338                         break;
3339                 }
3340         }
3341         if (debug_levels[i].description == NULL) {
3342                 printf("Invalid debug level, must be one of\n");
3343                 for (i=0; debug_levels[i].description != NULL; i++) {
3344                         printf("%s (%d)\n", debug_levels[i].description, debug_levels[i].level);
3345                 }
3346                 return -1;
3347         }
3348
3349         ret = ctdb_ctrl_set_debuglevel(ctdb, options.pnn, level);
3350         if (ret != 0) {
3351                 DEBUG(DEBUG_ERR, ("Unable to set debug level on node %u\n", options.pnn));
3352         }
3353         return 0;
3354 }
3355
3356
3357 /*
3358   thaw a node
3359  */
3360 static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
3361 {
3362         int ret;
3363         uint32_t priority;
3364
3365         if (argc == 1) {
3366                 priority = strtol(argv[0], NULL, 0);
3367         } else {
3368                 priority = 0;
3369         }
3370         DEBUG(DEBUG_ERR,("Thaw by priority %u\n", priority));
3371
3372         ret = ctdb_ctrl_thaw_priority(ctdb, TIMELIMIT(), options.pnn, priority);
3373         if (ret != 0) {
3374                 DEBUG(DEBUG_ERR, ("Unable to thaw node %u\n", options.pnn));
3375         }
3376         return 0;
3377 }
3378
3379
3380 /*
3381   attach to a database
3382  */
3383 static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
3384 {
3385         const char *db_name;
3386         struct ctdb_db_context *ctdb_db;
3387
3388         if (argc < 1) {
3389                 usage();
3390         }
3391         db_name = argv[0];
3392
3393         ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
3394         if (ctdb_db == NULL) {
3395                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
3396                 return -1;
3397         }
3398
3399         return 0;
3400 }
3401
3402 /*
3403   set db priority
3404  */
3405 static int control_setdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
3406 {
3407         struct ctdb_db_priority db_prio;
3408         int ret;
3409
3410         if (argc < 2) {
3411                 usage();
3412         }
3413
3414         db_prio.db_id    = strtoul(argv[0], NULL, 0);
3415         db_prio.priority = strtoul(argv[1], NULL, 0);
3416
3417         ret = ctdb_ctrl_set_db_priority(ctdb, TIMELIMIT(), options.pnn, &db_prio);
3418         if (ret != 0) {
3419                 DEBUG(DEBUG_ERR,("Unable to set db prio\n"));
3420                 return -1;
3421         }
3422
3423         return 0;
3424 }
3425
3426 /*
3427   get db priority
3428  */
3429 static int control_getdbprio(struct ctdb_context *ctdb, int argc, const char **argv)
3430 {
3431         uint32_t db_id, priority;
3432         int ret;
3433
3434         if (argc < 1) {
3435                 usage();
3436         }
3437
3438         db_id = strtoul(argv[0], NULL, 0);
3439
3440         ret = ctdb_ctrl_get_db_priority(ctdb, TIMELIMIT(), options.pnn, db_id, &priority);
3441         if (ret != 0) {
3442                 DEBUG(DEBUG_ERR,("Unable to get db prio\n"));
3443                 return -1;
3444         }
3445
3446         DEBUG(DEBUG_ERR,("Priority:%u\n", priority));
3447
3448         return 0;
3449 }
3450
3451 /*
3452   run an eventscript on a node
3453  */
3454 static int control_eventscript(struct ctdb_context *ctdb, int argc, const char **argv)
3455 {
3456         TDB_DATA data;
3457         int ret;
3458         int32_t res;
3459         char *errmsg;
3460         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3461
3462         if (argc != 1) {
3463                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3464                 return -1;
3465         }
3466
3467         data.dptr = (unsigned char *)discard_const(argv[0]);
3468         data.dsize = strlen((char *)data.dptr) + 1;
3469
3470         DEBUG(DEBUG_ERR, ("Running eventscripts with arguments \"%s\" on node %u\n", data.dptr, options.pnn));
3471
3472         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_RUN_EVENTSCRIPTS,
3473                            0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
3474         if (ret != 0 || res != 0) {
3475                 DEBUG(DEBUG_ERR,("Failed to run eventscripts - %s\n", errmsg));
3476                 talloc_free(tmp_ctx);
3477                 return -1;
3478         }
3479         talloc_free(tmp_ctx);
3480         return 0;
3481 }
3482
3483 #define DB_VERSION 1
3484 #define MAX_DB_NAME 64
3485 struct db_file_header {
3486         unsigned long version;
3487         time_t timestamp;
3488         unsigned long persistent;
3489         unsigned long size;
3490         const char name[MAX_DB_NAME];
3491 };
3492
3493 struct backup_data {
3494         struct ctdb_marshall_buffer *records;
3495         uint32_t len;
3496         uint32_t total;
3497         bool traverse_error;
3498 };
3499
3500 static int backup_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private)
3501 {
3502         struct backup_data *bd = talloc_get_type(private, struct backup_data);
3503         struct ctdb_rec_data *rec;
3504
3505         /* add the record */
3506         rec = ctdb_marshall_record(bd->records, 0, key, NULL, data);
3507         if (rec == NULL) {
3508                 bd->traverse_error = true;
3509                 DEBUG(DEBUG_ERR,("Failed to marshall record\n"));
3510                 return -1;
3511         }
3512         bd->records = talloc_realloc_size(NULL, bd->records, rec->length + bd->len);
3513         if (bd->records == NULL) {
3514                 DEBUG(DEBUG_ERR,("Failed to expand marshalling buffer\n"));
3515                 bd->traverse_error = true;
3516                 return -1;
3517         }
3518         bd->records->count++;
3519         memcpy(bd->len+(uint8_t *)bd->records, rec, rec->length);
3520         bd->len += rec->length;
3521         talloc_free(rec);
3522
3523         bd->total++;
3524         return 0;
3525 }
3526
3527 /*
3528  * backup a database to a file
3529  */
3530 static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **argv)
3531 {
3532         int i, ret;
3533         struct ctdb_dbid_map *dbmap=NULL;
3534         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3535         struct db_file_header dbhdr;
3536         struct ctdb_db_context *ctdb_db;
3537         struct backup_data *bd;
3538         int fh = -1;
3539         int status = -1;
3540         const char *reason = NULL;
3541
3542         if (argc != 2) {
3543                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3544                 return -1;
3545         }
3546
3547         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
3548         if (ret != 0) {
3549                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
3550                 return ret;
3551         }
3552
3553         for(i=0;i<dbmap->num;i++){
3554                 const char *name;
3555
3556                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
3557                 if(!strcmp(argv[0], name)){
3558                         talloc_free(discard_const(name));
3559                         break;
3560                 }
3561                 talloc_free(discard_const(name));
3562         }
3563         if (i == dbmap->num) {
3564                 DEBUG(DEBUG_ERR,("No database with name '%s' found\n", argv[0]));
3565                 talloc_free(tmp_ctx);
3566                 return -1;
3567         }
3568
3569         ret = ctdb_ctrl_getdbhealth(ctdb, TIMELIMIT(), options.pnn,
3570                                     dbmap->dbs[i].dbid, tmp_ctx, &reason);
3571         if (ret != 0) {
3572                 DEBUG(DEBUG_ERR,("Unable to get dbhealth for database '%s'\n",
3573                                  argv[0]));
3574                 talloc_free(tmp_ctx);
3575                 return -1;
3576         }
3577         if (reason) {
3578                 uint32_t allow_unhealthy = 0;
3579
3580                 ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.pnn,
3581                                       "AllowUnhealthyDBRead",
3582                                       &allow_unhealthy);
3583
3584                 if (allow_unhealthy != 1) {
3585                         DEBUG(DEBUG_ERR,("database '%s' is unhealthy: %s\n",
3586                                          argv[0], reason));
3587
3588                         DEBUG(DEBUG_ERR,("disallow backup : tunnable AllowUnhealthyDBRead = %u\n",
3589                                          allow_unhealthy));
3590                         talloc_free(tmp_ctx);
3591                         return -1;
3592                 }
3593
3594                 DEBUG(DEBUG_WARNING,("WARNING database '%s' is unhealthy - see 'ctdb getdbstatus %s'\n",
3595                                      argv[0], argv[0]));
3596                 DEBUG(DEBUG_WARNING,("WARNING! allow backup of unhealthy database: "
3597                                      "tunnable AllowUnhealthyDBRead = %u\n",
3598                                      allow_unhealthy));
3599         }
3600
3601         ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
3602         if (ctdb_db == NULL) {
3603                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
3604                 talloc_free(tmp_ctx);
3605                 return -1;
3606         }
3607
3608
3609         ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
3610         if (ret == -1) {
3611                 DEBUG(DEBUG_ERR,("Failed to start transaction\n"));
3612                 talloc_free(tmp_ctx);
3613                 return -1;
3614         }
3615
3616
3617         bd = talloc_zero(tmp_ctx, struct backup_data);
3618         if (bd == NULL) {
3619                 DEBUG(DEBUG_ERR,("Failed to allocate backup_data\n"));
3620                 talloc_free(tmp_ctx);
3621                 return -1;
3622         }
3623
3624         bd->records = talloc_zero(bd, struct ctdb_marshall_buffer);
3625         if (bd->records == NULL) {
3626                 DEBUG(DEBUG_ERR,("Failed to allocate ctdb_marshall_buffer\n"));
3627                 talloc_free(tmp_ctx);
3628                 return -1;
3629         }
3630
3631         bd->len = offsetof(struct ctdb_marshall_buffer, data);
3632         bd->records->db_id = ctdb_db->db_id;
3633         /* traverse the database collecting all records */
3634         if (tdb_traverse_read(ctdb_db->ltdb->tdb, backup_traverse, bd) == -1 ||
3635             bd->traverse_error) {
3636                 DEBUG(DEBUG_ERR,("Traverse error\n"));
3637                 talloc_free(tmp_ctx);
3638                 return -1;
3639         }
3640
3641         tdb_transaction_cancel(ctdb_db->ltdb->tdb);
3642
3643
3644         fh = open(argv[1], O_RDWR|O_CREAT, 0600);
3645         if (fh == -1) {
3646                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[1]));
3647                 talloc_free(tmp_ctx);
3648                 return -1;
3649         }
3650
3651         dbhdr.version = DB_VERSION;
3652         dbhdr.timestamp = time(NULL);
3653         dbhdr.persistent = dbmap->dbs[i].persistent;
3654         dbhdr.size = bd->len;
3655         if (strlen(argv[0]) >= MAX_DB_NAME) {
3656                 DEBUG(DEBUG_ERR,("Too long dbname\n"));
3657                 goto done;
3658         }
3659         strncpy(discard_const(dbhdr.name), argv[0], MAX_DB_NAME);
3660         ret = write(fh, &dbhdr, sizeof(dbhdr));
3661         if (ret == -1) {
3662                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
3663                 goto done;
3664         }
3665         ret = write(fh, bd->records, bd->len);
3666         if (ret == -1) {
3667                 DEBUG(DEBUG_ERR,("write failed: %s\n", strerror(errno)));
3668                 goto done;
3669         }
3670
3671         status = 0;
3672 done:
3673         if (fh != -1) {
3674                 ret = close(fh);
3675                 if (ret == -1) {
3676                         DEBUG(DEBUG_ERR,("close failed: %s\n", strerror(errno)));
3677                 }
3678         }
3679         talloc_free(tmp_ctx);
3680         return status;
3681 }
3682
3683 /*
3684  * restore a database from a file
3685  */
3686 static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **argv)
3687 {
3688         int ret;
3689         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3690         TDB_DATA outdata;
3691         TDB_DATA data;
3692         struct db_file_header dbhdr;
3693         struct ctdb_db_context *ctdb_db;
3694         struct ctdb_node_map *nodemap=NULL;
3695         struct ctdb_vnn_map *vnnmap=NULL;
3696         int i, fh;
3697         struct ctdb_control_wipe_database w;
3698         uint32_t *nodes;
3699         uint32_t generation;
3700         struct tm *tm;
3701         char tbuf[100];
3702         char *dbname;
3703
3704         if (argc < 1 || argc > 2) {
3705                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3706                 return -1;
3707         }
3708
3709         fh = open(argv[0], O_RDONLY);
3710         if (fh == -1) {
3711                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
3712                 talloc_free(tmp_ctx);
3713                 return -1;
3714         }
3715
3716         read(fh, &dbhdr, sizeof(dbhdr));
3717         if (dbhdr.version != DB_VERSION) {
3718                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
3719                 talloc_free(tmp_ctx);
3720                 return -1;
3721         }
3722
3723         dbname = dbhdr.name;
3724         if (argc == 2) {
3725                 dbname = argv[1];
3726         }
3727
3728         outdata.dsize = dbhdr.size;
3729         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
3730         if (outdata.dptr == NULL) {
3731                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
3732                 close(fh);
3733                 talloc_free(tmp_ctx);
3734                 return -1;
3735         }
3736         read(fh, outdata.dptr, outdata.dsize);
3737         close(fh);
3738
3739         tm = localtime(&dbhdr.timestamp);
3740         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
3741         printf("Restoring database '%s' from backup @ %s\n",
3742                 dbname, tbuf);
3743
3744
3745         ctdb_db = ctdb_attach(ctdb, dbname, dbhdr.persistent, 0);
3746         if (ctdb_db == NULL) {
3747                 DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
3748                 talloc_free(tmp_ctx);
3749                 return -1;
3750         }
3751
3752         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
3753         if (ret != 0) {
3754                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
3755                 talloc_free(tmp_ctx);
3756                 return ret;
3757         }
3758
3759
3760         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &vnnmap);
3761         if (ret != 0) {
3762                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n", options.pnn));
3763                 talloc_free(tmp_ctx);
3764                 return ret;
3765         }
3766
3767         /* freeze all nodes */
3768         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3769         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
3770                 if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
3771                                         nodes, i,
3772                                         TIMELIMIT(),
3773                                         false, tdb_null,
3774                                         NULL, NULL,
3775                                         NULL) != 0) {
3776                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
3777                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3778                         talloc_free(tmp_ctx);
3779                         return -1;
3780                 }
3781         }
3782
3783         generation = vnnmap->generation;
3784         data.dptr = (void *)&generation;
3785         data.dsize = sizeof(generation);
3786
3787         /* start a cluster wide transaction */
3788         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3789         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
3790                                         nodes, 0,
3791                                         TIMELIMIT(), false, data,
3792                                         NULL, NULL,
3793                                         NULL) != 0) {
3794                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide transactions.\n"));
3795                 return -1;
3796         }
3797
3798
3799         w.db_id = ctdb_db->db_id;
3800         w.transaction_id = generation;
3801
3802         data.dptr = (void *)&w;
3803         data.dsize = sizeof(w);
3804
3805         /* wipe all the remote databases. */
3806         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3807         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
3808                                         nodes, 0,
3809                                         TIMELIMIT(), false, data,
3810                                         NULL, NULL,
3811                                         NULL) != 0) {
3812                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
3813                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3814                 talloc_free(tmp_ctx);
3815                 return -1;
3816         }
3817
3818         /* push the database */
3819         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3820         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
3821                                         nodes, 0,
3822                                         TIMELIMIT(), false, outdata,
3823                                         NULL, NULL,
3824                                         NULL) != 0) {
3825                 DEBUG(DEBUG_ERR, ("Failed to push database.\n"));
3826                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3827                 talloc_free(tmp_ctx);
3828                 return -1;
3829         }
3830
3831         data.dptr = (void *)&ctdb_db->db_id;
3832         data.dsize = sizeof(ctdb_db->db_id);
3833
3834         /* mark the database as healthy */
3835         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3836         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
3837                                         nodes, 0,
3838                                         TIMELIMIT(), false, data,
3839                                         NULL, NULL,
3840                                         NULL) != 0) {
3841                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
3842                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3843                 talloc_free(tmp_ctx);
3844                 return -1;
3845         }
3846
3847         data.dptr = (void *)&generation;
3848         data.dsize = sizeof(generation);
3849
3850         /* commit all the changes */
3851         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
3852                                         nodes, 0,
3853                                         TIMELIMIT(), false, data,
3854                                         NULL, NULL,
3855                                         NULL) != 0) {
3856                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
3857                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3858                 talloc_free(tmp_ctx);
3859                 return -1;
3860         }
3861
3862
3863         /* thaw all nodes */
3864         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
3865         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
3866                                         nodes, 0,
3867                                         TIMELIMIT(),
3868                                         false, tdb_null,
3869                                         NULL, NULL,
3870                                         NULL) != 0) {
3871                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
3872                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
3873                 talloc_free(tmp_ctx);
3874                 return -1;
3875         }
3876
3877
3878         talloc_free(tmp_ctx);
3879         return 0;
3880 }
3881
3882 /*
3883  * dump a database backup from a file
3884  */
3885 static int control_dumpdbbackup(struct ctdb_context *ctdb, int argc, const char **argv)
3886 {
3887         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3888         TDB_DATA outdata;
3889         struct db_file_header dbhdr;
3890         int i, fh;
3891         struct tm *tm;
3892         char tbuf[100];
3893         struct ctdb_rec_data *rec = NULL;
3894         struct ctdb_marshall_buffer *m;
3895
3896         if (argc != 1) {
3897                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3898                 return -1;
3899         }
3900
3901         fh = open(argv[0], O_RDONLY);
3902         if (fh == -1) {
3903                 DEBUG(DEBUG_ERR,("Failed to open file '%s'\n", argv[0]));
3904                 talloc_free(tmp_ctx);
3905                 return -1;
3906         }
3907
3908         read(fh, &dbhdr, sizeof(dbhdr));
3909         if (dbhdr.version != DB_VERSION) {
3910                 DEBUG(DEBUG_ERR,("Invalid version of database dump. File is version %lu but expected version was %u\n", dbhdr.version, DB_VERSION));
3911                 talloc_free(tmp_ctx);
3912                 return -1;
3913         }
3914
3915         outdata.dsize = dbhdr.size;
3916         outdata.dptr = talloc_size(tmp_ctx, outdata.dsize);
3917         if (outdata.dptr == NULL) {
3918                 DEBUG(DEBUG_ERR,("Failed to allocate data of size '%lu'\n", dbhdr.size));
3919                 close(fh);
3920                 talloc_free(tmp_ctx);
3921                 return -1;
3922         }
3923         read(fh, outdata.dptr, outdata.dsize);
3924         close(fh);
3925         m = (struct ctdb_marshall_buffer *)outdata.dptr;
3926
3927         tm = localtime(&dbhdr.timestamp);
3928         strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
3929         printf("Backup of database name:'%s' dbid:0x%x08x from @ %s\n",
3930                 dbhdr.name, m->db_id, tbuf);
3931
3932         for (i=0; i < m->count; i++) {
3933                 uint32_t reqid = 0;
3934                 TDB_DATA key, data;
3935
3936                 /* we do not want the header splitted, so we pass NULL*/
3937                 rec = ctdb_marshall_loop_next(m, rec, &reqid,
3938                                               NULL, &key, &data);
3939
3940                 ctdb_dumpdb_record(ctdb, key, data, stdout);
3941         }
3942
3943         printf("Dumped %d records\n", i);
3944         talloc_free(tmp_ctx);
3945         return 0;
3946 }
3947
3948 /*
3949  * wipe a database from a file
3950  */
3951 static int control_wipedb(struct ctdb_context *ctdb, int argc,
3952                           const char **argv)
3953 {
3954         int ret;
3955         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
3956         TDB_DATA data;
3957         struct ctdb_db_context *ctdb_db;
3958         struct ctdb_node_map *nodemap = NULL;
3959         struct ctdb_vnn_map *vnnmap = NULL;
3960         int i;
3961         struct ctdb_control_wipe_database w;
3962         uint32_t *nodes;
3963         uint32_t generation;
3964         struct ctdb_dbid_map *dbmap = NULL;
3965
3966         if (argc != 1) {
3967                 DEBUG(DEBUG_ERR,("Invalid arguments\n"));
3968                 return -1;
3969         }
3970
3971         ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
3972                                  &dbmap);
3973         if (ret != 0) {
3974                 DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n",
3975                                   options.pnn));
3976                 return ret;
3977         }
3978
3979         for(i=0;i<dbmap->num;i++){
3980                 const char *name;
3981
3982                 ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn,
3983                                     dbmap->dbs[i].dbid, tmp_ctx, &name);
3984                 if(!strcmp(argv[0], name)){
3985                         talloc_free(discard_const(name));
3986                         break;
3987                 }
3988                 talloc_free(discard_const(name));
3989         }
3990         if (i == dbmap->num) {
3991                 DEBUG(DEBUG_ERR, ("No database with name '%s' found\n",
3992                                   argv[0]));
3993                 talloc_free(tmp_ctx);
3994                 return -1;
3995         }
3996
3997         ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
3998         if (ctdb_db == NULL) {
3999                 DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
4000                                   argv[0]));
4001                 talloc_free(tmp_ctx);
4002                 return -1;
4003         }
4004
4005         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb,
4006                                    &nodemap);
4007         if (ret != 0) {
4008                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n",
4009                                   options.pnn));
4010                 talloc_free(tmp_ctx);
4011                 return ret;
4012         }
4013
4014         ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx,
4015                                   &vnnmap);
4016         if (ret != 0) {
4017                 DEBUG(DEBUG_ERR, ("Unable to get vnnmap from node %u\n",
4018                                   options.pnn));
4019                 talloc_free(tmp_ctx);
4020                 return ret;
4021         }
4022
4023         /* freeze all nodes */
4024         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4025         for (i=1; i<=NUM_DB_PRIORITIES; i++) {
4026                 ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
4027                                                 nodes, i,
4028                                                 TIMELIMIT(),
4029                                                 false, tdb_null,
4030                                                 NULL, NULL,
4031                                                 NULL);
4032                 if (ret != 0) {
4033                         DEBUG(DEBUG_ERR, ("Unable to freeze nodes.\n"));
4034                         ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn,
4035                                              CTDB_RECOVERY_ACTIVE);
4036                         talloc_free(tmp_ctx);
4037                         return -1;
4038                 }
4039         }
4040
4041         generation = vnnmap->generation;
4042         data.dptr = (void *)&generation;
4043         data.dsize = sizeof(generation);
4044
4045         /* start a cluster wide transaction */
4046         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4047         ret = ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
4048                                         nodes, 0,
4049                                         TIMELIMIT(), false, data,
4050                                         NULL, NULL,
4051                                         NULL);
4052         if (ret!= 0) {
4053                 DEBUG(DEBUG_ERR, ("Unable to start cluster wide "
4054                                   "transactions.\n"));
4055                 return -1;
4056         }
4057
4058         w.db_id = ctdb_db->db_id;
4059         w.transaction_id = generation;
4060
4061         data.dptr = (void *)&w;
4062         data.dsize = sizeof(w);
4063
4064         /* wipe all the remote databases. */
4065         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4066         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
4067                                         nodes, 0,
4068                                         TIMELIMIT(), false, data,
4069                                         NULL, NULL,
4070                                         NULL) != 0) {
4071                 DEBUG(DEBUG_ERR, ("Unable to wipe database.\n"));
4072                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4073                 talloc_free(tmp_ctx);
4074                 return -1;
4075         }
4076
4077         data.dptr = (void *)&ctdb_db->db_id;
4078         data.dsize = sizeof(ctdb_db->db_id);
4079
4080         /* mark the database as healthy */
4081         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4082         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_DB_SET_HEALTHY,
4083                                         nodes, 0,
4084                                         TIMELIMIT(), false, data,
4085                                         NULL, NULL,
4086                                         NULL) != 0) {
4087                 DEBUG(DEBUG_ERR, ("Failed to mark database as healthy.\n"));
4088                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4089                 talloc_free(tmp_ctx);
4090                 return -1;
4091         }
4092
4093         data.dptr = (void *)&generation;
4094         data.dsize = sizeof(generation);
4095
4096         /* commit all the changes */
4097         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
4098                                         nodes, 0,
4099                                         TIMELIMIT(), false, data,
4100                                         NULL, NULL,
4101                                         NULL) != 0) {
4102                 DEBUG(DEBUG_ERR, ("Unable to commit databases.\n"));
4103                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4104                 talloc_free(tmp_ctx);
4105                 return -1;
4106         }
4107
4108         /* thaw all nodes */
4109         nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
4110         if (ctdb_client_async_control(ctdb, CTDB_CONTROL_THAW,
4111                                         nodes, 0,
4112                                         TIMELIMIT(),
4113                                         false, tdb_null,
4114                                         NULL, NULL,
4115                                         NULL) != 0) {
4116                 DEBUG(DEBUG_ERR, ("Unable to thaw nodes.\n"));
4117                 ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.pnn, CTDB_RECOVERY_ACTIVE);
4118                 talloc_free(tmp_ctx);
4119                 return -1;
4120         }
4121
4122         talloc_free(tmp_ctx);
4123         return 0;
4124 }
4125
4126 /*
4127  * set flags of a node in the nodemap
4128  */
4129 static int control_setflags(struct ctdb_context *ctdb, int argc, const char **argv)
4130 {
4131         int ret;
4132         int32_t status;
4133         int node;
4134         int flags;
4135         TDB_DATA data;
4136         struct ctdb_node_flag_change c;
4137
4138         if (argc != 2) {
4139                 usage();
4140                 return -1;
4141         }
4142
4143         if (sscanf(argv[0], "%d", &node) != 1) {
4144                 DEBUG(DEBUG_ERR, ("Badly formed node\n"));
4145                 usage();
4146                 return -1;
4147         }
4148         if (sscanf(argv[1], "0x%x", &flags) != 1) {
4149                 DEBUG(DEBUG_ERR, ("Badly formed flags\n"));
4150                 usage();
4151                 return -1;
4152         }
4153
4154         c.pnn       = node;
4155         c.old_flags = 0;
4156         c.new_flags = flags;
4157
4158         data.dsize = sizeof(c);
4159         data.dptr = (unsigned char *)&c;
4160
4161         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_MODIFY_FLAGS, 0,
4162                            data, NULL, NULL, &status, NULL, NULL);
4163         if (ret != 0 || status != 0) {
4164                 DEBUG(DEBUG_ERR,("Failed to modify flags\n"));
4165                 return -1;
4166         }
4167         return 0;
4168 }
4169
4170 /*
4171   dump memory usage
4172  */
4173 static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
4174 {
4175         TDB_DATA data;
4176         int ret;
4177         int32_t res;
4178         char *errmsg;
4179         TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
4180         ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_DUMP_MEMORY,
4181                            0, tdb_null, tmp_ctx, &data, &res, NULL, &errmsg);
4182         if (ret != 0 || res != 0) {
4183                 DEBUG(DEBUG_ERR,("Failed to dump memory - %s\n", errmsg));
4184                 talloc_free(tmp_ctx);
4185                 return -1;
4186         }
4187         write(1, data.dptr, data.dsize);
4188         talloc_free(tmp_ctx);
4189         return 0;
4190 }
4191
4192 /*
4193   handler for memory dumps
4194 */
4195 static void mem_dump_handler(struct ctdb_context *ctdb, uint64_t srvid,
4196                              TDB_DATA data, void *private_data)
4197 {
4198         write(1, data.dptr, data.dsize);
4199         exit(0);
4200 }
4201
4202 /*
4203   dump memory usage on the recovery daemon
4204  */
4205 static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
4206 {
4207         int ret;
4208         TDB_DATA data;
4209         struct rd_memdump_reply rd;
4210
4211         rd.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
4212         if (rd.pnn == -1) {
4213                 DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
4214                 return -1;
4215         }
4216         rd.srvid = getpid();
4217
4218         /* register a message port for receiveing the reply so that we
4219            can receive the reply
4220         */
4221         ctdb_client_set_message_handler(ctdb, rd.srvid, mem_dump_handler, NULL);
4222
4223
4224         data.dptr = (uint8_t *)&rd;
4225         data.dsize = sizeof(rd);
4226
4227         ret = ctdb_client_send_message(ctdb, options.pnn, CTDB_SRVID_MEM_DUMP, data);
4228         if (ret != 0) {
4229                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
4230                 return -1;
4231         }
4232
4233         /* this loop will terminate when we have received the reply */
4234         while (1) {
4235                 event_loop_once(ctdb->ev);
4236         }
4237
4238         return 0;
4239 }
4240
4241 /*
4242   send a message to a srvid
4243  */
4244 static int control_msgsend(struct ctdb_context *ctdb, int argc, const char **argv)
4245 {
4246         unsigned long srvid;
4247         int ret;
4248         TDB_DATA data;
4249
4250         if (argc < 2) {
4251                 usage();
4252         }
4253
4254         srvid      = strtoul(argv[0], NULL, 0);
4255
4256         data.dptr = (uint8_t *)discard_const(argv[1]);
4257         data.dsize= strlen(argv[1]);
4258
4259         ret = ctdb_client_send_message(ctdb, CTDB_BROADCAST_CONNECTED, srvid, data);
4260         if (ret != 0) {
4261                 DEBUG(DEBUG_ERR,("Failed to send memdump request message to %u\n", options.pnn));
4262                 return -1;
4263         }
4264
4265         return 0;
4266 }
4267
4268 /*
4269   handler for msglisten
4270 */
4271 static void msglisten_handler(struct ctdb_context *ctdb, uint64_t srvid,
4272                              TDB_DATA data, void *private_data)
4273 {
4274         int i;
4275
4276         printf("Message received: ");
4277         for (i=0;i<data.dsize;i++) {
4278                 printf("%c", data.dptr[i]);
4279         }
4280         printf("\n");
4281 }
4282
4283 /*
4284   listen for messages on a messageport
4285  */
4286 static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **argv)
4287 {
4288         uint64_t srvid;
4289
4290         srvid = getpid();
4291
4292         /* register a message port and listen for messages
4293         */
4294         ctdb_client_set_message_handler(ctdb, srvid, msglisten_handler, NULL);
4295         printf("Listening for messages on srvid:%d\n", (int)srvid);
4296
4297         while (1) {
4298                 event_loop_once(ctdb->ev);
4299         }
4300
4301         return 0;
4302 }
4303
4304 /*
4305   list all nodes in the cluster
4306   if the daemon is running, we read the data from the daemon.
4307   if the daemon is not running we parse the nodes file directly
4308  */
4309 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
4310 {
4311         int i, ret;
4312         struct ctdb_node_map *nodemap=NULL;
4313
4314         if (ctdb != NULL) {
4315                 ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
4316                 if (ret != 0) {
4317                         DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
4318                         return ret;
4319                 }
4320
4321                 for(i=0;i<nodemap->num;i++){
4322                         if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
4323                                 continue;
4324                         }
4325                         if (options.machinereadable){
4326                                 printf(":%d:%s:\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
4327                         } else {
4328                                 printf("%s\n", ctdb_addr_to_str(&nodemap->nodes[i].addr));
4329                         }
4330                 }
4331         } else {
4332                 TALLOC_CTX *mem_ctx = talloc_new(NULL);
4333                 struct pnn_node *pnn_nodes;
4334                 struct pnn_node *pnn_node;
4335
4336                 pnn_nodes = read_nodes_file(mem_ctx);
4337                 if (pnn_nodes == NULL) {
4338                         DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
4339                         talloc_free(mem_ctx);
4340                         return -1;
4341                 }
4342
4343                 for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
4344                         ctdb_sock_addr addr;
4345
4346                         if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
4347                                 DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
4348                                 talloc_free(mem_ctx);
4349                                 return -1;
4350                         }
4351
4352                         if (options.machinereadable){
4353                                 printf(":%d:%s:\n", pnn_node->pnn, pnn_node->addr);
4354                         } else {
4355                                 printf("%s\n", pnn_node->addr);
4356                         }
4357                 }
4358                 talloc_free(mem_ctx);
4359         }
4360
4361         return 0;
4362 }
4363
4364 /*
4365   reload the nodes file on the local node
4366  */
4367 static int control_reload_nodes_file(struct ctdb_context *ctdb, int argc, const char **argv)
4368 {
4369         int i, ret;
4370         int mypnn;
4371         struct ctdb_node_map *nodemap=NULL;
4372
4373         mypnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
4374         if (mypnn == -1) {
4375                 DEBUG(DEBUG_ERR, ("Failed to read pnn of local node\n"));
4376                 return -1;
4377         }
4378
4379         ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap);
4380         if (ret != 0) {
4381                 DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n"));
4382                 return ret;
4383         }
4384
4385         /* reload the nodes file on all remote nodes */
4386         for (i=0;i<nodemap->num;i++) {
4387                 if (nodemap->nodes[i].pnn == mypnn) {
4388                         continue;
4389                 }
4390                 DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", nodemap->nodes[i].pnn));
4391                 ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(),
4392                         nodemap->nodes[i].pnn);
4393                 if (ret != 0) {
4394                         DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", nodemap->nodes[i].pnn));
4395                 }
4396         }
4397
4398         /* reload the nodes file on the local node */
4399         DEBUG(DEBUG_NOTICE, ("Reloading nodes file on node %u\n", mypnn));
4400         ret = ctdb_ctrl_reload_nodes_file(ctdb, TIMELIMIT(), mypnn);
4401         if (ret != 0) {
4402                 DEBUG(DEBUG_ERR, ("ERROR: Failed to reload nodes file on node %u. You MUST fix that node manually!\n", mypnn));
4403         }
4404
4405         /* initiate a recovery */
4406         control_recover(ctdb, argc, argv);
4407
4408         return 0;
4409 }
4410
4411
4412 static const struct {
4413         const char *name;
4414         int (*fn)(struct ctdb_context *, int, const char **);
4415         bool auto_all;
4416         bool without_daemon; /* can be run without daemon running ? */
4417         const char *msg;
4418         const char *args;
4419 } ctdb_commands[] = {
4420 #ifdef CTDB_VERS
4421         { "version",         control_version,           true,   false,  "show version of ctdb" },
4422 #endif
4423         { "status",          control_status,            true,   false,  "show node status" },
4424         { "uptime",          control_uptime,            true,   false,  "show node uptime" },
4425         { "ping",            control_ping,              true,   false,  "ping all nodes" },
4426         { "getvar",          control_getvar,            true,   false,  "get a tunable variable",               "<name>"},
4427         { "setvar",          control_setvar,            true,   false,  "set a tunable variable",               "<name> <value>"},
4428         { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
4429         { "statistics",      control_statistics,        false,  false, "show statistics" },
4430         { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
4431         { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
4432         { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
4433         { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
4434         { "setifacelink",    control_setifacelink,      true,   false,  "set interface link status", "<iface> <status>" },
4435         { "process-exists",  control_process_exists,    true,   false,  "check if a process exists on a node",  "<pid>"},
4436         { "getdbmap",        control_getdbmap,          true,   false,  "show the database map" },
4437         { "getdbstatus",     control_getdbstatus,       true,   false,  "show the status of a database", "<dbname>" },
4438         { "catdb",           control_catdb,             true,   false,  "dump a database" ,                     "<dbname>"},
4439         { "getmonmode",      control_getmonmode,        true,   false,  "show monitoring mode" },
4440         { "getcapabilities", control_getcapabilities,   true,   false,  "show node capabilities" },
4441         { "pnn",             control_pnn,               true,   false,  "show the pnn of the currnet node" },
4442         { "lvs",             control_lvs,               true,   false,  "show lvs configuration" },
4443         { "lvsmaster",       control_lvsmaster,         true,   false,  "show which node is the lvs master" },
4444         { "disablemonitor",      control_disable_monmode,true,  false,  "set monitoring mode to DISABLE" },
4445         { "enablemonitor",      control_enable_monmode, true,   false,  "set monitoring mode to ACTIVE" },
4446         { "setdebug",        control_setdebug,          true,   false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
4447         { "getdebug",        control_getdebug,          true,   false,  "get debug level" },
4448         { "getlog",          control_getlog,            true,   false,  "get the log data from the in memory ringbuffer", "<level>" },
4449         { "clearlog",          control_clearlog,        true,   false,  "clear the log data from the in memory ringbuffer" },
4450         { "attach",          control_attach,            true,   false,  "attach to a database",                 "<dbname>" },
4451         { "dumpmemory",      control_dumpmemory,        true,   false,  "dump memory map to stdout" },
4452         { "rddumpmemory",    control_rddumpmemory,      true,   false,  "dump memory map from the recovery daemon to stdout" },
4453         { "getpid",          control_getpid,            true,   false,  "get ctdbd process ID" },
4454         { "disable",         control_disable,           true,   false,  "disable a nodes public IP" },
4455         { "enable",          control_enable,            true,   false,  "enable a nodes public IP" },
4456         { "stop",            control_stop,              true,   false,  "stop a node" },
4457         { "continue",        control_continue,          true,   false,  "re-start a stopped node" },
4458         { "ban",             control_ban,               true,   false,  "ban a node from the cluster",          "<bantime|0>"},
4459         { "unban",           control_unban,             true,   false,  "unban a node" },
4460         { "showban",         control_showban,           true,   false,  "show ban information"},
4461         { "shutdown",        control_shutdown,          true,   false,  "shutdown ctdbd" },
4462         { "recover",         control_recover,           true,   false,  "force recovery" },
4463         { "sync",            control_ipreallocate,      true,   false,  "wait until ctdbd has synced all state changes" },
4464         { "ipreallocate",    control_ipreallocate,      true,   false,  "force the recovery daemon to perform a ip reallocation procedure" },
4465         { "thaw",            control_thaw,              true,   false,  "thaw databases", "[priority:1-3]" },
4466         { "isnotrecmaster",  control_isnotrecmaster,    false,  false,  "check if the local node is recmaster or not" },
4467         { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "<srcip:port> <dstip:port>" },
4468         { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
4469         { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
4470         { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip>" },
4471         { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
4472
4473         { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
4474
4475         { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
4476         { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
4477         { "chksrvid",        chksrvid,                  false,  false, "check if a server id exists", "<pnn> <type> <id>" },
4478         { "getsrvids",       getsrvids,                 false,  false, "get a list of all server ids"},
4479         { "vacuum",          ctdb_vacuum,               false,  false, "vacuum the databases of empty records", "[max_records]"},
4480         { "repack",          ctdb_repack,               false,  false, "repack all databases", "[max_freelist]"},
4481         { "listnodes",       control_listnodes,         false,  true, "list all nodes in the cluster"},
4482         { "reloadnodes",     control_reload_nodes_file, false,  false, "reload the nodes file and restart the transport on all nodes"},
4483         { "moveip",          control_moveip,            false,  false, "move/failover an ip address to another node", "<ip> <node>"},
4484         { "addip",           control_addip,             true,   false, "add a ip address to a node", "<ip/mask> <iface>"},
4485         { "delip",           control_delip,             false,  false, "delete an ip address from a node", "<ip>"},
4486         { "eventscript",     control_eventscript,       true,   false, "run the eventscript with the given parameters on a node", "<arguments>"},
4487         { "backupdb",        control_backupdb,          false,  false, "backup the database into a file.", "<database> <file>"},
4488         { "restoredb",        control_restoredb,        false,  false, "restore the database from a file.", "<file> [dbname]"},
4489         { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
4490         { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname>"},
4491         { "recmaster",        control_recmaster,        false,  false, "show the pnn for the recovery master."},
4492         { "setflags",        control_setflags,          false,  false, "set flags for a node in the nodemap.", "<node> <flags>"},
4493         { "scriptstatus",    control_scriptstatus,  false,      false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
4494         { "enablescript",     control_enablescript,  false,     false, "enable an eventscript", "<script>"},
4495         { "disablescript",    control_disablescript,  false,    false, "disable an eventscript", "<script>"},
4496         { "natgwlist",        control_natgwlist,        false,  false, "show the nodes belonging to this natgw configuration"},
4497         { "xpnn",             control_xpnn,             true,   true,  "find the pnn of the local node without talking to the daemon (unreliable)" },
4498         { "getreclock",       control_getreclock,       false,  false, "Show the reclock file of a node"},
4499         { "setreclock",       control_setreclock,       false,  false, "Set/clear the reclock file of a node", "[filename]"},
4500         { "setnatgwstate",    control_setnatgwstate,    false,  false, "Set NATGW state to on/off", "{on|off}"},
4501         { "setlmasterrole",   control_setlmasterrole,   false,  false, "Set LMASTER role to on/off", "{on|off}"},
4502         { "setrecmasterrole", control_setrecmasterrole, false,  false, "Set RECMASTER role to on/off", "{on|off}"},
4503         { "setdbprio",        control_setdbprio,        false,  false, "Set DB priority", "<dbid> <prio:1-3>"},
4504         { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbid>"},
4505         { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
4506         { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
4507         { "sync",            control_ipreallocate,      true,   false,  "wait until ctdbd has synced all state changes" },
4508 };
4509
4510 /*
4511   show usage message
4512  */
4513 static void usage(void)
4514 {
4515         int i;
4516         printf(
4517 "Usage: ctdb [options] <control>\n" \
4518 "Options:\n" \
4519 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
4520 "   -Y                 generate machinereadable output\n"
4521 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
4522         printf("Controls:\n");
4523         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
4524                 printf("  %-15s %-27s  %s\n",
4525                        ctdb_commands[i].name,
4526                        ctdb_commands[i].args?ctdb_commands[i].args:"",
4527                        ctdb_commands[i].msg);
4528         }
4529         exit(1);
4530 }
4531
4532
4533 static void ctdb_alarm(int sig)
4534 {
4535         printf("Maximum runtime exceeded - exiting\n");
4536         _exit(ERR_TIMEOUT);
4537 }
4538
4539 /*
4540   main program
4541 */
4542 int main(int argc, const char *argv[])
4543 {
4544         struct ctdb_context *ctdb;
4545         char *nodestring = NULL;
4546         struct poptOption popt_options[] = {
4547                 POPT_AUTOHELP
4548                 POPT_CTDB_CMDLINE
4549                 { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
4550                 { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
4551                 { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
4552                 { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
4553                 POPT_TABLEEND
4554         };
4555         int opt;
4556         const char **extra_argv;
4557         int extra_argc = 0;
4558         int ret=-1, i;
4559         poptContext pc;
4560         struct event_context *ev;
4561         const char *control;
4562
4563         setlinebuf(stdout);
4564
4565         /* set some defaults */
4566         options.maxruntime = 0;
4567         options.timelimit = 3;
4568         options.pnn = CTDB_CURRENT_NODE;
4569
4570         pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
4571
4572         while ((opt = poptGetNextOpt(pc)) != -1) {
4573                 switch (opt) {
4574                 default:
4575                         DEBUG(DEBUG_ERR, ("Invalid option %s: %s\n",
4576                                 poptBadOption(pc, 0), poptStrerror(opt)));
4577                         exit(1);
4578                 }
4579         }
4580
4581         /* setup the remaining options for the main program to use */
4582         extra_argv = poptGetArgs(pc);
4583         if (extra_argv) {
4584                 extra_argv++;
4585                 while (extra_argv[extra_argc]) extra_argc++;
4586         }
4587
4588         if (extra_argc < 1) {
4589                 usage();
4590         }
4591
4592         if (options.maxruntime == 0) {
4593                 const char *ctdb_timeout;
4594                 ctdb_timeout = getenv("CTDB_TIMEOUT");
4595                 if (ctdb_timeout != NULL) {
4596                         options.maxruntime = strtoul(ctdb_timeout, NULL, 0);
4597                 } else {
4598                         /* default timeout is 120 seconds */
4599                         options.maxruntime = 120;
4600                 }
4601         }
4602
4603         signal(SIGALRM, ctdb_alarm);
4604         alarm(options.maxruntime);
4605
4606         /* setup the node number to contact */
4607         if (nodestring != NULL) {
4608                 if (strcmp(nodestring, "all") == 0) {
4609                         options.pnn = CTDB_BROADCAST_ALL;
4610                 } else {
4611                         options.pnn = strtoul(nodestring, NULL, 0);
4612                 }
4613         }
4614
4615         control = extra_argv[0];
4616
4617         ev = event_context_init(NULL);
4618         if (!ev) {
4619                 DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
4620                 exit(1);
4621         }
4622         tevent_loop_allow_nesting(ev);
4623
4624         for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
4625                 if (strcmp(control, ctdb_commands[i].name) == 0) {
4626                         int j;
4627
4628                         if (ctdb_commands[i].without_daemon == true) {
4629                                 close(2);
4630                         }
4631
4632                         /* initialise ctdb */
4633                         ctdb = ctdb_cmdline_client(ev);
4634
4635                         if (ctdb_commands[i].without_daemon == false) {
4636                                 const char *socket_name;
4637
4638                                 if (ctdb == NULL) {
4639                                         DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
4640                                         exit(1);
4641                                 }
4642
4643                                 /* initialize a libctdb connection as well */
4644                                 socket_name = ctdb_get_socketname(ctdb);
4645                                 ctdb_connection = ctdb_connect(socket_name,
4646                                                        ctdb_log_file, stderr);
4647                                 if (ctdb_connection == NULL) {
4648                                         fprintf(stderr, "Failed to connect to daemon from libctdb\n");
4649                                         exit(1);
4650                                 }
4651
4652                                 /* verify the node exists */
4653                                 verify_node(ctdb);
4654
4655                                 if (options.pnn == CTDB_CURRENT_NODE) {
4656                                         int pnn;
4657                                         pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), options.pnn);
4658                                         if (pnn == -1) {
4659                                                 return -1;
4660                                         }
4661                                         options.pnn = pnn;
4662                                 }
4663                         }
4664
4665                         if (ctdb_commands[i].auto_all &&
4666                             options.pnn == CTDB_BROADCAST_ALL) {
4667                                 uint32_t *nodes;
4668                                 uint32_t num_nodes;
4669                                 ret = 0;
4670
4671                                 nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
4672                                 CTDB_NO_MEMORY(ctdb, nodes);
4673
4674                                 for (j=0;j<num_nodes;j++) {
4675                                         options.pnn = nodes[j];
4676                                         ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
4677                                 }
4678                                 talloc_free(nodes);
4679                         } else {
4680                                 ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
4681                         }
4682                         break;
4683                 }
4684         }
4685
4686         if (i == ARRAY_SIZE(ctdb_commands)) {
4687                 DEBUG(DEBUG_ERR, ("Unknown control '%s'\n", control));
4688                 exit(1);
4689         }
4690
4691         return ret;
4692 }