add new command line functions
[sahlberg/ctdb.git] / tools / ctdb.c
index 76d1febaf2840c5fda5b48819ed1508798263f77..d54ff10a07723ecabafc7f30d07d025077d83454 100644 (file)
@@ -19,7 +19,7 @@
 */
 
 #include "includes.h"
-#include "lib/events/events.h"
+#include "lib/tevent/tevent.h"
 #include "system/time.h"
 #include "system/filesys.h"
 #include "system/network.h"
@@ -44,6 +44,7 @@ static struct {
        int timelimit;
        uint32_t pnn;
        int machinereadable;
+       int verbose;
        int maxruntime;
 } options;
 
@@ -156,7 +157,7 @@ static int control_process_exists(struct ctdb_context *ctdb, int argc, const cha
 /*
   display statistics structure
  */
-static void show_statistics(struct ctdb_statistics *s)
+static void show_statistics(struct ctdb_statistics *s, int show_header)
 {
        TALLOC_CTX *tmp_ctx = talloc_new(NULL);
        int i;
@@ -210,33 +211,100 @@ static void show_statistics(struct ctdb_statistics *s)
        tmp    /= 24;
        days    = tmp;
 
-       printf("CTDB version %u\n", CTDB_VERSION);
-       printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
-       printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
-
-       for (i=0;i<ARRAY_SIZE(fields);i++) {
-               if (strchr(fields[i].name, '.')) {
-                       preflen = strcspn(fields[i].name, ".")+1;
-                       if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
-                               prefix = fields[i].name;
-                               printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
+       if (options.machinereadable){
+               if (show_header) {
+                       printf("CTDB version:");
+                       printf("Current time of statistics:");
+                       printf("Statistics collected since:");
+                       for (i=0;i<ARRAY_SIZE(fields);i++) {
+                               printf("%s:", fields[i].name);
                        }
-               } else {
-                       preflen = 0;
+                       printf("num_reclock_ctdbd_latency:");
+                       printf("min_reclock_ctdbd_latency:");
+                       printf("avg_reclock_ctdbd_latency:");
+                       printf("max_reclock_ctdbd_latency:");
+
+                       printf("num_reclock_recd_latency:");
+                       printf("min_reclock_recd_latency:");
+                       printf("avg_reclock_recd_latency:");
+                       printf("max_reclock_recd_latency:");
+
+                       printf("num_call_latency:");
+                       printf("min_call_latency:");
+                       printf("avg_call_latency:");
+                       printf("max_call_latency:");
+
+                       printf("num_lockwait_latency:");
+                       printf("min_lockwait_latency:");
+                       printf("avg_lockwait_latency:");
+                       printf("max_lockwait_latency:");
+
+                       printf("num_childwrite_latency:");
+                       printf("min_childwrite_latency:");
+                       printf("avg_childwrite_latency:");
+                       printf("max_childwrite_latency:");
+                       printf("\n");
                }
-               printf(" %*s%-22s%*s%10u\n", 
-                      preflen?4:0, "",
-                      fields[i].name+preflen, 
-                      preflen?0:4, "",
-                      *(uint32_t *)(fields[i].offset+(uint8_t *)s));
-       }
-       printf(" %-30s     %.6f sec\n", "max_reclock_ctdbd", s->reclock.ctdbd);
-       printf(" %-30s     %.6f sec\n", "max_reclock_recd", s->reclock.recd);
+               printf("%d:", CTDB_VERSION);
+               printf("%d:", (int)s->statistics_current_time.tv_sec);
+               printf("%d:", (int)s->statistics_start_time.tv_sec);
+               for (i=0;i<ARRAY_SIZE(fields);i++) {
+                       printf("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s));
+               }
+               printf("%d:", s->reclock.ctdbd.num);
+               printf("%.6f:", s->reclock.ctdbd.min);
+               printf("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0);
+               printf("%.6f:", s->reclock.ctdbd.max);
+
+               printf("%d:", s->reclock.recd.num);
+               printf("%.6f:", s->reclock.recd.min);
+               printf("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0);
+               printf("%.6f:", s->reclock.recd.max);
+
+               printf("%d:", s->call_latency.num);
+               printf("%.6f:", s->call_latency.min);
+               printf("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
+               printf("%.6f:", s->call_latency.max);
+
+               printf("%d:", s->lockwait_latency.num);
+               printf("%.6f:", s->lockwait_latency.min);
+               printf("%.6f:", s->lockwait_latency.num?s->lockwait_latency.total/s->lockwait_latency.num:0.0);
+               printf("%.6f:", s->lockwait_latency.max);
+
+               printf("%d:", s->childwrite_latency.num);
+               printf("%.6f:", s->childwrite_latency.min);
+               printf("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
+               printf("%.6f:", s->childwrite_latency.max);
+               printf("\n");
+       } else {
+               printf("CTDB version %u\n", CTDB_VERSION);
+               printf("Current time of statistics  :                %s", ctime(&s->statistics_current_time.tv_sec));
+               printf("Statistics collected since  : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&s->statistics_start_time.tv_sec));
+
+               for (i=0;i<ARRAY_SIZE(fields);i++) {
+                       if (strchr(fields[i].name, '.')) {
+                               preflen = strcspn(fields[i].name, ".")+1;
+                               if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
+                                       prefix = fields[i].name;
+                                       printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
+                               }
+                       } else {
+                               preflen = 0;
+                       }
+                       printf(" %*s%-22s%*s%10u\n", 
+                              preflen?4:0, "",
+                              fields[i].name+preflen, 
+                              preflen?0:4, "",
+                              *(uint32_t *)(fields[i].offset+(uint8_t *)s));
+               }
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_ctdbd       MIN/AVG/MAX", s->reclock.ctdbd.min, s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0, s->reclock.ctdbd.max, s->reclock.ctdbd.num);
+
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 
-       printf(" %-30s     %.6f sec\n", "max_call_latency", s->max_call_latency);
-       printf(" %-30s     %.6f sec\n", "max_lockwait_latency", s->max_lockwait_latency);
-       printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
-       printf(" %-30s     %.6f sec\n", "max_childwrite_latency", s->max_childwrite_latency);
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "lockwait_latency   MIN/AVG/MAX", s->lockwait_latency.min, s->lockwait_latency.num?s->lockwait_latency.total/s->lockwait_latency.num:0.0, s->lockwait_latency.max, s->lockwait_latency.num);
+               printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
+       }
 
        talloc_free(tmp_ctx);
 }
@@ -273,14 +341,14 @@ static int control_statistics_all(struct ctdb_context *ctdb)
                }
                statistics.max_hop_count = 
                        MAX(statistics.max_hop_count, s1.max_hop_count);
-               statistics.max_call_latency = 
-                       MAX(statistics.max_call_latency, s1.max_call_latency);
-               statistics.max_lockwait_latency = 
-                       MAX(statistics.max_lockwait_latency, s1.max_lockwait_latency);
+               statistics.call_latency.max = 
+                       MAX(statistics.call_latency.max, s1.call_latency.max);
+               statistics.lockwait_latency.max = 
+                       MAX(statistics.lockwait_latency.max, s1.lockwait_latency.max);
        }
        talloc_free(nodes);
        printf("Gathered statistics for %u nodes\n", num_nodes);
-       show_statistics(&statistics);
+       show_statistics(&statistics, 1);
        return 0;
 }
 
@@ -301,7 +369,7 @@ static int control_statistics(struct ctdb_context *ctdb, int argc, const char **
                DEBUG(DEBUG_ERR, ("Unable to get statistics from node %u\n", options.pnn));
                return ret;
        }
-       show_statistics(&statistics);
+       show_statistics(&statistics, 1);
        return 0;
 }
 
@@ -322,6 +390,37 @@ static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const c
 }
 
 
+/*
+  display remote ctdb rolling statistics
+ */
+static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       int ret;
+       struct ctdb_statistics_wire *stats;
+       int i, num_records = -1;
+
+       if (argc ==1) {
+               num_records = atoi(argv[0]) - 1;
+       }
+
+       ret = ctdb_ctrl_getstathistory(ctdb, TIMELIMIT(), options.pnn, ctdb, &stats);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR, ("Unable to get rolling statistics from node %u\n", options.pnn));
+               return ret;
+       }
+       for (i=0;i<stats->num;i++) {
+               if (stats->stats[i].statistics_start_time.tv_sec == 0) {
+                       continue;
+               }
+               show_statistics(&stats->stats[i], i==0);
+               if (i == num_records) {
+                       break;
+               }
+       }
+       return 0;
+}
+
+
 /*
   display uptime of remote node
  */
@@ -523,40 +622,19 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
        }
 
        if(options.machinereadable){
-               printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:\n");
+               printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:\n");
                for(i=0;i<nodemap->num;i++){
-                       int partially_online = 0;
-                       int j;
-
                        if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
                                continue;
                        }
-                       if (nodemap->nodes[i].flags == 0) {
-                               struct ctdb_control_get_ifaces *ifaces;
-
-                               ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
-                                                          nodemap->nodes[i].pnn,
-                                                          ctdb, &ifaces);
-                               if (ret == 0) {
-                                       for (j=0; j < ifaces->num; j++) {
-                                               if (ifaces->ifaces[j].link_state != 0) {
-                                                       continue;
-                                               }
-                                               partially_online = 1;
-                                               break;
-                                       }
-                                       talloc_free(ifaces);
-                               }
-                       }
-                       printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
+                       printf(":%d:%s:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
                                ctdb_addr_to_str(&nodemap->nodes[i].addr),
                               !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
                               !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
                               !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED),
                               !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
                               !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED),
-                              !!(nodemap->nodes[i].flags&NODE_FLAGS_INACTIVE),
-                              partially_online);
+                              !!(nodemap->nodes[i].flags&NODE_FLAGS_INACTIVE));
                }
                return 0;
        }
@@ -581,23 +659,6 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
                if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
                        continue;
                }
-               if (nodemap->nodes[i].flags == 0) {
-                       struct ctdb_control_get_ifaces *ifaces;
-
-                       ret = ctdb_ctrl_get_ifaces(ctdb, TIMELIMIT(),
-                                                  nodemap->nodes[i].pnn,
-                                                  ctdb, &ifaces);
-                       if (ret == 0) {
-                               for (j=0; j < ifaces->num; j++) {
-                                       if (ifaces->ifaces[j].link_state != 0) {
-                                               continue;
-                                       }
-                                       flags_str = talloc_strdup(ctdb, "PARTIALLYONLINE");
-                                       break;
-                               }
-                               talloc_free(ifaces);
-                       }
-               }
                for (j=0;j<ARRAY_SIZE(flag_names);j++) {
                        if (nodemap->nodes[i].flags & flag_names[j].flag) {
                                if (flags_str == NULL) {
@@ -664,6 +725,7 @@ struct natgw_node {
 static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **argv)
 {
        int i, ret;
+       uint32_t capabilities;
        const char *natgw_list;
        int nlines;
        char **lines;
@@ -742,6 +804,14 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
         */
        for(i=0;i<nodemap->num;i++){
                if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED|NODE_FLAGS_BANNED|NODE_FLAGS_UNHEALTHY))) {
+                       ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
+                       if (ret != 0) {
+                               DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
+                               return ret;
+                       }
+                       if (!(capabilities&CTDB_CAP_NATGW)) {
+                               continue;
+                       }
                        printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
                        break;
                }
@@ -750,6 +820,14 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
        if (i == nodemap->num) {
                for(i=0;i<nodemap->num;i++){
                        if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_STOPPED|NODE_FLAGS_DELETED))) {
+                               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
+                               if (ret != 0) {
+                                       DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
+                                       return ret;
+                               }
+                               if (!(capabilities&CTDB_CAP_NATGW)) {
+                                       continue;
+                               }
                                printf("%d %s\n", nodemap->nodes[i].pnn,ctdb_addr_to_str(&nodemap->nodes[i].addr));
                                break;
                        }
@@ -759,6 +837,14 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a
        if (i == nodemap->num) {
                for(i=0;i<nodemap->num;i++){
                        if (!(nodemap->nodes[i].flags & (NODE_FLAGS_DISCONNECTED|NODE_FLAGS_DELETED))) {
+                               ret = ctdb_ctrl_getcapabilities(ctdb, TIMELIMIT(), nodemap->nodes[i].pnn, &capabilities);
+                               if (ret != 0) {
+                                       DEBUG(DEBUG_ERR, ("Unable to get capabilities from node %u\n", nodemap->nodes[i].pnn));
+                                       return ret;
+                               }
+                               if (!(capabilities&CTDB_CAP_NATGW)) {
+                                       continue;
+                               }
                                printf("%d %s\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
                                break;
                        }
@@ -980,6 +1066,80 @@ static int control_recmaster(struct ctdb_context *ctdb, int argc, const char **a
        return 0;
 }
 
+/*
+  add a tickle to a public address
+ */
+static int control_add_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       struct ctdb_tcp_connection t;
+       TDB_DATA data;
+       int ret;
+
+       if (argc < 2) {
+               usage();
+       }
+
+       if (parse_ip_port(argv[0], &t.src_addr) == 0) {
+               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
+               return -1;
+       }
+       if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
+               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
+               return -1;
+       }
+
+       data.dptr = (uint8_t *)&t;
+       data.dsize = sizeof(t);
+
+       /* tell all nodes about this tcp connection */
+       ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE,
+                          0, data, ctdb, NULL, NULL, NULL, NULL);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to add tickle\n"));
+               return -1;
+       }
+       
+       return 0;
+}
+
+
+/*
+  delete a tickle from a node
+ */
+static int control_del_tickle(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       struct ctdb_tcp_connection t;
+       TDB_DATA data;
+       int ret;
+
+       if (argc < 2) {
+               usage();
+       }
+
+       if (parse_ip_port(argv[0], &t.src_addr) == 0) {
+               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
+               return -1;
+       }
+       if (parse_ip_port(argv[1], &t.dst_addr) == 0) {
+               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[1]));
+               return -1;
+       }
+
+       data.dptr = (uint8_t *)&t;
+       data.dsize = sizeof(t);
+
+       /* tell all nodes about this tcp connection */
+       ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_TCP_REMOVE,
+                          0, data, ctdb, NULL, NULL, NULL, NULL);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to remove tickle\n"));
+               return -1;
+       }
+       
+       return 0;
+}
+
+
 /*
   get a list of all tickles for this pnn
  */
@@ -988,11 +1148,16 @@ static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char *
        struct ctdb_control_tcp_tickle_list *list;
        ctdb_sock_addr addr;
        int i, ret;
+       unsigned port = 0;
 
        if (argc < 1) {
                usage();
        }
 
+       if (argc == 2) {
+               port = atoi(argv[1]);
+       }
+
        if (parse_ip(argv[0], NULL, 0, &addr) == 0) {
                DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s'\n", argv[0]));
                return -1;
@@ -1004,11 +1169,25 @@ static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char *
                return -1;
        }
 
-       printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
-       printf("Num tickles:%u\n", list->tickles.num);
-       for (i=0;i<list->tickles.num;i++) {
-               printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
-               printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
+       if (options.machinereadable){
+               printf(":source ip:port:destination ip:port:\n");
+               for (i=0;i<list->tickles.num;i++) {
+                       if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
+                               continue;
+                       }
+                       printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
+                       printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
+               }
+       } else {
+               printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr));
+               printf("Num tickles:%u\n", list->tickles.num);
+               for (i=0;i<list->tickles.num;i++) {
+                       if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) {
+                               continue;
+                       }
+                       printf("SRC: %s:%u   ", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port));
+                       printf("DST: %s:%u\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port));
+               }
        }
 
        talloc_free(list);
@@ -1727,7 +1906,11 @@ static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
        }
 
        if (options.machinereadable){
-               printf(":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:\n");
+               printf(":Public IP:Node:");
+               if (options.verbose){
+                       printf("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:");
+               }
+               printf("\n");
        } else {
                if (options.pnn == CTDB_BROADCAST_ALL) {
                        printf("Public IPs on ALL nodes\n");
@@ -1787,19 +1970,29 @@ static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
                }
 
                if (options.machinereadable){
-                       printf(":%s:%d:%s:%s:%s:\n",
-                              ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
-                              ips->ips[ips->num-i].pnn,
-                              aciface?aciface:"",
-                              avifaces?avifaces:"",
-                              cifaces?cifaces:"");
+                       printf(":%s:%d:",
+                               ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
+                               ips->ips[ips->num-i].pnn);
+                       if (options.verbose){
+                               printf("%s:%s:%s:",
+                                       aciface?aciface:"",
+                                       avifaces?avifaces:"",
+                                       cifaces?cifaces:"");
+                       }
+                       printf("\n");
                } else {
-                       printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
-                              ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
-                              ips->ips[ips->num-i].pnn,
-                              aciface?aciface:"",
-                              avifaces?avifaces:"",
-                              cifaces?cifaces:"");
+                       if (options.verbose) {
+                               printf("%s node[%d] active[%s] available[%s] configured[%s]\n",
+                                       ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
+                                       ips->ips[ips->num-i].pnn,
+                                       aciface?aciface:"",
+                                       avifaces?avifaces:"",
+                                       cifaces?cifaces:"");
+                       } else {
+                               printf("%s %d\n",
+                                       ctdb_addr_to_str(&ips->ips[ips->num-i].addr),
+                                       ips->ips[ips->num-i].pnn);
+                       }
                }
                talloc_free(info);
        }
@@ -2100,7 +2293,6 @@ again:
                return 0;
        }
 
-       DEBUG(DEBUG_ERR,("Timed out waiting for recmaster ipreallocate. Trying again\n"));
        retries++;
        sleep(1);
        goto again;
@@ -2747,6 +2939,332 @@ static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
        return 0;
 }
 
+/*
+  display the content of a database key
+ */
+static int control_readkey(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       const char *db_name;
+       struct ctdb_db_context *ctdb_db;
+       struct ctdb_record_handle *h;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+       TDB_DATA key, data;
+
+       if (argc < 2) {
+               usage();
+       }
+
+       db_name = argv[0];
+
+
+       if (db_exists(ctdb, db_name)) {
+               DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
+               return -1;
+       }
+
+       ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
+
+       if (ctdb_db == NULL) {
+               DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
+               return -1;
+       }
+
+       key.dptr  = discard_const(argv[1]);
+       key.dsize = strlen((char *)key.dptr);
+       h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+       if (h == NULL) {
+               printf("Failed to fetch record '%s' on node %d\n", 
+                       (const char *)key.dptr, ctdb_get_pnn(ctdb));
+               talloc_free(tmp_ctx);
+               exit(10);
+       }
+
+       printf("Data: size:%d ptr:[%s]\n", (int)data.dsize, data.dptr);
+
+       talloc_free(ctdb_db);
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
+/*
+  display the content of a database key
+ */
+static int control_writekey(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       const char *db_name;
+       struct ctdb_db_context *ctdb_db;
+       struct ctdb_record_handle *h;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+       TDB_DATA key, data;
+
+       if (argc < 3) {
+               usage();
+       }
+
+       db_name = argv[0];
+
+
+       if (db_exists(ctdb, db_name)) {
+               DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
+               return -1;
+       }
+
+       ctdb_db = ctdb_attach(ctdb, db_name, false, 0);
+
+       if (ctdb_db == NULL) {
+               DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
+               return -1;
+       }
+
+       key.dptr  = discard_const(argv[1]);
+       key.dsize = strlen((char *)key.dptr);
+       h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+       if (h == NULL) {
+               printf("Failed to fetch record '%s' on node %d\n", 
+                       (const char *)key.dptr, ctdb_get_pnn(ctdb));
+               talloc_free(tmp_ctx);
+               exit(10);
+       }
+
+       data.dptr  = discard_const(argv[2]);
+       data.dsize = strlen((char *)data.dptr);
+       if (ctdb_record_store(h, data) != 0) {
+               printf("Failed to store record\n");
+       }
+
+       talloc_free(h);
+       talloc_free(ctdb_db);
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
+/*
+  fetch a record from a persistent database
+ */
+static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       const char *db_name;
+       struct ctdb_db_context *ctdb_db;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+       struct ctdb_transaction_handle *h;
+       TDB_DATA key, data;
+       int fd, ret;
+
+       if (argc < 2) {
+               talloc_free(tmp_ctx);
+               usage();
+       }
+
+       db_name = argv[0];
+
+
+       if (db_exists(ctdb, db_name)) {
+               DEBUG(DEBUG_ERR,("Database '%s' does not exist\n", db_name));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       ctdb_db = ctdb_attach(ctdb, db_name, true, 0);
+
+       if (ctdb_db == NULL) {
+               DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       h = ctdb_transaction_start(ctdb_db, tmp_ctx);
+       if (h == NULL) {
+               DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       key.dptr  = discard_const(argv[1]);
+       key.dsize = strlen(argv[1]);
+       ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to fetch record\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       if (data.dsize == 0 || data.dptr == NULL) {
+               DEBUG(DEBUG_ERR,("Record is empty\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       if (argc == 3) {
+         fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
+               if (fd == -1) {
+                       DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
+                       talloc_free(tmp_ctx);
+                       return -1;
+               }
+               write(fd, data.dptr, data.dsize);
+               close(fd);
+       } else {
+               write(1, data.dptr, data.dsize);
+       }
+
+       /* abort the transaction */
+       talloc_free(h);
+
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
+
+/*
+  fetch a record from a tdb-file
+ */
+static int control_tfetch(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       const char *tdb_file;
+       TDB_CONTEXT *tdb;
+       TDB_DATA key, data;
+       int fd;
+
+       if (argc < 2) {
+               usage();
+       }
+
+       tdb_file = argv[0];
+
+       tdb = tdb_open(tdb_file, 0, 0, O_RDONLY, 0);
+       if (tdb == NULL) {
+               DEBUG(DEBUG_ERR,("Failed to open TDB file %s\n", tdb_file));
+               return -1;
+       }
+
+       key.dptr  = discard_const(argv[1]);
+       key.dsize = strlen(argv[1]);
+       data = tdb_fetch(tdb, key);
+       if (data.dptr == NULL || data.dsize < sizeof(struct ctdb_ltdb_header)) {
+               DEBUG(DEBUG_ERR,("Failed to read record %s from tdb %s\n", argv[1], tdb_file));
+               tdb_close(tdb);
+               return -1;
+       }
+
+       tdb_close(tdb);
+
+       if (argc == 3) {
+         fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0600);
+               if (fd == -1) {
+                       DEBUG(DEBUG_ERR,("Failed to open output file %s\n", argv[2]));
+                       return -1;
+               }
+               write(fd, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
+               close(fd);
+       } else {
+               write(1, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
+       }
+
+       return 0;
+}
+
+/*
+  write a record to a persistent database
+ */
+static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+       const char *db_name;
+       struct ctdb_db_context *ctdb_db;
+       TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+       struct ctdb_transaction_handle *h;
+       struct stat st;
+       TDB_DATA key, data;
+       int fd, ret;
+
+       if (argc < 3) {
+               talloc_free(tmp_ctx);
+               usage();
+       }
+
+       fd = open(argv[2], O_RDONLY);
+       if (fd == -1) {
+               DEBUG(DEBUG_ERR,("Failed to open file containing record data : %s  %s\n", argv[2], strerror(errno)));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+       
+       ret = fstat(fd, &st);
+       if (ret == -1) {
+               DEBUG(DEBUG_ERR,("fstat of file %s failed: %s\n", argv[2], strerror(errno)));
+               close(fd);
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       if (!S_ISREG(st.st_mode)) {
+               DEBUG(DEBUG_ERR,("Not a regular file %s\n", argv[2]));
+               close(fd);
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       data.dsize = st.st_size;
+       if (data.dsize == 0) {
+               data.dptr  = NULL;
+       } else {
+               data.dptr = talloc_size(tmp_ctx, data.dsize);
+               if (data.dptr == NULL) {
+                       DEBUG(DEBUG_ERR,("Failed to talloc %d of memory to store record data\n", (int)data.dsize));
+                       close(fd);
+                       talloc_free(tmp_ctx);
+                       return -1;
+               }
+               ret = read(fd, data.dptr, data.dsize);
+               if (ret != data.dsize) {
+                       DEBUG(DEBUG_ERR,("Failed to read %d bytes of record data\n", (int)data.dsize));
+                       close(fd);
+                       talloc_free(tmp_ctx);
+                       return -1;
+               }
+       }
+       close(fd);
+
+
+       db_name = argv[0];
+
+       ctdb_db = ctdb_attach(ctdb, db_name, true, 0);
+
+       if (ctdb_db == NULL) {
+               DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       h = ctdb_transaction_start(ctdb_db, tmp_ctx);
+       if (h == NULL) {
+               DEBUG(DEBUG_ERR,("Failed to start transaction on database %s\n", db_name));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       key.dptr  = discard_const(argv[1]);
+       key.dsize = strlen(argv[1]);
+       ret = ctdb_transaction_store(h, key, data);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to store record\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+       ret = ctdb_transaction_commit(h);
+       if (ret != 0) {
+               DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
+               talloc_free(tmp_ctx);
+               return -1;
+       }
+
+
+       talloc_free(tmp_ctx);
+       return 0;
+}
 
 static void log_handler(struct ctdb_context *ctdb, uint64_t srvid, 
                             TDB_DATA data, void *private_data)
@@ -3638,9 +4156,9 @@ static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **a
                return -1;
        }
 
-       dbname = dbhdr.name;
+       dbname = discard_const(dbhdr.name);
        if (argc == 2) {
-               dbname = argv[1];
+               dbname = discard_const(argv[1]);
        }
 
        outdata.dsize = dbhdr.size;
@@ -4041,50 +4559,6 @@ static int control_wipedb(struct ctdb_context *ctdb, int argc,
        return 0;
 }
 
-/*
- * set flags of a node in the nodemap
- */
-static int control_setflags(struct ctdb_context *ctdb, int argc, const char **argv)
-{
-       int ret;
-       int32_t status;
-       int node;
-       int flags;
-       TDB_DATA data;
-       struct ctdb_node_flag_change c;
-
-       if (argc != 2) {
-               usage();
-               return -1;
-       }
-
-       if (sscanf(argv[0], "%d", &node) != 1) {
-               DEBUG(DEBUG_ERR, ("Badly formed node\n"));
-               usage();
-               return -1;
-       }
-       if (sscanf(argv[1], "0x%x", &flags) != 1) {
-               DEBUG(DEBUG_ERR, ("Badly formed flags\n"));
-               usage();
-               return -1;
-       }
-
-       c.pnn       = node;
-       c.old_flags = 0;
-       c.new_flags = flags;
-
-       data.dsize = sizeof(c);
-       data.dptr = (unsigned char *)&c;
-
-       ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_MODIFY_FLAGS, 0, 
-                          data, NULL, NULL, &status, NULL, NULL);
-       if (ret != 0 || status != 0) {
-               DEBUG(DEBUG_ERR,("Failed to modify flags\n"));
-               return -1;
-       }
-       return 0;
-}
-
 /*
   dump memory usage
  */
@@ -4221,60 +4695,35 @@ static int control_msglisten(struct ctdb_context *ctdb, int argc, const char **a
 
 /*
   list all nodes in the cluster
-  if the daemon is running, we read the data from the daemon.
-  if the daemon is not running we parse the nodes file directly
+  we parse the nodes file directly
  */
 static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **argv)
 {
-       int i, ret;
-       struct ctdb_node_map *nodemap=NULL;
+       TALLOC_CTX *mem_ctx = talloc_new(NULL);
+       struct pnn_node *pnn_nodes;
+       struct pnn_node *pnn_node;
 
-       if (ctdb != NULL) {
-               ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.pnn, ctdb, &nodemap);
-               if (ret != 0) {
-                       DEBUG(DEBUG_ERR, ("Unable to get nodemap from node %u\n", options.pnn));
-                       return ret;
-               }
+       pnn_nodes = read_nodes_file(mem_ctx);
+       if (pnn_nodes == NULL) {
+               DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
+               talloc_free(mem_ctx);
+               return -1;
+       }
 
-               for(i=0;i<nodemap->num;i++){
-                       if (nodemap->nodes[i].flags & NODE_FLAGS_DELETED) {
-                               continue;
-                       }
-                       if (options.machinereadable){
-                               printf(":%d:%s:\n", nodemap->nodes[i].pnn, ctdb_addr_to_str(&nodemap->nodes[i].addr));
-                       } else {
-                               printf("%s\n", ctdb_addr_to_str(&nodemap->nodes[i].addr));
-                       }
-               }
-       } else {
-               TALLOC_CTX *mem_ctx = talloc_new(NULL);
-               struct pnn_node *pnn_nodes;
-               struct pnn_node *pnn_node;
-       
-               pnn_nodes = read_nodes_file(mem_ctx);
-               if (pnn_nodes == NULL) {
-                       DEBUG(DEBUG_ERR,("Failed to read nodes file\n"));
+       for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
+               ctdb_sock_addr addr;
+               if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
+                       DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
                        talloc_free(mem_ctx);
                        return -1;
                }
-
-               for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) {
-                       ctdb_sock_addr addr;
-
-                       if (parse_ip(pnn_node->addr, NULL, 63999, &addr) == 0) {
-                               DEBUG(DEBUG_ERR,("Wrongly formed ip address '%s' in nodes file\n", pnn_node->addr));
-                               talloc_free(mem_ctx);
-                               return -1;
-                       }
-
-                       if (options.machinereadable){
-                               printf(":%d:%s:\n", pnn_node->pnn, pnn_node->addr);
-                       } else {
-                               printf("%s\n", pnn_node->addr);
-                       }
+               if (options.machinereadable){
+                       printf(":%d:%s:\n", pnn_node->pnn, pnn_node->addr);
+               } else {
+                       printf("%s\n", pnn_node->addr);
                }
-               talloc_free(mem_ctx);
        }
+       talloc_free(mem_ctx);
 
        return 0;
 }
@@ -4346,6 +4795,7 @@ static const struct {
        { "listvars",        control_listvars,          true,   false,  "list tunable variables"},
        { "statistics",      control_statistics,        false,  false, "show statistics" },
        { "statisticsreset", control_statistics_reset,  true,   false,  "reset statistics"},
+       { "stats",           control_stats,             false,  false,  "show rolling statistics", "[number of history records]" },
        { "ip",              control_ip,                false,  false,  "show which public ip's that ctdb manages" },
        { "ipinfo",          control_ipinfo,            true,   false,  "show details about a public ip that ctdb manages", "<ip>" },
        { "ifaces",          control_ifaces,            true,   false,  "show which interfaces that ctdb manages" },
@@ -4385,7 +4835,10 @@ static const struct {
        { "killtcp",         kill_tcp,                  false,  false, "kill a tcp connection.", "<srcip:port> <dstip:port>" },
        { "gratiousarp",     control_gratious_arp,      false,  false, "send a gratious arp", "<ip> <interface>" },
        { "tickle",          tickle_tcp,                false,  false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
-       { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip>" },
+       { "gettickles",      control_get_tickles,       false,  false, "get the list of tickles registered for this ip", "<ip> [<port>]" },
+       { "addtickle",       control_add_tickle,        false,  false, "add a tickle for this ip", "<ip>:<port> <ip>:<port>" },
+
+       { "deltickle",       control_del_tickle,        false,  false, "delete a tickle from this ip", "<ip>:<port> <ip>:<port>" },
 
        { "regsrvid",        regsrvid,                  false,  false, "register a server id", "<pnn> <type> <id>" },
        { "unregsrvid",      unregsrvid,                false,  false, "unregister a server id", "<pnn> <type> <id>" },
@@ -4404,7 +4857,6 @@ static const struct {
        { "dumpdbbackup",    control_dumpdbbackup,      false,  true,  "dump database backup from a file.", "<file>"},
        { "wipedb",           control_wipedb,        false,     false, "wipe the contents of a database.", "<dbname>"},
        { "recmaster",        control_recmaster,        false,  false, "show the pnn for the recovery master."},
-       { "setflags",        control_setflags,          false,  false, "set flags for a node in the nodemap.", "<node> <flags>"},
        { "scriptstatus",    control_scriptstatus,  false,      false, "show the status of the monitoring scripts (or all scripts)", "[all]"},
        { "enablescript",     control_enablescript,  false,     false, "enable an eventscript", "<script>"},
        { "disablescript",    control_disablescript,  false,    false, "disable an eventscript", "<script>"},
@@ -4419,7 +4871,12 @@ static const struct {
        { "getdbprio",        control_getdbprio,        false,  false, "Get DB priority", "<dbid>"},
        { "msglisten",        control_msglisten,        false,  false, "Listen on a srvid port for messages", "<msg srvid>"},
        { "msgsend",          control_msgsend,  false,  false, "Send a message to srvid", "<srvid> <message>"},
-       { "sync",            control_ipreallocate,      true,   false,  "wait until ctdbd has synced all state changes" },
+       { "sync",            control_ipreallocate,      false,  false,  "wait until ctdbd has synced all state changes" },
+       { "pfetch",          control_pfetch,            false,  false,  "fetch a record from a persistent database", "<db> <key> [<file>]" },
+       { "pstore",          control_pstore,            false,  false,  "write a record to a persistent database", "<db> <key> <file containing record>" },
+       { "tfetch",          control_tfetch,            false,  true,  "fetch a record from a [c]tdb-file", "<tdb-file> <key> [<file>]" },
+       { "readkey",         control_readkey,           true,   false,  "read the content off a database key", "<tdb-file> <key>" },
+       { "writekey",        control_writekey,          true,   false,  "write to a database key", "<tdb-file> <key> <value>" },
 };
 
 /*
@@ -4433,6 +4890,7 @@ static void usage(void)
 "Options:\n" \
 "   -n <node>          choose node number, or 'all' (defaults to local node)\n"
 "   -Y                 generate machinereadable output\n"
+"   -v                 generate verbose output\n"
 "   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
        printf("Controls:\n");
        for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
@@ -4464,6 +4922,7 @@ int main(int argc, const char *argv[])
                { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
                { "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
                { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
+               { "verbose",    'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL },
                { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" },
                POPT_TABLEEND
        };
@@ -4534,6 +4993,7 @@ int main(int argc, const char *argv[])
                DEBUG(DEBUG_ERR, ("Failed to initialize event system\n"));
                exit(1);
        }
+       tevent_loop_allow_nesting(ev);
 
        for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
                if (strcmp(control, ctdb_commands[i].name) == 0) {
@@ -4543,12 +5003,12 @@ int main(int argc, const char *argv[])
                                close(2);
                        }
 
-                       /* initialise ctdb */
-                       ctdb = ctdb_cmdline_client(ev);
-
                        if (ctdb_commands[i].without_daemon == false) {
                                const char *socket_name;
 
+                               /* initialise ctdb */
+                               ctdb = ctdb_cmdline_client(ev);
+
                                if (ctdb == NULL) {
                                        DEBUG(DEBUG_ERR, ("Failed to init ctdb\n"));
                                        exit(1);