during shutdown there is a window after we have stopped TCP and disconnected from...

[metze/ctdb/wip.git] / server / ctdb_daemon.c

diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 1d803058e9877d053960a860dc17f5ee2e466f40..5eca7275c685dd5cdf3524e3abf1a3d00a7ba6d1 100644 (file)
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -97,7 +97,7 @@ static void block_signal(int signum)
  */
 static int daemon_queue_send(struct ctdb_client *client, struct ctdb_req_header *hdr)
 {
-       client->ctdb->statistics.client_packets_sent++;
+       CTDB_INCREMENT_STAT(client->ctdb, client_packets_sent);
        if (hdr->operation == CTDB_REQ_MESSAGE) {
                if (ctdb_queue_length(client->queue) > client->ctdb->tunable.max_queue_depth_drop_msg) {
                        DEBUG(DEBUG_ERR,("CTDB_REQ_MESSAGE queue full - killing client connection.\n"));
@@ -184,9 +184,7 @@ static int ctdb_client_destructor(struct ctdb_client *client)
 
        ctdb_takeover_client_destructor_hook(client);
        ctdb_reqid_remove(client->ctdb, client->client_id);
-       if (client->ctdb->statistics.num_clients) {
-               client->ctdb->statistics.num_clients--;
-       }
+       CTDB_DECREMENT_STAT(client->ctdb, num_clients);
 
        if (client->num_persistent_updates != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Client disconnecting with %u persistent updates in flight. Starting recovery\n", client->num_persistent_updates));
@@ -258,10 +256,9 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
        res = ctdb_daemon_call_recv(state, dstate->call);
        if (res != 0) {
                DEBUG(DEBUG_ERR, (__location__ " ctdbd_call_recv() returned error\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       client->ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client_cb 1", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
+
+               CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 1", call_latency, dstate->start_time);
                return;
        }
 
@@ -270,10 +267,8 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
                               length, struct ctdb_reply_call);
        if (r == NULL) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to allocate reply_call in ctdb daemon\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       client->ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client_cb 2", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 2", call_latency, dstate->start_time);
                return;
        }
        r->hdr.reqid        = dstate->reqid;
@@ -288,11 +283,9 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
        if (res != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to queue packet from daemon to client\n"));
        }
-       ctdb_latency(ctdb_db, "call_from_client_cb 3", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+       CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 3", call_latency, dstate->start_time);
+       CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
        talloc_free(dstate);
-       if (client->ctdb->statistics.pending_calls > 0) {
-               client->ctdb->statistics.pending_calls--;
-       }
 }
 
 struct ctdb_daemon_packet_wrap {
@@ -344,18 +337,14 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
        struct ctdb_context *ctdb = client->ctdb;
        struct ctdb_daemon_packet_wrap *w;
 
-       ctdb->statistics.total_calls++;
-       if (client->ctdb->statistics.pending_calls > 0) {
-               ctdb->statistics.pending_calls++;
-       }
+       CTDB_INCREMENT_STAT(ctdb, total_calls);
+       CTDB_DECREMENT_STAT(ctdb, pending_calls);
 
        ctdb_db = find_ctdb_db(client->ctdb, c->db_id);
        if (!ctdb_db) {
                DEBUG(DEBUG_ERR, (__location__ " Unknown database in request. db_id==0x%08x",
                          c->db_id));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -383,9 +372,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                                           daemon_incoming_packet_wrap, w, True);
        if (ret == -2) {
                /* will retry later */
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -393,9 +380,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 
        if (ret != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Unable to fetch record\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -407,9 +392,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                }
 
                DEBUG(DEBUG_ERR,(__location__ " Unable to allocate dstate\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
        dstate->start_time = timeval_current();
@@ -425,10 +408,8 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                }
 
                DEBUG(DEBUG_ERR,(__location__ " Unable to allocate call\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client 1", &ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(ctdb, ctdb_db, "call_from_client 1", call_latency, dstate->start_time);
                return;
        }
 
@@ -451,10 +432,8 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 
        if (state == NULL) {
                DEBUG(DEBUG_ERR,(__location__ " Unable to setup call send\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client 2", &ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(ctdb, ctdb_db, "call_from_client 2", call_latency, dstate->start_time);
                return;
        }
        talloc_steal(state, dstate);
@@ -494,17 +473,17 @@ static void daemon_incoming_packet(void *p, struct ctdb_req_header *hdr)
 
        switch (hdr->operation) {
        case CTDB_REQ_CALL:
-               ctdb->statistics.client.req_call++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_call);
                daemon_request_call_from_client(client, (struct ctdb_req_call *)hdr);
                break;
 
        case CTDB_REQ_MESSAGE:
-               ctdb->statistics.client.req_message++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_message);
                daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr);
                break;
 
        case CTDB_REQ_CONTROL:
-               ctdb->statistics.client.req_control++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_control);
                daemon_request_control_from_client(client, (struct ctdb_req_control *)hdr);
                break;
 
@@ -530,7 +509,7 @@ static void ctdb_daemon_read_cb(uint8_t *data, size_t cnt, void *args)
                return;
        }
 
-       client->ctdb->statistics.client_packets_recv++;
+       CTDB_INCREMENT_STAT(client->ctdb, client_packets_recv);
 
        if (cnt < sizeof(*hdr)) {
                ctdb_set_error(client->ctdb, "Bad packet length %u in daemon\n", 
@@ -635,7 +614,7 @@ static void ctdb_accept_client(struct event_context *ev, struct fd_event *fde,
 
        talloc_set_destructor(client, ctdb_client_destructor);
        talloc_set_destructor(client_pid, ctdb_clientpid_destructor);
-       ctdb->statistics.num_clients++;
+       CTDB_INCREMENT_STAT(ctdb, num_clients);
 }
 
 
@@ -774,9 +753,17 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
 
        ctdb->ev = event_context_init(NULL);
        tevent_loop_allow_nesting(ctdb->ev);
+       ret = ctdb_init_tevent_logging(ctdb);
+       if (ret != 0) {
+               DEBUG(DEBUG_ALERT,("Failed to initialize TEVENT logging\n"));
+               exit(1);
+       }
 
        ctdb_set_child_logging(ctdb);
 
+       /* initialize statistics collection */
+       ctdb_statistics_init(ctdb);
+
        /* force initial recovery for election */
        ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
 
@@ -810,17 +797,17 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
                ctdb_fatal(ctdb, "Failed to attach to databases\n");
        }
 
-       /* start frozen, then let the first election sort things out */
-       if (ctdb_blocking_freeze(ctdb)) {
-               ctdb_fatal(ctdb, "Failed to get initial freeze\n");
-       }
-
        ret = ctdb_event_script(ctdb, CTDB_EVENT_INIT);
        if (ret != 0) {
                ctdb_fatal(ctdb, "Failed to run init event\n");
        }
        ctdb_run_notification_script(ctdb, "init");
 
+       /* start frozen, then let the first election sort things out */
+       if (ctdb_blocking_freeze(ctdb)) {
+               ctdb_fatal(ctdb, "Failed to get initial freeze\n");
+       }
+
        /* now start accepting clients, only can do this once frozen */
        fde = event_add_fd(ctdb->ev, ctdb, ctdb->daemon.sd, 
                           EVENT_FD_READ,
@@ -887,7 +874,7 @@ struct ctdb_req_header *_ctdb_transport_allocate(struct ctdb_context *ctdb,
        size = (length+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1);
 
        if (ctdb->methods == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Unable to allocate transport packet for operation %u of length %u. Transport is DOWN.\n",
+               DEBUG(DEBUG_INFO,(__location__ " Unable to allocate transport packet for operation %u of length %u. Transport is DOWN.\n",
                         operation, (unsigned)length));
                return NULL;
        }
@@ -1112,7 +1099,7 @@ int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t pnn,
        int len;
 
        if (ctdb->methods == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed to send message. Transport is DOWN\n"));
+               DEBUG(DEBUG_INFO,(__location__ " Failed to send message. Transport is DOWN\n"));
                return -1;
        }