We can not always rely on the recovery daemon pinging us in a timely manner
[sahlberg/ctdb.git] / server / ctdb_daemon.c
index 5d73b0d5d9b99a21dba6cc7cb3246b99949f6a08..362f1cee5768a4fc098c59b36b3d25bb7973af07 100644 (file)
@@ -43,6 +43,33 @@ static void print_exit_message(void)
        DEBUG(DEBUG_NOTICE,("CTDB daemon shutting down\n"));
 }
 
+
+
+static void ctdb_time_tick(struct event_context *ev, struct timed_event *te, 
+                                 struct timeval t, void *private_data)
+{
+       struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+       if (getpid() != ctdbd_pid) {
+               return;
+       }
+
+       event_add_timed(ctdb->ev, ctdb, 
+                       timeval_current_ofs(1, 0), 
+                       ctdb_time_tick, ctdb);
+}
+
+/* Used to trigger a dummy event once per second, to make
+ * detection of hangs more reliable.
+ */
+static void ctdb_start_time_tickd(struct ctdb_context *ctdb)
+{
+       event_add_timed(ctdb->ev, ctdb, 
+                       timeval_current_ofs(1, 0), 
+                       ctdb_time_tick, ctdb);
+}
+
+
 /* called when the "startup" event script has finished */
 static void ctdb_start_transport(struct ctdb_context *ctdb)
 {
@@ -77,6 +104,9 @@ static void ctdb_start_transport(struct ctdb_context *ctdb)
 
        /* start listening for recovery daemon pings */
        ctdb_control_recd_ping(ctdb);
+
+       /* start listening to timer ticks */
+       ctdb_start_time_tickd(ctdb);
 }
 
 static void block_signal(int signum)
@@ -97,7 +127,7 @@ static void block_signal(int signum)
  */
 static int daemon_queue_send(struct ctdb_client *client, struct ctdb_req_header *hdr)
 {
-       client->ctdb->statistics.client_packets_sent++;
+       CTDB_INCREMENT_STAT(client->ctdb, client_packets_sent);
        if (hdr->operation == CTDB_REQ_MESSAGE) {
                if (ctdb_queue_length(client->queue) > client->ctdb->tunable.max_queue_depth_drop_msg) {
                        DEBUG(DEBUG_ERR,("CTDB_REQ_MESSAGE queue full - killing client connection.\n"));
@@ -184,9 +214,7 @@ static int ctdb_client_destructor(struct ctdb_client *client)
 
        ctdb_takeover_client_destructor_hook(client);
        ctdb_reqid_remove(client->ctdb, client->client_id);
-       if (client->ctdb->statistics.num_clients) {
-               client->ctdb->statistics.num_clients--;
-       }
+       CTDB_DECREMENT_STAT(client->ctdb, num_clients);
 
        if (client->num_persistent_updates != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Client disconnecting with %u persistent updates in flight. Starting recovery\n", client->num_persistent_updates));
@@ -258,10 +286,9 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
        res = ctdb_daemon_call_recv(state, dstate->call);
        if (res != 0) {
                DEBUG(DEBUG_ERR, (__location__ " ctdbd_call_recv() returned error\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       client->ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client_cb 1", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
+
+               CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 1", call_latency, dstate->start_time);
                return;
        }
 
@@ -270,10 +297,8 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
                               length, struct ctdb_reply_call);
        if (r == NULL) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to allocate reply_call in ctdb daemon\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       client->ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client_cb 2", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 2", call_latency, dstate->start_time);
                return;
        }
        r->hdr.reqid        = dstate->reqid;
@@ -288,11 +313,9 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
        if (res != 0) {
                DEBUG(DEBUG_ERR, (__location__ " Failed to queue packet from daemon to client\n"));
        }
-       ctdb_latency(ctdb_db, "call_from_client_cb 3", &client->ctdb->statistics.max_call_latency, dstate->start_time);
+       CTDB_UPDATE_LATENCY(client->ctdb, ctdb_db, "call_from_client_cb 3", call_latency, dstate->start_time);
+       CTDB_DECREMENT_STAT(client->ctdb, pending_calls);
        talloc_free(dstate);
-       if (client->ctdb->statistics.pending_calls > 0) {
-               client->ctdb->statistics.pending_calls--;
-       }
 }
 
 struct ctdb_daemon_packet_wrap {
@@ -344,18 +367,14 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
        struct ctdb_context *ctdb = client->ctdb;
        struct ctdb_daemon_packet_wrap *w;
 
-       ctdb->statistics.total_calls++;
-       if (client->ctdb->statistics.pending_calls > 0) {
-               ctdb->statistics.pending_calls++;
-       }
+       CTDB_INCREMENT_STAT(ctdb, total_calls);
+       CTDB_DECREMENT_STAT(ctdb, pending_calls);
 
        ctdb_db = find_ctdb_db(client->ctdb, c->db_id);
        if (!ctdb_db) {
                DEBUG(DEBUG_ERR, (__location__ " Unknown database in request. db_id==0x%08x",
                          c->db_id));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -383,9 +402,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                                           daemon_incoming_packet_wrap, w, True);
        if (ret == -2) {
                /* will retry later */
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -393,9 +410,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 
        if (ret != 0) {
                DEBUG(DEBUG_ERR,(__location__ " Unable to fetch record\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
 
@@ -407,9 +422,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                }
 
                DEBUG(DEBUG_ERR,(__location__ " Unable to allocate dstate\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
                return;
        }
        dstate->start_time = timeval_current();
@@ -425,10 +438,8 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
                }
 
                DEBUG(DEBUG_ERR,(__location__ " Unable to allocate call\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client 1", &ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(ctdb, ctdb_db, "call_from_client 1", call_latency, dstate->start_time);
                return;
        }
 
@@ -451,10 +462,8 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 
        if (state == NULL) {
                DEBUG(DEBUG_ERR,(__location__ " Unable to setup call send\n"));
-               if (client->ctdb->statistics.pending_calls > 0) {
-                       ctdb->statistics.pending_calls--;
-               }
-               ctdb_latency(ctdb_db, "call_from_client 2", &ctdb->statistics.max_call_latency, dstate->start_time);
+               CTDB_DECREMENT_STAT(ctdb, pending_calls);
+               CTDB_UPDATE_LATENCY(ctdb, ctdb_db, "call_from_client 2", call_latency, dstate->start_time);
                return;
        }
        talloc_steal(state, dstate);
@@ -494,17 +503,17 @@ static void daemon_incoming_packet(void *p, struct ctdb_req_header *hdr)
 
        switch (hdr->operation) {
        case CTDB_REQ_CALL:
-               ctdb->statistics.client.req_call++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_call);
                daemon_request_call_from_client(client, (struct ctdb_req_call *)hdr);
                break;
 
        case CTDB_REQ_MESSAGE:
-               ctdb->statistics.client.req_message++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_message);
                daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr);
                break;
 
        case CTDB_REQ_CONTROL:
-               ctdb->statistics.client.req_control++;
+               CTDB_INCREMENT_STAT(ctdb, client.req_control);
                daemon_request_control_from_client(client, (struct ctdb_req_control *)hdr);
                break;
 
@@ -530,7 +539,7 @@ static void ctdb_daemon_read_cb(uint8_t *data, size_t cnt, void *args)
                return;
        }
 
-       client->ctdb->statistics.client_packets_recv++;
+       CTDB_INCREMENT_STAT(client->ctdb, client_packets_recv);
 
        if (cnt < sizeof(*hdr)) {
                ctdb_set_error(client->ctdb, "Bad packet length %u in daemon\n", 
@@ -635,7 +644,7 @@ static void ctdb_accept_client(struct event_context *ev, struct fd_event *fde,
 
        talloc_set_destructor(client, ctdb_client_destructor);
        talloc_set_destructor(client_pid, ctdb_clientpid_destructor);
-       ctdb->statistics.num_clients++;
+       CTDB_INCREMENT_STAT(ctdb, num_clients);
 }
 
 
@@ -725,7 +734,7 @@ static void ctdb_setup_event_callback(struct ctdb_context *ctdb, int status,
 /*
   start the protocol going as a daemon
 */
-int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog, const char *public_address_list)
 {
        int res, ret = -1;
        struct fd_event *fde;
@@ -763,7 +772,10 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
 
        DEBUG(DEBUG_ERR, ("Starting CTDBD as pid : %u\n", ctdbd_pid));
 
-       ctdb_high_priority(ctdb);
+       if (ctdb->do_setsched) {
+               /* try to set us up as realtime */
+               ctdb_set_scheduler(ctdb);
+       }
 
        /* ensure the socket is deleted on exit of the daemon */
        domain_socket_name = talloc_strdup(talloc_autofree_context(), ctdb->daemon.name);
@@ -774,9 +786,17 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
 
        ctdb->ev = event_context_init(NULL);
        tevent_loop_allow_nesting(ctdb->ev);
+       ret = ctdb_init_tevent_logging(ctdb);
+       if (ret != 0) {
+               DEBUG(DEBUG_ALERT,("Failed to initialize TEVENT logging\n"));
+               exit(1);
+       }
 
        ctdb_set_child_logging(ctdb);
 
+       /* initialize statistics collection */
+       ctdb_statistics_init(ctdb);
+
        /* force initial recovery for election */
        ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
 
@@ -804,6 +824,14 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
        if (ctdb->methods->initialise(ctdb) != 0) {
                ctdb_fatal(ctdb, "transport failed to initialise");
        }
+       if (public_address_list) {
+               ret = ctdb_set_public_addresses(ctdb, public_address_list);
+               if (ret == -1) {
+                       DEBUG(DEBUG_ALERT,("Unable to setup public address list\n"));
+                       exit(1);
+               }
+       }
+
 
        /* attach to existing databases */
        if (ctdb_attach_databases(ctdb) != 0) {
@@ -828,7 +856,9 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog)
        tevent_fd_set_auto_close(fde);
 
        /* release any IPs we hold from previous runs of the daemon */
-       ctdb_release_all_ips(ctdb);
+       if (ctdb->tunable.disable_ip_failover == 0) {
+               ctdb_release_all_ips(ctdb);
+       }
 
        /* start the transport going */
        ctdb_start_transport(ctdb);
@@ -887,7 +917,7 @@ struct ctdb_req_header *_ctdb_transport_allocate(struct ctdb_context *ctdb,
        size = (length+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1);
 
        if (ctdb->methods == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Unable to allocate transport packet for operation %u of length %u. Transport is DOWN.\n",
+               DEBUG(DEBUG_INFO,(__location__ " Unable to allocate transport packet for operation %u of length %u. Transport is DOWN.\n",
                         operation, (unsigned)length));
                return NULL;
        }
@@ -1112,7 +1142,7 @@ int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t pnn,
        int len;
 
        if (ctdb->methods == NULL) {
-               DEBUG(DEBUG_ERR,(__location__ " Failed to send message. Transport is DOWN\n"));
+               DEBUG(DEBUG_INFO,(__location__ " Failed to send message. Transport is DOWN\n"));
                return -1;
        }