We only queued up to 1000 packets per queue before we start dropping

packets, to avoid the queue to grow excessively if smbd has blocked.

This could cause traverse packets to become discarded in case the main
smbd daemon does a traverse of a database while there is a recovery
(sending a erconfigured message to smbd, causing an avalanche of unlock
messages to be sent across the cluster.)

This avalance of messages could cause also the tranversal message to be
discarded  causing the main smbd process to hang indefinitely waiting
for the traversal message that will never arrive.

Bump the maximum queue length before starting to discard messages from
1000 to 1000000 and at the same time rework the queueing slightly so we
can append messages cheaply to the queue instead of walking the list
from head to tail every time.

diff --git a/common/ctdb_io.c b/common/ctdb_io.c
index 28830d5f3700d67bb38fb38ebb4d55a918b0abbb..47681d2a5660788a0c8e3813dadb5f9593c4216d 100644 (file)
--- a/common/ctdb_io.c
+++ b/common/ctdb_io.c
@@ -46,6 +46,12 @@ struct ctdb_queue {
        struct ctdb_context *ctdb;
        struct ctdb_partial partial; /* partial input packet */
        struct ctdb_queue_pkt *out_queue;
+       /* This field is used to track the last added item so we
+          can append new items to the end cheaply.
+          This relies of that items are always appended to the tail
+          and that when reamoving items we only remove the head.
+       */
+       struct ctdb_queue_pkt *out_queue_last_added;
        uint32_t out_queue_length;
        struct fd_event *fde;
        int fd;
@@ -294,7 +300,18 @@ int ctdb_queue_send(struct ctdb_queue *queue, uint8_t *data, uint32_t length)
                EVENT_FD_WRITEABLE(queue->fde);
        }
 
-       DLIST_ADD_END(queue->out_queue, pkt, struct ctdb_queue_pkt *);
+       /* This relies on that when adding items to the queue, we always add
+          them to the tail and that when removing items we only remove
+          the head of queue item.
+          The last_added item thus allows non n^2 behaviour when appending to
+          very long queues.
+       */
+       if (queue->out_queue == NULL) {
+               DLIST_ADD(queue->out_queue, pkt);
+       } else {
+               DLIST_ADD_END(queue->out_queue_last_added, pkt, struct ctdb_queue_pkt *);
+       }
+       queue->out_queue_last_added = pkt;
        queue->out_queue_length++;
 
        if (queue->ctdb->tunable.verbose_memory_names != 0) {

diff --git a/server/ctdb_tunables.c b/server/ctdb_tunables.c
index e75dcbd74f0067a055f112660bf8cb75c3ee04d7..cca270b95c484970c2d76a0f5e52a851afab3383 100644 (file)
--- a/server/ctdb_tunables.c
+++ b/server/ctdb_tunables.c
@@ -62,7 +62,7 @@ static const struct {
        { "VacuumLimit",       5000,  offsetof(struct ctdb_tunable, vacuum_limit) },
        { "VacuumMinInterval",   60,  offsetof(struct ctdb_tunable, vacuum_min_interval) },
        { "VacuumMaxInterval",  600,  offsetof(struct ctdb_tunable, vacuum_max_interval) },
-       { "MaxQueueDropMsg",  1000,  offsetof(struct ctdb_tunable, max_queue_depth_drop_msg) },
+       { "MaxQueueDropMsg",  1000000, offsetof(struct ctdb_tunable, max_queue_depth_drop_msg) },
        { "UseStatusEvents",     0,  offsetof(struct ctdb_tunable, use_status_events_for_monitoring) },
        { "AllowUnhealthyDBRead", 0,  offsetof(struct ctdb_tunable, allow_unhealthy_db_read) }
 };