tsocket: optimize tdgram_bsd a lot

The desire is to do as less syscalls during the
tdgram_sendto_send/recv() and tdgram_recvfrom_send/recv()
operations.

1. we first try the sendto()/recvfrom() syscall and
   only use a fd event if we got EAGAIN.

2. we cache the fd event and only change it's flags
   if really needed.

For the highload case we do almost no epoll_ctl() and epoll_wait()/select()
syscalls anymore. This speeds up the LDAP-BENCH-CLDAP test
by more than 20%. (With a modified version of this test
which let the server skip any ldb calls and just return success
I'm getting about 8000 requests per second, while I'm getting
just about 6000 requests per second without optimization)

metze

diff --git a/lib/tsocket/tsocket_bsd.c b/lib/tsocket/tsocket_bsd.c
index 60f74204c858492720d90cc3bdf501993902fd83..2f03d3b19d46e7e2f3c631da0c657bf2c690098b 100644 (file)
--- a/lib/tsocket/tsocket_bsd.c
+++ b/lib/tsocket/tsocket_bsd.c
@@ -1313,6 +1313,10 @@ static void tdgram_bsd_fde_handler(struct tevent_context *ev,
                return;
        }
        if (flags & TEVENT_FD_READ) {
+               if (!bsds->readable_handler) {
+                       TEVENT_FD_NOT_READABLE(bsds->fde);
+                       return;
+               }
                bsds->readable_handler(bsds->readable_private);
                return;
        }
@@ -1328,19 +1332,25 @@ static int tdgram_bsd_set_readable_handler(struct tdgram_bsd *bsds,
                        errno = EINVAL;
                        return -1;
                }
-
+               if (!bsds->readable_handler) {
+                       return 0;
+               }
                bsds->readable_handler = NULL;
                bsds->readable_private = NULL;
-               TEVENT_FD_NOT_READABLE(bsds->fde);
 
-               if (bsds->fde && !bsds->writeable_handler) {
-                       /* we don't need the fd event anymore */
-                       bsds->event_ptr = NULL;
-                       TALLOC_FREE(bsds->fde);
-               }
                return 0;
        }
 
+       /* read and write must use the same tevent_context */
+       if (bsds->event_ptr != ev) {
+               if (bsds->readable_handler || bsds->writeable_handler) {
+                       errno = EINVAL;
+                       return -1;
+               }
+               bsds->event_ptr = NULL;
+               TALLOC_FREE(bsds->fde);
+       }
+
        if (bsds->fde == NULL) {
                bsds->fde = tevent_add_fd(ev, bsds,
                                          bsds->fd, TEVENT_FD_READ,
@@ -1352,15 +1362,10 @@ static int tdgram_bsd_set_readable_handler(struct tdgram_bsd *bsds,
 
                /* cache the event context we're running on */
                bsds->event_ptr = ev;
+       } else if (!bsds->readable_handler) {
+               TEVENT_FD_READABLE(bsds->fde);
        }
 
-       /* read and write must use the same tevent_context */
-       if (bsds->event_ptr != ev) {
-               errno = EINVAL;
-               return -1;
-       }
-
-       TEVENT_FD_READABLE(bsds->fde);
        bsds->readable_handler = handler;
        bsds->readable_private = private_data;
 
@@ -1377,19 +1382,26 @@ static int tdgram_bsd_set_writeable_handler(struct tdgram_bsd *bsds,
                        errno = EINVAL;
                        return -1;
                }
-
+               if (!bsds->writeable_handler) {
+                       return 0;
+               }
                bsds->writeable_handler = NULL;
                bsds->writeable_private = NULL;
                TEVENT_FD_NOT_WRITEABLE(bsds->fde);
 
-               if (bsds->fde && !bsds->readable_handler) {
-                       /* we don't need the fd event anymore */
-                       bsds->event_ptr = NULL;
-                       TALLOC_FREE(bsds->fde);
-               }
                return 0;
        }
 
+       /* read and write must use the same tevent_context */
+       if (bsds->event_ptr != ev) {
+               if (bsds->readable_handler || bsds->writeable_handler) {
+                       errno = EINVAL;
+                       return -1;
+               }
+               bsds->event_ptr = NULL;
+               TALLOC_FREE(bsds->fde);
+       }
+
        if (bsds->fde == NULL) {
                bsds->fde = tevent_add_fd(ev, bsds,
                                          bsds->fd, TEVENT_FD_WRITE,
@@ -1401,15 +1413,10 @@ static int tdgram_bsd_set_writeable_handler(struct tdgram_bsd *bsds,
 
                /* cache the event context we're running on */
                bsds->event_ptr = ev;
+       } else if (!bsds->writeable_handler) {
+               TEVENT_FD_WRITEABLE(bsds->fde);
        }
 
-       /* read and write must use the same tevent_context */
-       if (bsds->event_ptr != ev) {
-               errno = EINVAL;
-               return -1;
-       }
-
-       TEVENT_FD_WRITEABLE(bsds->fde);
        bsds->writeable_handler = handler;
        bsds->writeable_private = private_data;
 
@@ -1470,6 +1477,16 @@ static struct tevent_req *tdgram_bsd_recvfrom_send(TALLOC_CTX *mem_ctx,
                goto post;
        }
 
+       /*
+        * this is a fast path, not waiting for the
+        * socket to become explicit readable gains
+        * about 10%-20% performance in benchmark tests.
+        */
+       tdgram_bsd_recvfrom_handler(req);
+       if (!tevent_req_is_in_progress(req)) {
+               goto post;
+       }
+
        ret = tdgram_bsd_set_readable_handler(bsds, ev,
                                              tdgram_bsd_recvfrom_handler,
                                              req);