source3/modules/vfs_io_uring.c

   1 /*
   2  * Use the io_uring of Linux (>= 5.1)
   3  *
   4  * Copyright (C) Volker Lendecke 2008
   5  * Copyright (C) Jeremy Allison 2010
   6  * Copyright (C) Stefan Metzmacher 2019
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21  */
  22
  23 #include "includes.h"
  24 #include "system/filesys.h"
  25 #include "smbd/smbd.h"
  26 #include "smbd/globals.h"
  27 #include "lib/util/tevent_unix.h"
  28 #include "lib/util/sys_rw.h"
  29 #include "smbprofile.h"
  30 #include <liburing.h>
  31
  32 struct vfs_io_uring_request;
  33
  34 struct vfs_io_uring_config {
  35         struct io_uring uring;
  36         struct tevent_fd *fde;
  37         /* recursion guard. See comment above vfs_io_uring_queue_run() */
  38         bool busy;
  39         /* recursion guard. See comment above vfs_io_uring_queue_run() */
  40         bool need_retry;
  41         struct vfs_io_uring_request *queue;
  42         struct vfs_io_uring_request *pending;
  43 };
  44
  45 struct vfs_io_uring_request {
  46         struct vfs_io_uring_request *prev, *next;
  47         struct vfs_io_uring_request **list_head;
  48         struct vfs_io_uring_config *config;
  49         struct tevent_req *req;
  50         struct io_uring_sqe sqe;
  51         struct io_uring_cqe cqe;
  52         void (*completion_fn)(struct vfs_io_uring_request *cur,
  53                               const char *location);
  54         struct timespec start_time;
  55         struct timespec end_time;
  56         SMBPROFILE_BYTES_ASYNC_STATE(profile_bytes);
  57 };
  58
  59 static void vfs_io_uring_finish_req(struct vfs_io_uring_request *cur,
  60                                     const struct io_uring_cqe *cqe,
  61                                     struct timespec end_time,
  62                                     const char *location)
  63 {
  64         struct tevent_req *req =
  65                 talloc_get_type_abort(cur->req,
  66                 struct tevent_req);
  67         void *state = _tevent_req_data(req);
  68
  69         talloc_set_destructor(state, NULL);
  70         if (cur->list_head != NULL) {
  71                 DLIST_REMOVE((*cur->list_head), cur);
  72                 cur->list_head = NULL;
  73         }
  74         cur->cqe = *cqe;
  75
  76         SMBPROFILE_BYTES_ASYNC_SET_IDLE(cur->profile_bytes);
  77         cur->end_time = end_time;
  78
  79         /*
  80          * We rely on being inside the _send() function
  81          * or tevent_req_defer_callback() being called
  82          * already.
  83          */
  84         cur->completion_fn(cur, location);
  85 }
  86
  87 static void vfs_io_uring_config_destroy(struct vfs_io_uring_config *config,
  88                                         int ret,
  89                                         const char *location)
  90 {
  91         struct vfs_io_uring_request *cur = NULL, *next = NULL;
  92         struct timespec start_time;
  93         struct timespec end_time;
  94         struct io_uring_cqe err_cqe = {
  95                 .res = ret,
  96         };
  97
  98         PROFILE_TIMESTAMP(&start_time);
  99
 100         if (config->uring.ring_fd != -1) {
 101                 /* TODO: cancel queued and pending requests */
 102                 TALLOC_FREE(config->fde);
 103                 io_uring_queue_exit(&config->uring);
 104                 config->uring.ring_fd = -1;
 105         }
 106
 107         PROFILE_TIMESTAMP(&end_time);
 108
 109         for (cur = config->pending; cur != NULL; cur = next) {
 110                 next = cur->next;
 111                 err_cqe.user_data = (uintptr_t)(void *)cur;
 112                 vfs_io_uring_finish_req(cur, &err_cqe, end_time, location);
 113         }
 114
 115         for (cur = config->queue; cur != NULL; cur = next) {
 116                 next = cur->next;
 117                 err_cqe.user_data = (uintptr_t)(void *)cur;
 118                 cur->start_time = start_time;
 119                 vfs_io_uring_finish_req(cur, &err_cqe, end_time, location);
 120         }
 121 }
 122
 123 static int vfs_io_uring_config_destructor(struct vfs_io_uring_config *config)
 124 {
 125         vfs_io_uring_config_destroy(config, -EUCLEAN, __location__);
 126         return 0;
 127 }
 128
 129 static int vfs_io_uring_request_state_deny_destructor(void *_state)
 130 {
 131         struct __vfs_io_uring_generic_state {
 132                 struct vfs_io_uring_request ur;
 133         } *state = (struct __vfs_io_uring_generic_state *)_state;
 134         struct vfs_io_uring_request *cur = &state->ur;
 135
 136         /* our parent is gone */
 137         cur->req = NULL;
 138
 139         /* remove ourself from any list */
 140         DLIST_REMOVE((*cur->list_head), cur);
 141         cur->list_head = NULL;
 142
 143         /*
 144          * Our state is about to go away,
 145          * all we can do is shutting down the whole uring.
 146          * But that's ok as we're most likely called from exit_server()
 147          */
 148         vfs_io_uring_config_destroy(cur->config, -ESHUTDOWN, __location__);
 149         return 0;
 150 }
 151
 152 static void vfs_io_uring_fd_handler(struct tevent_context *ev,
 153                                     struct tevent_fd *fde,
 154                                     uint16_t flags,
 155                                     void *private_data);
 156
 157 static int vfs_io_uring_connect(vfs_handle_struct *handle, const char *service,
 158                             const char *user)
 159 {
 160         int ret;
 161         struct vfs_io_uring_config *config;
 162         unsigned num_entries;
 163         bool sqpoll;
 164         unsigned flags = 0;
 165
 166         config = talloc_zero(handle->conn, struct vfs_io_uring_config);
 167         if (config == NULL) {
 168                 DEBUG(0, ("talloc_zero() failed\n"));
 169                 return -1;
 170         }
 171
 172         SMB_VFS_HANDLE_SET_DATA(handle, config,
 173                                 NULL, struct vfs_io_uring_config,
 174                                 return -1);
 175
 176         ret = SMB_VFS_NEXT_CONNECT(handle, service, user);
 177         if (ret < 0) {
 178                 return ret;
 179         }
 180
 181         num_entries = lp_parm_ulong(SNUM(handle->conn),
 182                                     "io_uring",
 183                                     "num_entries",
 184                                     128);
 185         num_entries = MAX(num_entries, 1);
 186
 187         sqpoll = lp_parm_bool(SNUM(handle->conn),
 188                              "io_uring",
 189                              "sqpoll",
 190                              false);
 191         if (sqpoll) {
 192                 flags |= IORING_SETUP_SQPOLL;
 193         }
 194
 195         ret = io_uring_queue_init(num_entries, &config->uring, flags);
 196         if (ret < 0) {
 197                 SMB_VFS_NEXT_DISCONNECT(handle);
 198                 errno = -ret;
 199                 return -1;
 200         }
 201
 202         talloc_set_destructor(config, vfs_io_uring_config_destructor);
 203
 204 #ifdef HAVE_IO_URING_RING_DONTFORK
 205         ret = io_uring_ring_dontfork(&config->uring);
 206         if (ret < 0) {
 207                 SMB_VFS_NEXT_DISCONNECT(handle);
 208                 errno = -ret;
 209                 return -1;
 210         }
 211 #endif /* HAVE_IO_URING_RING_DONTFORK */
 212
 213         config->fde = tevent_add_fd(handle->conn->sconn->ev_ctx,
 214                                     config,
 215                                     config->uring.ring_fd,
 216                                     TEVENT_FD_READ,
 217                                     vfs_io_uring_fd_handler,
 218                                     handle);
 219         if (config->fde == NULL) {
 220                 ret = errno;
 221                 SMB_VFS_NEXT_DISCONNECT(handle);
 222                 errno = ret;
 223                 return -1;
 224         }
 225
 226         return 0;
 227 }
 228
 229 static void _vfs_io_uring_queue_run(struct vfs_io_uring_config *config)
 230 {
 231         struct vfs_io_uring_request *cur = NULL, *next = NULL;
 232         struct io_uring_cqe *cqe = NULL;
 233         unsigned cqhead;
 234         unsigned nr = 0;
 235         struct timespec start_time;
 236         struct timespec end_time;
 237         int ret;
 238
 239         PROFILE_TIMESTAMP(&start_time);
 240
 241         if (config->uring.ring_fd == -1) {
 242                 vfs_io_uring_config_destroy(config, -ESTALE, __location__);
 243                 return;
 244         }
 245
 246         for (cur = config->queue; cur != NULL; cur = next) {
 247                 struct io_uring_sqe *sqe = NULL;
 248                 void *state = _tevent_req_data(cur->req);
 249
 250                 next = cur->next;
 251
 252                 sqe = io_uring_get_sqe(&config->uring);
 253                 if (sqe == NULL) {
 254                         break;
 255                 }
 256
 257                 talloc_set_destructor(state,
 258                         vfs_io_uring_request_state_deny_destructor);
 259                 DLIST_REMOVE(config->queue, cur);
 260                 *sqe = cur->sqe;
 261                 DLIST_ADD_END(config->pending, cur);
 262                 cur->list_head = &config->pending;
 263                 SMBPROFILE_BYTES_ASYNC_SET_BUSY(cur->profile_bytes);
 264
 265                 cur->start_time = start_time;
 266         }
 267
 268         ret = io_uring_submit(&config->uring);
 269         if (ret == -EAGAIN || ret == -EBUSY) {
 270                 /* We just retry later */
 271         } else if (ret < 0) {
 272                 vfs_io_uring_config_destroy(config, ret, __location__);
 273                 return;
 274         }
 275
 276         PROFILE_TIMESTAMP(&end_time);
 277
 278         io_uring_for_each_cqe(&config->uring, cqhead, cqe) {
 279                 cur = (struct vfs_io_uring_request *)io_uring_cqe_get_data(cqe);
 280                 vfs_io_uring_finish_req(cur, cqe, end_time, __location__);
 281                 nr++;
 282         }
 283
 284         io_uring_cq_advance(&config->uring, nr);
 285 }
 286
 287 /*
 288  * Wrapper function to prevent recursion which could happen
 289  * if we called _vfs_io_uring_queue_run() directly without
 290  * recursion checks.
 291  *
 292  * Looking at the pread call, we can have:
 293  *
 294  * vfs_io_uring_pread_send()
 295  *        ->vfs_io_uring_pread_submit()  <-----------------------------------
 296  *                ->vfs_io_uring_request_submit()                           |
 297  *                        ->vfs_io_uring_queue_run()                        |
 298  *                                ->_vfs_io_uring_queue_run()               |
 299  *                                                                          |
 300  * But inside _vfs_io_uring_queue_run() looks like:                         |
 301  *                                                                          |
 302  * _vfs_io_uring_queue_run() {                                              |
 303  *      if (THIS_IO_COMPLETED) {                                            |
 304  *              ->vfs_io_uring_finish_req()                                 |
 305  *                      ->cur->completion_fn()                              |
 306  *      }                                                                   |
 307  * }                                                                        |
 308  *                                                                          |
 309  * cur->completion_fn() for pread is set to vfs_io_uring_pread_completion() |
 310  *                                                                          |
 311  * vfs_io_uring_pread_completion() {                                        |
 312  *      if (READ_TERMINATED) {                                              |
 313  *              -> tevent_req_done() - We're done, go back up the stack.    |
 314  *              return;                                                     |
 315  *      }                                                                   |
 316  *                                                                          |
 317  *      We have a short read - adjust the io vectors                        |
 318  *                                                                          |
 319  *      ->vfs_io_uring_pread_submit() ---------------------------------------
 320  * }
 321  *
 322  * So before calling _vfs_io_uring_queue_run() we backet it with setting
 323  * a flag config->busy, and unset it once _vfs_io_uring_queue_run() finally
 324  * exits the retry loop.
 325  *
 326  * If we end up back into vfs_io_uring_queue_run() we notice we've done so
 327  * as config->busy is set and don't recurse into _vfs_io_uring_queue_run().
 328  *
 329  * We set the second flag config->need_retry that tells us to loop in the
 330  * vfs_io_uring_queue_run() call above us in the stack and return.
 331  *
 332  * When the outer call to _vfs_io_uring_queue_run() returns we are in
 333  * a loop checking if config->need_retry was set. That happens if
 334  * the short read case occurs and _vfs_io_uring_queue_run() ended up
 335  * recursing into vfs_io_uring_queue_run().
 336  *
 337  * Once vfs_io_uring_pread_completion() finishes without a short
 338  * read (the READ_TERMINATED case, tevent_req_done() is called)
 339  * then config->need_retry is left as false, we exit the loop,
 340  * set config->busy to false so the next top level call into
 341  * vfs_io_uring_queue_run() won't think it's a recursed call
 342  * and return.
 343  *
 344  */
 345
 346 static void vfs_io_uring_queue_run(struct vfs_io_uring_config *config)
 347 {
 348         if (config->busy) {
 349                 /*
 350                  * We've recursed due to short read/write.
 351                  * Set need_retry to ensure we retry the
 352                  * io_uring_submit().
 353                  */
 354                 config->need_retry = true;
 355                 return;
 356         }
 357
 358         /*
 359          * Bracket the loop calling _vfs_io_uring_queue_run()
 360          * with busy = true / busy = false.
 361          * so we can detect recursion above.
 362          */
 363
 364         config->busy = true;
 365
 366         do {
 367                 config->need_retry = false;
 368                 _vfs_io_uring_queue_run(config);
 369         } while (config->need_retry);
 370
 371         config->busy = false;
 372 }
 373
 374 static void vfs_io_uring_request_submit(struct vfs_io_uring_request *cur)
 375 {
 376         struct vfs_io_uring_config *config = cur->config;
 377
 378         io_uring_sqe_set_data(&cur->sqe, cur);
 379         DLIST_ADD_END(config->queue, cur);
 380         cur->list_head = &config->queue;
 381
 382         vfs_io_uring_queue_run(config);
 383 }
 384
 385 static void vfs_io_uring_fd_handler(struct tevent_context *ev,
 386                                     struct tevent_fd *fde,
 387                                     uint16_t flags,
 388                                     void *private_data)
 389 {
 390         vfs_handle_struct *handle = (vfs_handle_struct *)private_data;
 391         struct vfs_io_uring_config *config = NULL;
 392
 393         SMB_VFS_HANDLE_GET_DATA(handle, config,
 394                                 struct vfs_io_uring_config,
 395                                 smb_panic(__location__));
 396
 397         vfs_io_uring_queue_run(config);
 398 }
 399
 400 struct vfs_io_uring_pread_state {
 401         struct vfs_io_uring_request ur;
 402         struct iovec iov;
 403         size_t nread;
 404 };
 405
 406 static void vfs_io_uring_pread_completion(struct vfs_io_uring_request *cur,
 407                                           const char *location);
 408
 409 static struct tevent_req *vfs_io_uring_pread_send(struct vfs_handle_struct *handle,
 410                                              TALLOC_CTX *mem_ctx,
 411                                              struct tevent_context *ev,
 412                                              struct files_struct *fsp,
 413                                              void *data,
 414                                              size_t n, off_t offset)
 415 {
 416         struct tevent_req *req = NULL;
 417         struct vfs_io_uring_pread_state *state = NULL;
 418         struct vfs_io_uring_config *config = NULL;
 419         bool ok;
 420
 421         SMB_VFS_HANDLE_GET_DATA(handle, config,
 422                                 struct vfs_io_uring_config,
 423                                 smb_panic(__location__));
 424
 425         req = tevent_req_create(mem_ctx, &state,
 426                                 struct vfs_io_uring_pread_state);
 427         if (req == NULL) {
 428                 return NULL;
 429         }
 430         state->ur.config = config;
 431         state->ur.req = req;
 432         state->ur.completion_fn = vfs_io_uring_pread_completion;
 433
 434         SMBPROFILE_BYTES_ASYNC_START(syscall_asys_pread, profile_p,
 435                                      state->ur.profile_bytes, n);
 436         SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes);
 437
 438         ok = sys_valid_io_range(offset, n);
 439         if (!ok) {
 440                 tevent_req_error(req, EINVAL);
 441                 return tevent_req_post(req, ev);
 442         }
 443
 444         state->iov.iov_base = (void *)data;
 445         state->iov.iov_len = n;
 446         io_uring_prep_readv(&state->ur.sqe,
 447                             fsp->fh->fd,
 448                             &state->iov, 1,
 449                             offset);
 450         vfs_io_uring_request_submit(&state->ur);
 451
 452         if (!tevent_req_is_in_progress(req)) {
 453                 return tevent_req_post(req, ev);
 454         }
 455
 456         tevent_req_defer_callback(req, ev);
 457         return req;
 458 }
 459
 460 static void vfs_io_uring_pread_completion(struct vfs_io_uring_request *cur,
 461                                           const char *location)
 462 {
 463         struct vfs_io_uring_pread_state *state = tevent_req_data(
 464                 cur->req, struct vfs_io_uring_pread_state);
 465
 466         /*
 467          * We rely on being inside the _send() function
 468          * or tevent_req_defer_callback() being called
 469          * already.
 470          */
 471
 472         if (cur->cqe.res < 0) {
 473                 int err = -cur->cqe.res;
 474                 _tevent_req_error(cur->req, err, location);
 475                 return;
 476         }
 477
 478         state->nread = state->ur.cqe.res;
 479         tevent_req_done(cur->req);
 480 }
 481
 482 static ssize_t vfs_io_uring_pread_recv(struct tevent_req *req,
 483                                   struct vfs_aio_state *vfs_aio_state)
 484 {
 485         struct vfs_io_uring_pread_state *state = tevent_req_data(
 486                 req, struct vfs_io_uring_pread_state);
 487         ssize_t ret;
 488
 489         SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes);
 490         vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time,
 491                                                  &state->ur.start_time);
 492
 493         if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) {
 494                 tevent_req_received(req);
 495                 return -1;
 496         }
 497
 498         vfs_aio_state->error = 0;
 499         ret = state->nread;
 500
 501         tevent_req_received(req);
 502         return ret;
 503 }
 504
 505 struct vfs_io_uring_pwrite_state {
 506         struct vfs_io_uring_request ur;
 507         struct iovec iov;
 508         size_t nwritten;
 509 };
 510
 511 static void vfs_io_uring_pwrite_completion(struct vfs_io_uring_request *cur,
 512                                            const char *location);
 513
 514 static struct tevent_req *vfs_io_uring_pwrite_send(struct vfs_handle_struct *handle,
 515                                               TALLOC_CTX *mem_ctx,
 516                                               struct tevent_context *ev,
 517                                               struct files_struct *fsp,
 518                                               const void *data,
 519                                               size_t n, off_t offset)
 520 {
 521         struct tevent_req *req = NULL;
 522         struct vfs_io_uring_pwrite_state *state = NULL;
 523         struct vfs_io_uring_config *config = NULL;
 524         bool ok;
 525
 526         SMB_VFS_HANDLE_GET_DATA(handle, config,
 527                                 struct vfs_io_uring_config,
 528                                 smb_panic(__location__));
 529
 530         req = tevent_req_create(mem_ctx, &state,
 531                                 struct vfs_io_uring_pwrite_state);
 532         if (req == NULL) {
 533                 return NULL;
 534         }
 535         state->ur.config = config;
 536         state->ur.req = req;
 537         state->ur.completion_fn = vfs_io_uring_pwrite_completion;
 538
 539         SMBPROFILE_BYTES_ASYNC_START(syscall_asys_pwrite, profile_p,
 540                                      state->ur.profile_bytes, n);
 541         SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes);
 542
 543         ok = sys_valid_io_range(offset, n);
 544         if (!ok) {
 545                 tevent_req_error(req, EINVAL);
 546                 return tevent_req_post(req, ev);
 547         }
 548
 549         state->iov.iov_base = discard_const(data);
 550         state->iov.iov_len = n;
 551         io_uring_prep_writev(&state->ur.sqe,
 552                              fsp->fh->fd,
 553                              &state->iov, 1,
 554                              offset);
 555         vfs_io_uring_request_submit(&state->ur);
 556
 557         if (!tevent_req_is_in_progress(req)) {
 558                 return tevent_req_post(req, ev);
 559         }
 560
 561         tevent_req_defer_callback(req, ev);
 562         return req;
 563 }
 564
 565 static void vfs_io_uring_pwrite_completion(struct vfs_io_uring_request *cur,
 566                                            const char *location)
 567 {
 568         struct vfs_io_uring_pwrite_state *state = tevent_req_data(
 569                 cur->req, struct vfs_io_uring_pwrite_state);
 570
 571         /*
 572          * We rely on being inside the _send() function
 573          * or tevent_req_defer_callback() being called
 574          * already.
 575          */
 576
 577         if (cur->cqe.res < 0) {
 578                 int err = -cur->cqe.res;
 579                 _tevent_req_error(cur->req, err, location);
 580                 return;
 581         }
 582
 583         state->nwritten = state->ur.cqe.res;
 584         tevent_req_done(cur->req);
 585 }
 586
 587 static ssize_t vfs_io_uring_pwrite_recv(struct tevent_req *req,
 588                                    struct vfs_aio_state *vfs_aio_state)
 589 {
 590         struct vfs_io_uring_pwrite_state *state = tevent_req_data(
 591                 req, struct vfs_io_uring_pwrite_state);
 592         ssize_t ret;
 593
 594         SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes);
 595         vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time,
 596                                                  &state->ur.start_time);
 597
 598         if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) {
 599                 tevent_req_received(req);
 600                 return -1;
 601         }
 602
 603         vfs_aio_state->error = 0;
 604         ret = state->nwritten;
 605
 606         tevent_req_received(req);
 607         return ret;
 608 }
 609
 610 struct vfs_io_uring_fsync_state {
 611         struct vfs_io_uring_request ur;
 612 };
 613
 614 static void vfs_io_uring_fsync_completion(struct vfs_io_uring_request *cur,
 615                                           const char *location);
 616
 617 static struct tevent_req *vfs_io_uring_fsync_send(struct vfs_handle_struct *handle,
 618                                              TALLOC_CTX *mem_ctx,
 619                                              struct tevent_context *ev,
 620                                              struct files_struct *fsp)
 621 {
 622         struct tevent_req *req = NULL;
 623         struct vfs_io_uring_fsync_state *state = NULL;
 624         struct vfs_io_uring_config *config = NULL;
 625
 626         SMB_VFS_HANDLE_GET_DATA(handle, config,
 627                                 struct vfs_io_uring_config,
 628                                 smb_panic(__location__));
 629
 630         req = tevent_req_create(mem_ctx, &state,
 631                                 struct vfs_io_uring_fsync_state);
 632         if (req == NULL) {
 633                 return NULL;
 634         }
 635         state->ur.config = config;
 636         state->ur.req = req;
 637         state->ur.completion_fn = vfs_io_uring_fsync_completion;
 638
 639         SMBPROFILE_BYTES_ASYNC_START(syscall_asys_fsync, profile_p,
 640                                      state->ur.profile_bytes, 0);
 641         SMBPROFILE_BYTES_ASYNC_SET_IDLE(state->ur.profile_bytes);
 642
 643         io_uring_prep_fsync(&state->ur.sqe,
 644                             fsp->fh->fd,
 645                             0); /* fsync_flags */
 646         vfs_io_uring_request_submit(&state->ur);
 647
 648         if (!tevent_req_is_in_progress(req)) {
 649                 return tevent_req_post(req, ev);
 650         }
 651
 652         tevent_req_defer_callback(req, ev);
 653         return req;
 654 }
 655
 656 static void vfs_io_uring_fsync_completion(struct vfs_io_uring_request *cur,
 657                                           const char *location)
 658 {
 659         /*
 660          * We rely on being inside the _send() function
 661          * or tevent_req_defer_callback() being called
 662          * already.
 663          */
 664
 665         if (cur->cqe.res < 0) {
 666                 int err = -cur->cqe.res;
 667                 _tevent_req_error(cur->req, err, location);
 668                 return;
 669         }
 670
 671         tevent_req_done(cur->req);
 672 }
 673
 674 static int vfs_io_uring_fsync_recv(struct tevent_req *req,
 675                               struct vfs_aio_state *vfs_aio_state)
 676 {
 677         struct vfs_io_uring_fsync_state *state = tevent_req_data(
 678                 req, struct vfs_io_uring_fsync_state);
 679
 680         SMBPROFILE_BYTES_ASYNC_END(state->ur.profile_bytes);
 681         vfs_aio_state->duration = nsec_time_diff(&state->ur.end_time,
 682                                                  &state->ur.start_time);
 683
 684         if (tevent_req_is_unix_error(req, &vfs_aio_state->error)) {
 685                 tevent_req_received(req);
 686                 return -1;
 687         }
 688
 689         vfs_aio_state->error = 0;
 690
 691         tevent_req_received(req);
 692         return 0;
 693 }
 694
 695 static struct vfs_fn_pointers vfs_io_uring_fns = {
 696         .connect_fn = vfs_io_uring_connect,
 697         .pread_send_fn = vfs_io_uring_pread_send,
 698         .pread_recv_fn = vfs_io_uring_pread_recv,
 699         .pwrite_send_fn = vfs_io_uring_pwrite_send,
 700         .pwrite_recv_fn = vfs_io_uring_pwrite_recv,
 701         .fsync_send_fn = vfs_io_uring_fsync_send,
 702         .fsync_recv_fn = vfs_io_uring_fsync_recv,
 703 };
 704
 705 static_decl_vfs;
 706 NTSTATUS vfs_io_uring_init(TALLOC_CTX *ctx)
 707 {
 708         return smb_register_vfs(SMB_VFS_INTERFACE_VERSION,
 709                                 "io_uring", &vfs_io_uring_fns);
 710 }