io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36 #include "waitid.h"
  37 #include "futex.h"
  38 #include "truncate.h"
  39
  40 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  41 {
  42         WARN_ON_ONCE(1);
  43         return -ECANCELED;
  44 }
  45
  46 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  47                                              const struct io_uring_sqe *sqe)
  48 {
  49         return -EOPNOTSUPP;
  50 }
  51
  52 const struct io_issue_def io_issue_defs[] = {
  53         [IORING_OP_NOP] = {
  54                 .audit_skip             = 1,
  55                 .iopoll                 = 1,
  56                 .prep                   = io_nop_prep,
  57                 .issue                  = io_nop,
  58         },
  59         [IORING_OP_READV] = {
  60                 .needs_file             = 1,
  61                 .unbound_nonreg_file    = 1,
  62                 .pollin                 = 1,
  63                 .buffer_select          = 1,
  64                 .plug                   = 1,
  65                 .audit_skip             = 1,
  66                 .ioprio                 = 1,
  67                 .iopoll                 = 1,
  68                 .iopoll_queue           = 1,
  69                 .vectored               = 1,
  70                 .prep                   = io_prep_rwv,
  71                 .issue                  = io_read,
  72         },
  73         [IORING_OP_WRITEV] = {
  74                 .needs_file             = 1,
  75                 .hash_reg_file          = 1,
  76                 .unbound_nonreg_file    = 1,
  77                 .pollout                = 1,
  78                 .plug                   = 1,
  79                 .audit_skip             = 1,
  80                 .ioprio                 = 1,
  81                 .iopoll                 = 1,
  82                 .iopoll_queue           = 1,
  83                 .vectored               = 1,
  84                 .prep                   = io_prep_rwv,
  85                 .issue                  = io_write,
  86         },
  87         [IORING_OP_FSYNC] = {
  88                 .needs_file             = 1,
  89                 .audit_skip             = 1,
  90                 .prep                   = io_fsync_prep,
  91                 .issue                  = io_fsync,
  92         },
  93         [IORING_OP_READ_FIXED] = {
  94                 .needs_file             = 1,
  95                 .unbound_nonreg_file    = 1,
  96                 .pollin                 = 1,
  97                 .plug                   = 1,
  98                 .audit_skip             = 1,
  99                 .ioprio                 = 1,
 100                 .iopoll                 = 1,
 101                 .iopoll_queue           = 1,
 102                 .prep                   = io_prep_rw_fixed,
 103                 .issue                  = io_read,
 104         },
 105         [IORING_OP_WRITE_FIXED] = {
 106                 .needs_file             = 1,
 107                 .hash_reg_file          = 1,
 108                 .unbound_nonreg_file    = 1,
 109                 .pollout                = 1,
 110                 .plug                   = 1,
 111                 .audit_skip             = 1,
 112                 .ioprio                 = 1,
 113                 .iopoll                 = 1,
 114                 .iopoll_queue           = 1,
 115                 .prep                   = io_prep_rw_fixed,
 116                 .issue                  = io_write,
 117         },
 118         [IORING_OP_POLL_ADD] = {
 119                 .needs_file             = 1,
 120                 .unbound_nonreg_file    = 1,
 121                 .audit_skip             = 1,
 122                 .prep                   = io_poll_add_prep,
 123                 .issue                  = io_poll_add,
 124         },
 125         [IORING_OP_POLL_REMOVE] = {
 126                 .audit_skip             = 1,
 127                 .prep                   = io_poll_remove_prep,
 128                 .issue                  = io_poll_remove,
 129         },
 130         [IORING_OP_SYNC_FILE_RANGE] = {
 131                 .needs_file             = 1,
 132                 .audit_skip             = 1,
 133                 .prep                   = io_sfr_prep,
 134                 .issue                  = io_sync_file_range,
 135         },
 136         [IORING_OP_SENDMSG] = {
 137                 .needs_file             = 1,
 138                 .unbound_nonreg_file    = 1,
 139                 .pollout                = 1,
 140                 .ioprio                 = 1,
 141                 .manual_alloc           = 1,
 142 #if defined(CONFIG_NET)
 143                 .prep                   = io_sendmsg_prep,
 144                 .issue                  = io_sendmsg,
 145 #else
 146                 .prep                   = io_eopnotsupp_prep,
 147 #endif
 148         },
 149         [IORING_OP_RECVMSG] = {
 150                 .needs_file             = 1,
 151                 .unbound_nonreg_file    = 1,
 152                 .pollin                 = 1,
 153                 .buffer_select          = 1,
 154                 .ioprio                 = 1,
 155                 .manual_alloc           = 1,
 156 #if defined(CONFIG_NET)
 157                 .prep                   = io_recvmsg_prep,
 158                 .issue                  = io_recvmsg,
 159 #else
 160                 .prep                   = io_eopnotsupp_prep,
 161 #endif
 162         },
 163         [IORING_OP_TIMEOUT] = {
 164                 .audit_skip             = 1,
 165                 .prep                   = io_timeout_prep,
 166                 .issue                  = io_timeout,
 167         },
 168         [IORING_OP_TIMEOUT_REMOVE] = {
 169                 /* used by timeout updates' prep() */
 170                 .audit_skip             = 1,
 171                 .prep                   = io_timeout_remove_prep,
 172                 .issue                  = io_timeout_remove,
 173         },
 174         [IORING_OP_ACCEPT] = {
 175                 .needs_file             = 1,
 176                 .unbound_nonreg_file    = 1,
 177                 .pollin                 = 1,
 178                 .poll_exclusive         = 1,
 179                 .ioprio                 = 1,    /* used for flags */
 180 #if defined(CONFIG_NET)
 181                 .prep                   = io_accept_prep,
 182                 .issue                  = io_accept,
 183 #else
 184                 .prep                   = io_eopnotsupp_prep,
 185 #endif
 186         },
 187         [IORING_OP_ASYNC_CANCEL] = {
 188                 .audit_skip             = 1,
 189                 .prep                   = io_async_cancel_prep,
 190                 .issue                  = io_async_cancel,
 191         },
 192         [IORING_OP_LINK_TIMEOUT] = {
 193                 .audit_skip             = 1,
 194                 .prep                   = io_link_timeout_prep,
 195                 .issue                  = io_no_issue,
 196         },
 197         [IORING_OP_CONNECT] = {
 198                 .needs_file             = 1,
 199                 .unbound_nonreg_file    = 1,
 200                 .pollout                = 1,
 201 #if defined(CONFIG_NET)
 202                 .prep                   = io_connect_prep,
 203                 .issue                  = io_connect,
 204 #else
 205                 .prep                   = io_eopnotsupp_prep,
 206 #endif
 207         },
 208         [IORING_OP_FALLOCATE] = {
 209                 .needs_file             = 1,
 210                 .prep                   = io_fallocate_prep,
 211                 .issue                  = io_fallocate,
 212         },
 213         [IORING_OP_OPENAT] = {
 214                 .prep                   = io_openat_prep,
 215                 .issue                  = io_openat,
 216         },
 217         [IORING_OP_CLOSE] = {
 218                 .prep                   = io_close_prep,
 219                 .issue                  = io_close,
 220         },
 221         [IORING_OP_FILES_UPDATE] = {
 222                 .audit_skip             = 1,
 223                 .iopoll                 = 1,
 224                 .prep                   = io_files_update_prep,
 225                 .issue                  = io_files_update,
 226         },
 227         [IORING_OP_STATX] = {
 228                 .audit_skip             = 1,
 229                 .prep                   = io_statx_prep,
 230                 .issue                  = io_statx,
 231         },
 232         [IORING_OP_READ] = {
 233                 .needs_file             = 1,
 234                 .unbound_nonreg_file    = 1,
 235                 .pollin                 = 1,
 236                 .buffer_select          = 1,
 237                 .plug                   = 1,
 238                 .audit_skip             = 1,
 239                 .ioprio                 = 1,
 240                 .iopoll                 = 1,
 241                 .iopoll_queue           = 1,
 242                 .prep                   = io_prep_rw,
 243                 .issue                  = io_read,
 244         },
 245         [IORING_OP_WRITE] = {
 246                 .needs_file             = 1,
 247                 .hash_reg_file          = 1,
 248                 .unbound_nonreg_file    = 1,
 249                 .pollout                = 1,
 250                 .plug                   = 1,
 251                 .audit_skip             = 1,
 252                 .ioprio                 = 1,
 253                 .iopoll                 = 1,
 254                 .iopoll_queue           = 1,
 255                 .prep                   = io_prep_rw,
 256                 .issue                  = io_write,
 257         },
 258         [IORING_OP_FADVISE] = {
 259                 .needs_file             = 1,
 260                 .audit_skip             = 1,
 261                 .prep                   = io_fadvise_prep,
 262                 .issue                  = io_fadvise,
 263         },
 264         [IORING_OP_MADVISE] = {
 265                 .audit_skip             = 1,
 266                 .prep                   = io_madvise_prep,
 267                 .issue                  = io_madvise,
 268         },
 269         [IORING_OP_SEND] = {
 270                 .needs_file             = 1,
 271                 .unbound_nonreg_file    = 1,
 272                 .pollout                = 1,
 273                 .audit_skip             = 1,
 274                 .ioprio                 = 1,
 275                 .manual_alloc           = 1,
 276 #if defined(CONFIG_NET)
 277                 .prep                   = io_sendmsg_prep,
 278                 .issue                  = io_send,
 279 #else
 280                 .prep                   = io_eopnotsupp_prep,
 281 #endif
 282         },
 283         [IORING_OP_RECV] = {
 284                 .needs_file             = 1,
 285                 .unbound_nonreg_file    = 1,
 286                 .pollin                 = 1,
 287                 .buffer_select          = 1,
 288                 .audit_skip             = 1,
 289                 .ioprio                 = 1,
 290 #if defined(CONFIG_NET)
 291                 .prep                   = io_recvmsg_prep,
 292                 .issue                  = io_recv,
 293 #else
 294                 .prep                   = io_eopnotsupp_prep,
 295 #endif
 296         },
 297         [IORING_OP_OPENAT2] = {
 298                 .prep                   = io_openat2_prep,
 299                 .issue                  = io_openat2,
 300         },
 301         [IORING_OP_EPOLL_CTL] = {
 302                 .unbound_nonreg_file    = 1,
 303                 .audit_skip             = 1,
 304 #if defined(CONFIG_EPOLL)
 305                 .prep                   = io_epoll_ctl_prep,
 306                 .issue                  = io_epoll_ctl,
 307 #else
 308                 .prep                   = io_eopnotsupp_prep,
 309 #endif
 310         },
 311         [IORING_OP_SPLICE] = {
 312                 .needs_file             = 1,
 313                 .hash_reg_file          = 1,
 314                 .unbound_nonreg_file    = 1,
 315                 .audit_skip             = 1,
 316                 .prep                   = io_splice_prep,
 317                 .issue                  = io_splice,
 318         },
 319         [IORING_OP_PROVIDE_BUFFERS] = {
 320                 .audit_skip             = 1,
 321                 .iopoll                 = 1,
 322                 .prep                   = io_provide_buffers_prep,
 323                 .issue                  = io_provide_buffers,
 324         },
 325         [IORING_OP_REMOVE_BUFFERS] = {
 326                 .audit_skip             = 1,
 327                 .iopoll                 = 1,
 328                 .prep                   = io_remove_buffers_prep,
 329                 .issue                  = io_remove_buffers,
 330         },
 331         [IORING_OP_TEE] = {
 332                 .needs_file             = 1,
 333                 .hash_reg_file          = 1,
 334                 .unbound_nonreg_file    = 1,
 335                 .audit_skip             = 1,
 336                 .prep                   = io_tee_prep,
 337                 .issue                  = io_tee,
 338         },
 339         [IORING_OP_SHUTDOWN] = {
 340                 .needs_file             = 1,
 341 #if defined(CONFIG_NET)
 342                 .prep                   = io_shutdown_prep,
 343                 .issue                  = io_shutdown,
 344 #else
 345                 .prep                   = io_eopnotsupp_prep,
 346 #endif
 347         },
 348         [IORING_OP_RENAMEAT] = {
 349                 .prep                   = io_renameat_prep,
 350                 .issue                  = io_renameat,
 351         },
 352         [IORING_OP_UNLINKAT] = {
 353                 .prep                   = io_unlinkat_prep,
 354                 .issue                  = io_unlinkat,
 355         },
 356         [IORING_OP_MKDIRAT] = {
 357                 .prep                   = io_mkdirat_prep,
 358                 .issue                  = io_mkdirat,
 359         },
 360         [IORING_OP_SYMLINKAT] = {
 361                 .prep                   = io_symlinkat_prep,
 362                 .issue                  = io_symlinkat,
 363         },
 364         [IORING_OP_LINKAT] = {
 365                 .prep                   = io_linkat_prep,
 366                 .issue                  = io_linkat,
 367         },
 368         [IORING_OP_MSG_RING] = {
 369                 .needs_file             = 1,
 370                 .iopoll                 = 1,
 371                 .prep                   = io_msg_ring_prep,
 372                 .issue                  = io_msg_ring,
 373         },
 374         [IORING_OP_FSETXATTR] = {
 375                 .needs_file = 1,
 376                 .prep                   = io_fsetxattr_prep,
 377                 .issue                  = io_fsetxattr,
 378         },
 379         [IORING_OP_SETXATTR] = {
 380                 .prep                   = io_setxattr_prep,
 381                 .issue                  = io_setxattr,
 382         },
 383         [IORING_OP_FGETXATTR] = {
 384                 .needs_file = 1,
 385                 .prep                   = io_fgetxattr_prep,
 386                 .issue                  = io_fgetxattr,
 387         },
 388         [IORING_OP_GETXATTR] = {
 389                 .prep                   = io_getxattr_prep,
 390                 .issue                  = io_getxattr,
 391         },
 392         [IORING_OP_SOCKET] = {
 393                 .audit_skip             = 1,
 394 #if defined(CONFIG_NET)
 395                 .prep                   = io_socket_prep,
 396                 .issue                  = io_socket,
 397 #else
 398                 .prep                   = io_eopnotsupp_prep,
 399 #endif
 400         },
 401         [IORING_OP_URING_CMD] = {
 402                 .needs_file             = 1,
 403                 .plug                   = 1,
 404                 .iopoll                 = 1,
 405                 .iopoll_queue           = 1,
 406                 .prep                   = io_uring_cmd_prep,
 407                 .issue                  = io_uring_cmd,
 408         },
 409         [IORING_OP_SEND_ZC] = {
 410                 .needs_file             = 1,
 411                 .unbound_nonreg_file    = 1,
 412                 .pollout                = 1,
 413                 .audit_skip             = 1,
 414                 .ioprio                 = 1,
 415                 .manual_alloc           = 1,
 416 #if defined(CONFIG_NET)
 417                 .prep                   = io_send_zc_prep,
 418                 .issue                  = io_send_zc,
 419 #else
 420                 .prep                   = io_eopnotsupp_prep,
 421 #endif
 422         },
 423         [IORING_OP_SENDMSG_ZC] = {
 424                 .needs_file             = 1,
 425                 .unbound_nonreg_file    = 1,
 426                 .pollout                = 1,
 427                 .ioprio                 = 1,
 428                 .manual_alloc           = 1,
 429 #if defined(CONFIG_NET)
 430                 .prep                   = io_send_zc_prep,
 431                 .issue                  = io_sendmsg_zc,
 432 #else
 433                 .prep                   = io_eopnotsupp_prep,
 434 #endif
 435         },
 436         [IORING_OP_READ_MULTISHOT] = {
 437                 .needs_file             = 1,
 438                 .unbound_nonreg_file    = 1,
 439                 .pollin                 = 1,
 440                 .buffer_select          = 1,
 441                 .audit_skip             = 1,
 442                 .prep                   = io_read_mshot_prep,
 443                 .issue                  = io_read_mshot,
 444         },
 445         [IORING_OP_WAITID] = {
 446                 .prep                   = io_waitid_prep,
 447                 .issue                  = io_waitid,
 448         },
 449         [IORING_OP_FUTEX_WAIT] = {
 450 #if defined(CONFIG_FUTEX)
 451                 .prep                   = io_futex_prep,
 452                 .issue                  = io_futex_wait,
 453 #else
 454                 .prep                   = io_eopnotsupp_prep,
 455 #endif
 456         },
 457         [IORING_OP_FUTEX_WAKE] = {
 458 #if defined(CONFIG_FUTEX)
 459                 .prep                   = io_futex_prep,
 460                 .issue                  = io_futex_wake,
 461 #else
 462                 .prep                   = io_eopnotsupp_prep,
 463 #endif
 464         },
 465         [IORING_OP_FUTEX_WAITV] = {
 466 #if defined(CONFIG_FUTEX)
 467                 .prep                   = io_futexv_prep,
 468                 .issue                  = io_futexv_wait,
 469 #else
 470                 .prep                   = io_eopnotsupp_prep,
 471 #endif
 472         },
 473         [IORING_OP_FIXED_FD_INSTALL] = {
 474                 .needs_file             = 1,
 475                 .prep                   = io_install_fixed_fd_prep,
 476                 .issue                  = io_install_fixed_fd,
 477         },
 478         [IORING_OP_FTRUNCATE] = {
 479                 .needs_file             = 1,
 480                 .hash_reg_file          = 1,
 481                 .prep                   = io_ftruncate_prep,
 482                 .issue                  = io_ftruncate,
 483         },
 484 };
 485
 486 const struct io_cold_def io_cold_defs[] = {
 487         [IORING_OP_NOP] = {
 488                 .name                   = "NOP",
 489         },
 490         [IORING_OP_READV] = {
 491                 .async_size             = sizeof(struct io_async_rw),
 492                 .name                   = "READV",
 493                 .prep_async             = io_readv_prep_async,
 494                 .cleanup                = io_readv_writev_cleanup,
 495                 .fail                   = io_rw_fail,
 496         },
 497         [IORING_OP_WRITEV] = {
 498                 .async_size             = sizeof(struct io_async_rw),
 499                 .name                   = "WRITEV",
 500                 .prep_async             = io_writev_prep_async,
 501                 .cleanup                = io_readv_writev_cleanup,
 502                 .fail                   = io_rw_fail,
 503         },
 504         [IORING_OP_FSYNC] = {
 505                 .name                   = "FSYNC",
 506         },
 507         [IORING_OP_READ_FIXED] = {
 508                 .async_size             = sizeof(struct io_async_rw),
 509                 .name                   = "READ_FIXED",
 510                 .fail                   = io_rw_fail,
 511         },
 512         [IORING_OP_WRITE_FIXED] = {
 513                 .async_size             = sizeof(struct io_async_rw),
 514                 .name                   = "WRITE_FIXED",
 515                 .fail                   = io_rw_fail,
 516         },
 517         [IORING_OP_POLL_ADD] = {
 518                 .name                   = "POLL_ADD",
 519         },
 520         [IORING_OP_POLL_REMOVE] = {
 521                 .name                   = "POLL_REMOVE",
 522         },
 523         [IORING_OP_SYNC_FILE_RANGE] = {
 524                 .name                   = "SYNC_FILE_RANGE",
 525         },
 526         [IORING_OP_SENDMSG] = {
 527                 .name                   = "SENDMSG",
 528 #if defined(CONFIG_NET)
 529                 .async_size             = sizeof(struct io_async_msghdr),
 530                 .prep_async             = io_sendmsg_prep_async,
 531                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 532                 .fail                   = io_sendrecv_fail,
 533 #endif
 534         },
 535         [IORING_OP_RECVMSG] = {
 536                 .name                   = "RECVMSG",
 537 #if defined(CONFIG_NET)
 538                 .async_size             = sizeof(struct io_async_msghdr),
 539                 .prep_async             = io_recvmsg_prep_async,
 540                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 541                 .fail                   = io_sendrecv_fail,
 542 #endif
 543         },
 544         [IORING_OP_TIMEOUT] = {
 545                 .async_size             = sizeof(struct io_timeout_data),
 546                 .name                   = "TIMEOUT",
 547         },
 548         [IORING_OP_TIMEOUT_REMOVE] = {
 549                 .name                   = "TIMEOUT_REMOVE",
 550         },
 551         [IORING_OP_ACCEPT] = {
 552                 .name                   = "ACCEPT",
 553         },
 554         [IORING_OP_ASYNC_CANCEL] = {
 555                 .name                   = "ASYNC_CANCEL",
 556         },
 557         [IORING_OP_LINK_TIMEOUT] = {
 558                 .async_size             = sizeof(struct io_timeout_data),
 559                 .name                   = "LINK_TIMEOUT",
 560         },
 561         [IORING_OP_CONNECT] = {
 562                 .name                   = "CONNECT",
 563 #if defined(CONFIG_NET)
 564                 .async_size             = sizeof(struct io_async_connect),
 565                 .prep_async             = io_connect_prep_async,
 566 #endif
 567         },
 568         [IORING_OP_FALLOCATE] = {
 569                 .name                   = "FALLOCATE",
 570         },
 571         [IORING_OP_OPENAT] = {
 572                 .name                   = "OPENAT",
 573                 .cleanup                = io_open_cleanup,
 574         },
 575         [IORING_OP_CLOSE] = {
 576                 .name                   = "CLOSE",
 577         },
 578         [IORING_OP_FILES_UPDATE] = {
 579                 .name                   = "FILES_UPDATE",
 580         },
 581         [IORING_OP_STATX] = {
 582                 .name                   = "STATX",
 583                 .cleanup                = io_statx_cleanup,
 584         },
 585         [IORING_OP_READ] = {
 586                 .async_size             = sizeof(struct io_async_rw),
 587                 .name                   = "READ",
 588                 .fail                   = io_rw_fail,
 589         },
 590         [IORING_OP_WRITE] = {
 591                 .async_size             = sizeof(struct io_async_rw),
 592                 .name                   = "WRITE",
 593                 .fail                   = io_rw_fail,
 594         },
 595         [IORING_OP_FADVISE] = {
 596                 .name                   = "FADVISE",
 597         },
 598         [IORING_OP_MADVISE] = {
 599                 .name                   = "MADVISE",
 600         },
 601         [IORING_OP_SEND] = {
 602                 .name                   = "SEND",
 603 #if defined(CONFIG_NET)
 604                 .async_size             = sizeof(struct io_async_msghdr),
 605                 .fail                   = io_sendrecv_fail,
 606                 .prep_async             = io_send_prep_async,
 607 #endif
 608         },
 609         [IORING_OP_RECV] = {
 610                 .name                   = "RECV",
 611 #if defined(CONFIG_NET)
 612                 .fail                   = io_sendrecv_fail,
 613 #endif
 614         },
 615         [IORING_OP_OPENAT2] = {
 616                 .name                   = "OPENAT2",
 617                 .cleanup                = io_open_cleanup,
 618         },
 619         [IORING_OP_EPOLL_CTL] = {
 620                 .name                   = "EPOLL",
 621         },
 622         [IORING_OP_SPLICE] = {
 623                 .name                   = "SPLICE",
 624         },
 625         [IORING_OP_PROVIDE_BUFFERS] = {
 626                 .name                   = "PROVIDE_BUFFERS",
 627         },
 628         [IORING_OP_REMOVE_BUFFERS] = {
 629                 .name                   = "REMOVE_BUFFERS",
 630         },
 631         [IORING_OP_TEE] = {
 632                 .name                   = "TEE",
 633         },
 634         [IORING_OP_SHUTDOWN] = {
 635                 .name                   = "SHUTDOWN",
 636         },
 637         [IORING_OP_RENAMEAT] = {
 638                 .name                   = "RENAMEAT",
 639                 .cleanup                = io_renameat_cleanup,
 640         },
 641         [IORING_OP_UNLINKAT] = {
 642                 .name                   = "UNLINKAT",
 643                 .cleanup                = io_unlinkat_cleanup,
 644         },
 645         [IORING_OP_MKDIRAT] = {
 646                 .name                   = "MKDIRAT",
 647                 .cleanup                = io_mkdirat_cleanup,
 648         },
 649         [IORING_OP_SYMLINKAT] = {
 650                 .name                   = "SYMLINKAT",
 651                 .cleanup                = io_link_cleanup,
 652         },
 653         [IORING_OP_LINKAT] = {
 654                 .name                   = "LINKAT",
 655                 .cleanup                = io_link_cleanup,
 656         },
 657         [IORING_OP_MSG_RING] = {
 658                 .name                   = "MSG_RING",
 659                 .cleanup                = io_msg_ring_cleanup,
 660         },
 661         [IORING_OP_FSETXATTR] = {
 662                 .name                   = "FSETXATTR",
 663                 .cleanup                = io_xattr_cleanup,
 664         },
 665         [IORING_OP_SETXATTR] = {
 666                 .name                   = "SETXATTR",
 667                 .cleanup                = io_xattr_cleanup,
 668         },
 669         [IORING_OP_FGETXATTR] = {
 670                 .name                   = "FGETXATTR",
 671                 .cleanup                = io_xattr_cleanup,
 672         },
 673         [IORING_OP_GETXATTR] = {
 674                 .name                   = "GETXATTR",
 675                 .cleanup                = io_xattr_cleanup,
 676         },
 677         [IORING_OP_SOCKET] = {
 678                 .name                   = "SOCKET",
 679         },
 680         [IORING_OP_URING_CMD] = {
 681                 .name                   = "URING_CMD",
 682                 .async_size             = 2 * sizeof(struct io_uring_sqe),
 683                 .prep_async             = io_uring_cmd_prep_async,
 684         },
 685         [IORING_OP_SEND_ZC] = {
 686                 .name                   = "SEND_ZC",
 687 #if defined(CONFIG_NET)
 688                 .async_size             = sizeof(struct io_async_msghdr),
 689                 .prep_async             = io_send_prep_async,
 690                 .cleanup                = io_send_zc_cleanup,
 691                 .fail                   = io_sendrecv_fail,
 692 #endif
 693         },
 694         [IORING_OP_SENDMSG_ZC] = {
 695                 .name                   = "SENDMSG_ZC",
 696 #if defined(CONFIG_NET)
 697                 .async_size             = sizeof(struct io_async_msghdr),
 698                 .prep_async             = io_sendmsg_prep_async,
 699                 .cleanup                = io_send_zc_cleanup,
 700                 .fail                   = io_sendrecv_fail,
 701 #endif
 702         },
 703         [IORING_OP_READ_MULTISHOT] = {
 704                 .name                   = "READ_MULTISHOT",
 705         },
 706         [IORING_OP_WAITID] = {
 707                 .name                   = "WAITID",
 708                 .async_size             = sizeof(struct io_waitid_async),
 709         },
 710         [IORING_OP_FUTEX_WAIT] = {
 711                 .name                   = "FUTEX_WAIT",
 712         },
 713         [IORING_OP_FUTEX_WAKE] = {
 714                 .name                   = "FUTEX_WAKE",
 715         },
 716         [IORING_OP_FUTEX_WAITV] = {
 717                 .name                   = "FUTEX_WAITV",
 718         },
 719         [IORING_OP_FIXED_FD_INSTALL] = {
 720                 .name                   = "FIXED_FD_INSTALL",
 721         },
 722         [IORING_OP_FTRUNCATE] = {
 723                 .name                   = "FTRUNCATE",
 724         },
 725 };
 726
 727 const char *io_uring_get_opcode(u8 opcode)
 728 {
 729         if (opcode < IORING_OP_LAST)
 730                 return io_cold_defs[opcode].name;
 731         return "INVALID";
 732 }
 733
 734 void __init io_uring_optable_init(void)
 735 {
 736         int i;
 737
 738         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 739         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 740
 741         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 742                 BUG_ON(!io_issue_defs[i].prep);
 743                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 744                         BUG_ON(!io_issue_defs[i].issue);
 745                 WARN_ON_ONCE(!io_cold_defs[i].name);
 746         }
 747 }