drivers/net/vmxnet3/vmxnet3_xdp.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Linux driver for VMware's vmxnet3 ethernet NIC.
   4  * Copyright (C) 2008-2023, VMware, Inc. All Rights Reserved.
   5  * Maintained by: pv-drivers@vmware.com
   6  *
   7  */
   8
   9 #include "vmxnet3_int.h"
  10 #include "vmxnet3_xdp.h"
  11
  12 static void
  13 vmxnet3_xdp_exchange_program(struct vmxnet3_adapter *adapter,
  14                              struct bpf_prog *prog)
  15 {
  16         rcu_assign_pointer(adapter->xdp_bpf_prog, prog);
  17 }
  18
  19 static inline struct vmxnet3_tx_queue *
  20 vmxnet3_xdp_get_tq(struct vmxnet3_adapter *adapter)
  21 {
  22         struct vmxnet3_tx_queue *tq;
  23         int tq_number;
  24         int cpu;
  25
  26         tq_number = adapter->num_tx_queues;
  27         cpu = smp_processor_id();
  28         if (likely(cpu < tq_number))
  29                 tq = &adapter->tx_queue[cpu];
  30         else
  31                 tq = &adapter->tx_queue[reciprocal_scale(cpu, tq_number)];
  32
  33         return tq;
  34 }
  35
  36 static int
  37 vmxnet3_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf,
  38                 struct netlink_ext_ack *extack)
  39 {
  40         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
  41         struct bpf_prog *new_bpf_prog = bpf->prog;
  42         struct bpf_prog *old_bpf_prog;
  43         bool need_update;
  44         bool running;
  45         int err;
  46
  47         if (new_bpf_prog && netdev->mtu > VMXNET3_XDP_MAX_MTU) {
  48                 NL_SET_ERR_MSG_FMT_MOD(extack, "MTU %u too large for XDP",
  49                                        netdev->mtu);
  50                 return -EOPNOTSUPP;
  51         }
  52
  53         if (adapter->netdev->features & NETIF_F_LRO) {
  54                 NL_SET_ERR_MSG_MOD(extack, "LRO is not supported with XDP");
  55                 adapter->netdev->features &= ~NETIF_F_LRO;
  56         }
  57
  58         old_bpf_prog = rcu_dereference(adapter->xdp_bpf_prog);
  59         if (!new_bpf_prog && !old_bpf_prog)
  60                 return 0;
  61
  62         running = netif_running(netdev);
  63         need_update = !!old_bpf_prog != !!new_bpf_prog;
  64
  65         if (running && need_update)
  66                 vmxnet3_quiesce_dev(adapter);
  67
  68         vmxnet3_xdp_exchange_program(adapter, new_bpf_prog);
  69         if (old_bpf_prog)
  70                 bpf_prog_put(old_bpf_prog);
  71
  72         if (!running || !need_update)
  73                 return 0;
  74
  75         if (new_bpf_prog)
  76                 xdp_features_set_redirect_target(netdev, false);
  77         else
  78                 xdp_features_clear_redirect_target(netdev);
  79
  80         vmxnet3_reset_dev(adapter);
  81         vmxnet3_rq_destroy_all(adapter);
  82         vmxnet3_adjust_rx_ring_size(adapter);
  83         err = vmxnet3_rq_create_all(adapter);
  84         if (err) {
  85                 NL_SET_ERR_MSG_MOD(extack,
  86                                    "failed to re-create rx queues for XDP.");
  87                 return -EOPNOTSUPP;
  88         }
  89         err = vmxnet3_activate_dev(adapter);
  90         if (err) {
  91                 NL_SET_ERR_MSG_MOD(extack,
  92                                    "failed to activate device for XDP.");
  93                 return -EOPNOTSUPP;
  94         }
  95         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
  96
  97         return 0;
  98 }
  99
 100 /* This is the main xdp call used by kernel to set/unset eBPF program. */
 101 int
 102 vmxnet3_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
 103 {
 104         switch (bpf->command) {
 105         case XDP_SETUP_PROG:
 106                 return vmxnet3_xdp_set(netdev, bpf, bpf->extack);
 107         default:
 108                 return -EINVAL;
 109         }
 110
 111         return 0;
 112 }
 113
 114 static int
 115 vmxnet3_xdp_xmit_frame(struct vmxnet3_adapter *adapter,
 116                        struct xdp_frame *xdpf,
 117                        struct vmxnet3_tx_queue *tq, bool dma_map)
 118 {
 119         struct vmxnet3_tx_buf_info *tbi = NULL;
 120         union Vmxnet3_GenericDesc *gdesc;
 121         struct vmxnet3_tx_ctx ctx;
 122         int tx_num_deferred;
 123         struct page *page;
 124         u32 buf_size;
 125         u32 dw2;
 126
 127         dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 128         dw2 |= xdpf->len;
 129         ctx.sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
 130         gdesc = ctx.sop_txd;
 131
 132         buf_size = xdpf->len;
 133         tbi = tq->buf_info + tq->tx_ring.next2fill;
 134
 135         if (vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) == 0) {
 136                 tq->stats.tx_ring_full++;
 137                 return -ENOSPC;
 138         }
 139
 140         tbi->map_type = VMXNET3_MAP_XDP;
 141         if (dma_map) { /* ndo_xdp_xmit */
 142                 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
 143                                                xdpf->data, buf_size,
 144                                                DMA_TO_DEVICE);
 145                 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
 146                         return -EFAULT;
 147                 tbi->map_type |= VMXNET3_MAP_SINGLE;
 148         } else { /* XDP buffer from page pool */
 149                 page = virt_to_page(xdpf->data);
 150                 tbi->dma_addr = page_pool_get_dma_addr(page) +
 151                                 VMXNET3_XDP_HEADROOM;
 152                 dma_sync_single_for_device(&adapter->pdev->dev,
 153                                            tbi->dma_addr, buf_size,
 154                                            DMA_TO_DEVICE);
 155         }
 156         tbi->xdpf = xdpf;
 157         tbi->len = buf_size;
 158
 159         gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 160         WARN_ON_ONCE(gdesc->txd.gen == tq->tx_ring.gen);
 161
 162         gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
 163         gdesc->dword[2] = cpu_to_le32(dw2);
 164
 165         /* Setup the EOP desc */
 166         gdesc->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
 167
 168         gdesc->txd.om = 0;
 169         gdesc->txd.msscof = 0;
 170         gdesc->txd.hlen = 0;
 171         gdesc->txd.ti = 0;
 172
 173         tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
 174         le32_add_cpu(&tq->shared->txNumDeferred, 1);
 175         tx_num_deferred++;
 176
 177         vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 178
 179         /* set the last buf_info for the pkt */
 180         tbi->sop_idx = ctx.sop_txd - tq->tx_ring.base;
 181
 182         dma_wmb();
 183         gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
 184                                                   VMXNET3_TXD_GEN);
 185
 186         /* No need to handle the case when tx_num_deferred doesn't reach
 187          * threshold. Backend driver at hypervisor side will poll and reset
 188          * tq->shared->txNumDeferred to 0.
 189          */
 190         if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
 191                 tq->shared->txNumDeferred = 0;
 192                 VMXNET3_WRITE_BAR0_REG(adapter,
 193                                        VMXNET3_REG_TXPROD + tq->qid * 8,
 194                                        tq->tx_ring.next2fill);
 195         }
 196
 197         return 0;
 198 }
 199
 200 static int
 201 vmxnet3_xdp_xmit_back(struct vmxnet3_adapter *adapter,
 202                       struct xdp_frame *xdpf)
 203 {
 204         struct vmxnet3_tx_queue *tq;
 205         struct netdev_queue *nq;
 206         int err;
 207
 208         tq = vmxnet3_xdp_get_tq(adapter);
 209         if (tq->stopped)
 210                 return -ENETDOWN;
 211
 212         nq = netdev_get_tx_queue(adapter->netdev, tq->qid);
 213
 214         __netif_tx_lock(nq, smp_processor_id());
 215         err = vmxnet3_xdp_xmit_frame(adapter, xdpf, tq, false);
 216         __netif_tx_unlock(nq);
 217
 218         return err;
 219 }
 220
 221 /* ndo_xdp_xmit */
 222 int
 223 vmxnet3_xdp_xmit(struct net_device *dev,
 224                  int n, struct xdp_frame **frames, u32 flags)
 225 {
 226         struct vmxnet3_adapter *adapter = netdev_priv(dev);
 227         struct vmxnet3_tx_queue *tq;
 228         int i;
 229
 230         if (unlikely(test_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)))
 231                 return -ENETDOWN;
 232         if (unlikely(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)))
 233                 return -EINVAL;
 234
 235         tq = vmxnet3_xdp_get_tq(adapter);
 236         if (tq->stopped)
 237                 return -ENETDOWN;
 238
 239         for (i = 0; i < n; i++) {
 240                 if (vmxnet3_xdp_xmit_frame(adapter, frames[i], tq, true)) {
 241                         tq->stats.xdp_xmit_err++;
 242                         break;
 243                 }
 244         }
 245         tq->stats.xdp_xmit += i;
 246
 247         return i;
 248 }
 249
 250 static int
 251 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp,
 252                 struct bpf_prog *prog)
 253 {
 254         struct xdp_frame *xdpf;
 255         struct page *page;
 256         int err;
 257         u32 act;
 258
 259         rq->stats.xdp_packets++;
 260         act = bpf_prog_run_xdp(prog, xdp);
 261         page = virt_to_page(xdp->data_hard_start);
 262
 263         switch (act) {
 264         case XDP_PASS:
 265                 return act;
 266         case XDP_REDIRECT:
 267                 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog);
 268                 if (!err) {
 269                         rq->stats.xdp_redirects++;
 270                 } else {
 271                         rq->stats.xdp_drops++;
 272                         page_pool_recycle_direct(rq->page_pool, page);
 273                 }
 274                 return act;
 275         case XDP_TX:
 276                 xdpf = xdp_convert_buff_to_frame(xdp);
 277                 if (unlikely(!xdpf ||
 278                              vmxnet3_xdp_xmit_back(rq->adapter, xdpf))) {
 279                         rq->stats.xdp_drops++;
 280                         page_pool_recycle_direct(rq->page_pool, page);
 281                 } else {
 282                         rq->stats.xdp_tx++;
 283                 }
 284                 return act;
 285         default:
 286                 bpf_warn_invalid_xdp_action(rq->adapter->netdev, prog, act);
 287                 fallthrough;
 288         case XDP_ABORTED:
 289                 trace_xdp_exception(rq->adapter->netdev, prog, act);
 290                 rq->stats.xdp_aborted++;
 291                 break;
 292         case XDP_DROP:
 293                 rq->stats.xdp_drops++;
 294                 break;
 295         }
 296
 297         page_pool_recycle_direct(rq->page_pool, page);
 298
 299         return act;
 300 }
 301
 302 static struct sk_buff *
 303 vmxnet3_build_skb(struct vmxnet3_rx_queue *rq, struct page *page,
 304                   const struct xdp_buff *xdp)
 305 {
 306         struct sk_buff *skb;
 307
 308         skb = build_skb(page_address(page), PAGE_SIZE);
 309         if (unlikely(!skb)) {
 310                 page_pool_recycle_direct(rq->page_pool, page);
 311                 rq->stats.rx_buf_alloc_failure++;
 312                 return NULL;
 313         }
 314
 315         /* bpf prog might change len and data position. */
 316         skb_reserve(skb, xdp->data - xdp->data_hard_start);
 317         skb_put(skb, xdp->data_end - xdp->data);
 318         skb_mark_for_recycle(skb);
 319
 320         return skb;
 321 }
 322
 323 /* Handle packets from DataRing. */
 324 int
 325 vmxnet3_process_xdp_small(struct vmxnet3_adapter *adapter,
 326                           struct vmxnet3_rx_queue *rq,
 327                           void *data, int len,
 328                           struct sk_buff **skb_xdp_pass)
 329 {
 330         struct bpf_prog *xdp_prog;
 331         struct xdp_buff xdp;
 332         struct page *page;
 333         int act;
 334
 335         page = page_pool_alloc_pages(rq->page_pool, GFP_ATOMIC);
 336         if (unlikely(!page)) {
 337                 rq->stats.rx_buf_alloc_failure++;
 338                 return XDP_DROP;
 339         }
 340
 341         xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq);
 342         xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset,
 343                          len, false);
 344         xdp_buff_clear_frags_flag(&xdp);
 345
 346         /* Must copy the data because it's at dataring. */
 347         memcpy(xdp.data, data, len);
 348
 349         xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog);
 350         if (!xdp_prog) {
 351                 act = XDP_PASS;
 352                 goto out_skb;
 353         }
 354         act = vmxnet3_run_xdp(rq, &xdp, xdp_prog);
 355         if (act != XDP_PASS)
 356                 return act;
 357
 358 out_skb:
 359         *skb_xdp_pass = vmxnet3_build_skb(rq, page, &xdp);
 360         if (!*skb_xdp_pass)
 361                 return XDP_DROP;
 362
 363         /* No need to refill. */
 364         return likely(*skb_xdp_pass) ? act : XDP_DROP;
 365 }
 366
 367 int
 368 vmxnet3_process_xdp(struct vmxnet3_adapter *adapter,
 369                     struct vmxnet3_rx_queue *rq,
 370                     struct Vmxnet3_RxCompDesc *rcd,
 371                     struct vmxnet3_rx_buf_info *rbi,
 372                     struct Vmxnet3_RxDesc *rxd,
 373                     struct sk_buff **skb_xdp_pass)
 374 {
 375         struct bpf_prog *xdp_prog;
 376         dma_addr_t new_dma_addr;
 377         struct xdp_buff xdp;
 378         struct page *page;
 379         void *new_data;
 380         int act;
 381
 382         page = rbi->page;
 383         dma_sync_single_for_cpu(&adapter->pdev->dev,
 384                                 page_pool_get_dma_addr(page) +
 385                                 rq->page_pool->p.offset, rbi->len,
 386                                 page_pool_get_dma_dir(rq->page_pool));
 387
 388         xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq);
 389         xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset,
 390                          rbi->len, false);
 391         xdp_buff_clear_frags_flag(&xdp);
 392
 393         xdp_prog = rcu_dereference(rq->adapter->xdp_bpf_prog);
 394         if (!xdp_prog) {
 395                 act = XDP_PASS;
 396                 goto out_skb;
 397         }
 398         act = vmxnet3_run_xdp(rq, &xdp, xdp_prog);
 399
 400         if (act == XDP_PASS) {
 401 out_skb:
 402                 *skb_xdp_pass = vmxnet3_build_skb(rq, page, &xdp);
 403                 if (!*skb_xdp_pass)
 404                         act = XDP_DROP;
 405         }
 406
 407         new_data = vmxnet3_pp_get_buff(rq->page_pool, &new_dma_addr,
 408                                        GFP_ATOMIC);
 409         if (!new_data) {
 410                 rq->stats.rx_buf_alloc_failure++;
 411                 return XDP_DROP;
 412         }
 413         rbi->page = virt_to_page(new_data);
 414         rbi->dma_addr = new_dma_addr;
 415         rxd->addr = cpu_to_le64(rbi->dma_addr);
 416         rxd->len = rbi->len;
 417
 418         return act;
 419 }