drivers/net/ethernet/pensando/ionic/ionic_txrx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip6_checksum.h>
   8 #include <net/netdev_queues.h>
   9
  10 #include "ionic.h"
  11 #include "ionic_lif.h"
  12 #include "ionic_txrx.h"
  13
  14 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
  15                                       void *data, size_t len);
  16
  17 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
  18                                     const skb_frag_t *frag,
  19                                     size_t offset, size_t len);
  20
  21 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
  22                                      struct ionic_tx_desc_info *desc_info);
  23
  24 static void ionic_tx_clean(struct ionic_queue *q,
  25                            struct ionic_tx_desc_info *desc_info,
  26                            struct ionic_txq_comp *comp);
  27
  28 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell)
  29 {
  30         ionic_q_post(q, ring_dbell);
  31 }
  32
  33 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell)
  34 {
  35         ionic_q_post(q, ring_dbell);
  36 }
  37
  38 bool ionic_txq_poke_doorbell(struct ionic_queue *q)
  39 {
  40         struct netdev_queue *netdev_txq;
  41         unsigned long now, then, dif;
  42         struct net_device *netdev;
  43
  44         netdev = q->lif->netdev;
  45         netdev_txq = netdev_get_tx_queue(netdev, q->index);
  46
  47         HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
  48
  49         if (q->tail_idx == q->head_idx) {
  50                 HARD_TX_UNLOCK(netdev, netdev_txq);
  51                 return false;
  52         }
  53
  54         now = READ_ONCE(jiffies);
  55         then = q->dbell_jiffies;
  56         dif = now - then;
  57
  58         if (dif > q->dbell_deadline) {
  59                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  60                                  q->dbval | q->head_idx);
  61
  62                 q->dbell_jiffies = now;
  63         }
  64
  65         HARD_TX_UNLOCK(netdev, netdev_txq);
  66
  67         return true;
  68 }
  69
  70 bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
  71 {
  72         unsigned long now, then, dif;
  73
  74         /* no lock, called from rx napi or txrx napi, nothing else can fill */
  75
  76         if (q->tail_idx == q->head_idx)
  77                 return false;
  78
  79         now = READ_ONCE(jiffies);
  80         then = q->dbell_jiffies;
  81         dif = now - then;
  82
  83         if (dif > q->dbell_deadline) {
  84                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  85                                  q->dbval | q->head_idx);
  86
  87                 q->dbell_jiffies = now;
  88
  89                 dif = 2 * q->dbell_deadline;
  90                 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
  91                         dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
  92
  93                 q->dbell_deadline = dif;
  94         }
  95
  96         return true;
  97 }
  98
  99 static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q)
 100 {
 101         if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1)))
 102                 return q->txq_sgl_v1[q->head_idx].elems;
 103         else
 104                 return q->txq_sgl[q->head_idx].elems;
 105 }
 106
 107 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev,
 108                                             struct ionic_queue *q)
 109 {
 110         return netdev_get_tx_queue(netdev, q->index);
 111 }
 112
 113 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
 114 {
 115         return page_address(buf_info->page) + buf_info->page_offset;
 116 }
 117
 118 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
 119 {
 120         return buf_info->dma_addr + buf_info->page_offset;
 121 }
 122
 123 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info)
 124 {
 125         return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset);
 126 }
 127
 128 static int ionic_rx_page_alloc(struct ionic_queue *q,
 129                                struct ionic_buf_info *buf_info)
 130 {
 131         struct ionic_rx_stats *stats;
 132         struct device *dev;
 133         struct page *page;
 134
 135         dev = q->dev;
 136         stats = q_to_rx_stats(q);
 137
 138         if (unlikely(!buf_info)) {
 139                 net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
 140                                     dev_name(dev), q->name);
 141                 return -EINVAL;
 142         }
 143
 144         page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
 145         if (unlikely(!page)) {
 146                 net_err_ratelimited("%s: %s page alloc failed\n",
 147                                     dev_name(dev), q->name);
 148                 stats->alloc_err++;
 149                 return -ENOMEM;
 150         }
 151
 152         buf_info->dma_addr = dma_map_page(dev, page, 0,
 153                                           IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 154         if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
 155                 __free_pages(page, 0);
 156                 net_err_ratelimited("%s: %s dma map failed\n",
 157                                     dev_name(dev), q->name);
 158                 stats->dma_map_err++;
 159                 return -EIO;
 160         }
 161
 162         buf_info->page = page;
 163         buf_info->page_offset = 0;
 164
 165         return 0;
 166 }
 167
 168 static void ionic_rx_page_free(struct ionic_queue *q,
 169                                struct ionic_buf_info *buf_info)
 170 {
 171         struct device *dev = q->dev;
 172
 173         if (unlikely(!buf_info)) {
 174                 net_err_ratelimited("%s: %s invalid buf_info in free\n",
 175                                     dev_name(dev), q->name);
 176                 return;
 177         }
 178
 179         if (!buf_info->page)
 180                 return;
 181
 182         dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 183         __free_pages(buf_info->page, 0);
 184         buf_info->page = NULL;
 185 }
 186
 187 static bool ionic_rx_buf_recycle(struct ionic_queue *q,
 188                                  struct ionic_buf_info *buf_info, u32 len)
 189 {
 190         u32 size;
 191
 192         /* don't re-use pages allocated in low-mem condition */
 193         if (page_is_pfmemalloc(buf_info->page))
 194                 return false;
 195
 196         /* don't re-use buffers from non-local numa nodes */
 197         if (page_to_nid(buf_info->page) != numa_mem_id())
 198                 return false;
 199
 200         size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ);
 201         buf_info->page_offset += size;
 202         if (buf_info->page_offset >= IONIC_PAGE_SIZE)
 203                 return false;
 204
 205         get_page(buf_info->page);
 206
 207         return true;
 208 }
 209
 210 static void ionic_rx_add_skb_frag(struct ionic_queue *q,
 211                                   struct sk_buff *skb,
 212                                   struct ionic_buf_info *buf_info,
 213                                   u32 off, u32 len,
 214                                   bool synced)
 215 {
 216         if (!synced)
 217                 dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info),
 218                                               off, len, DMA_FROM_DEVICE);
 219
 220         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 221                         buf_info->page, buf_info->page_offset + off,
 222                         len,
 223                         IONIC_PAGE_SIZE);
 224
 225         if (!ionic_rx_buf_recycle(q, buf_info, len)) {
 226                 dma_unmap_page(q->dev, buf_info->dma_addr,
 227                                IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 228                 buf_info->page = NULL;
 229         }
 230 }
 231
 232 static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
 233                                           struct ionic_rx_desc_info *desc_info,
 234                                           unsigned int headroom,
 235                                           unsigned int len,
 236                                           unsigned int num_sg_elems,
 237                                           bool synced)
 238 {
 239         struct ionic_buf_info *buf_info;
 240         struct ionic_rx_stats *stats;
 241         struct sk_buff *skb;
 242         unsigned int i;
 243         u16 frag_len;
 244
 245         stats = q_to_rx_stats(q);
 246
 247         buf_info = &desc_info->bufs[0];
 248         prefetchw(buf_info->page);
 249
 250         skb = napi_get_frags(&q_to_qcq(q)->napi);
 251         if (unlikely(!skb)) {
 252                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 253                                      dev_name(q->dev), q->name);
 254                 stats->alloc_err++;
 255                 return NULL;
 256         }
 257
 258         if (headroom)
 259                 frag_len = min_t(u16, len,
 260                                  IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 261         else
 262                 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
 263
 264         if (unlikely(!buf_info->page))
 265                 goto err_bad_buf_page;
 266         ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced);
 267         len -= frag_len;
 268         buf_info++;
 269
 270         for (i = 0; i < num_sg_elems; i++, buf_info++) {
 271                 if (unlikely(!buf_info->page))
 272                         goto err_bad_buf_page;
 273                 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
 274                 ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced);
 275                 len -= frag_len;
 276         }
 277
 278         return skb;
 279
 280 err_bad_buf_page:
 281         dev_kfree_skb(skb);
 282         return NULL;
 283 }
 284
 285 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
 286                                           struct ionic_queue *q,
 287                                           struct ionic_rx_desc_info *desc_info,
 288                                           unsigned int headroom,
 289                                           unsigned int len,
 290                                           bool synced)
 291 {
 292         struct ionic_buf_info *buf_info;
 293         struct ionic_rx_stats *stats;
 294         struct device *dev = q->dev;
 295         struct sk_buff *skb;
 296
 297         stats = q_to_rx_stats(q);
 298
 299         buf_info = &desc_info->bufs[0];
 300
 301         skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
 302         if (unlikely(!skb)) {
 303                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 304                                      dev_name(dev), q->name);
 305                 stats->alloc_err++;
 306                 return NULL;
 307         }
 308
 309         if (unlikely(!buf_info->page)) {
 310                 dev_kfree_skb(skb);
 311                 return NULL;
 312         }
 313
 314         if (!synced)
 315                 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
 316                                               headroom, len, DMA_FROM_DEVICE);
 317         skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
 318         dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info),
 319                                          headroom, len, DMA_FROM_DEVICE);
 320
 321         skb_put(skb, len);
 322         skb->protocol = eth_type_trans(skb, netdev);
 323
 324         return skb;
 325 }
 326
 327 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
 328                                     struct ionic_tx_desc_info *desc_info)
 329 {
 330         unsigned int nbufs = desc_info->nbufs;
 331         struct ionic_buf_info *buf_info;
 332         struct device *dev = q->dev;
 333         int i;
 334
 335         if (!nbufs)
 336                 return;
 337
 338         buf_info = desc_info->bufs;
 339         dma_unmap_single(dev, buf_info->dma_addr,
 340                          buf_info->len, DMA_TO_DEVICE);
 341         if (desc_info->act == XDP_TX)
 342                 __free_pages(buf_info->page, 0);
 343         buf_info->page = NULL;
 344
 345         buf_info++;
 346         for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) {
 347                 dma_unmap_page(dev, buf_info->dma_addr,
 348                                buf_info->len, DMA_TO_DEVICE);
 349                 if (desc_info->act == XDP_TX)
 350                         __free_pages(buf_info->page, 0);
 351                 buf_info->page = NULL;
 352         }
 353
 354         if (desc_info->act == XDP_REDIRECT)
 355                 xdp_return_frame(desc_info->xdpf);
 356
 357         desc_info->nbufs = 0;
 358         desc_info->xdpf = NULL;
 359         desc_info->act = 0;
 360 }
 361
 362 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
 363                                 enum xdp_action act, struct page *page, int off,
 364                                 bool ring_doorbell)
 365 {
 366         struct ionic_tx_desc_info *desc_info;
 367         struct ionic_buf_info *buf_info;
 368         struct ionic_tx_stats *stats;
 369         struct ionic_txq_desc *desc;
 370         size_t len = frame->len;
 371         dma_addr_t dma_addr;
 372         u64 cmd;
 373
 374         desc_info = &q->tx_info[q->head_idx];
 375         desc = &q->txq[q->head_idx];
 376         buf_info = desc_info->bufs;
 377         stats = q_to_tx_stats(q);
 378
 379         dma_addr = ionic_tx_map_single(q, frame->data, len);
 380         if (dma_mapping_error(q->dev, dma_addr)) {
 381                 stats->dma_map_err++;
 382                 return -EIO;
 383         }
 384         buf_info->dma_addr = dma_addr;
 385         buf_info->len = len;
 386         buf_info->page = page;
 387         buf_info->page_offset = off;
 388
 389         desc_info->nbufs = 1;
 390         desc_info->xdpf = frame;
 391         desc_info->act = act;
 392
 393         if (xdp_frame_has_frags(frame)) {
 394                 struct ionic_txq_sg_elem *elem;
 395                 struct skb_shared_info *sinfo;
 396                 struct ionic_buf_info *bi;
 397                 skb_frag_t *frag;
 398                 int i;
 399
 400                 bi = &buf_info[1];
 401                 sinfo = xdp_get_shared_info_from_frame(frame);
 402                 frag = sinfo->frags;
 403                 elem = ionic_tx_sg_elems(q);
 404                 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
 405                         dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
 406                         if (dma_mapping_error(q->dev, dma_addr)) {
 407                                 stats->dma_map_err++;
 408                                 ionic_tx_desc_unmap_bufs(q, desc_info);
 409                                 return -EIO;
 410                         }
 411                         bi->dma_addr = dma_addr;
 412                         bi->len = skb_frag_size(frag);
 413                         bi->page = skb_frag_page(frag);
 414
 415                         elem->addr = cpu_to_le64(bi->dma_addr);
 416                         elem->len = cpu_to_le16(bi->len);
 417                         elem++;
 418
 419                         desc_info->nbufs++;
 420                 }
 421         }
 422
 423         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
 424                                   0, (desc_info->nbufs - 1), buf_info->dma_addr);
 425         desc->cmd = cpu_to_le64(cmd);
 426         desc->len = cpu_to_le16(len);
 427         desc->csum_start = 0;
 428         desc->csum_offset = 0;
 429
 430         stats->xdp_frames++;
 431         stats->pkts++;
 432         stats->bytes += len;
 433
 434         ionic_txq_post(q, ring_doorbell);
 435
 436         return 0;
 437 }
 438
 439 int ionic_xdp_xmit(struct net_device *netdev, int n,
 440                    struct xdp_frame **xdp_frames, u32 flags)
 441 {
 442         struct ionic_lif *lif = netdev_priv(netdev);
 443         struct ionic_queue *txq;
 444         struct netdev_queue *nq;
 445         int nxmit;
 446         int space;
 447         int cpu;
 448         int qi;
 449
 450         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))
 451                 return -ENETDOWN;
 452
 453         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 454                 return -EINVAL;
 455
 456         /* AdminQ is assumed on cpu 0, while we attempt to affinitize the
 457          * TxRx queue pairs 0..n-1 on cpus 1..n.  We try to keep with that
 458          * affinitization here, but of course irqbalance and friends might
 459          * have juggled things anyway, so we have to check for the 0 case.
 460          */
 461         cpu = smp_processor_id();
 462         qi = cpu ? (cpu - 1) % lif->nxqs : cpu;
 463
 464         txq = &lif->txqcqs[qi]->q;
 465         nq = netdev_get_tx_queue(netdev, txq->index);
 466         __netif_tx_lock(nq, cpu);
 467         txq_trans_cond_update(nq);
 468
 469         if (netif_tx_queue_stopped(nq) ||
 470             !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 471                                   ionic_q_space_avail(txq),
 472                                   1, 1)) {
 473                 __netif_tx_unlock(nq);
 474                 return -EIO;
 475         }
 476
 477         space = min_t(int, n, ionic_q_space_avail(txq));
 478         for (nxmit = 0; nxmit < space ; nxmit++) {
 479                 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit],
 480                                          XDP_REDIRECT,
 481                                          virt_to_page(xdp_frames[nxmit]->data),
 482                                          0, false)) {
 483                         nxmit--;
 484                         break;
 485                 }
 486         }
 487
 488         if (flags & XDP_XMIT_FLUSH)
 489                 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,
 490                                  txq->dbval | txq->head_idx);
 491
 492         netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 493                              ionic_q_space_avail(txq),
 494                              4, 4);
 495         __netif_tx_unlock(nq);
 496
 497         return nxmit;
 498 }
 499
 500 static bool ionic_run_xdp(struct ionic_rx_stats *stats,
 501                           struct net_device *netdev,
 502                           struct bpf_prog *xdp_prog,
 503                           struct ionic_queue *rxq,
 504                           struct ionic_buf_info *buf_info,
 505                           int len)
 506 {
 507         u32 xdp_action = XDP_ABORTED;
 508         struct xdp_buff xdp_buf;
 509         struct ionic_queue *txq;
 510         struct netdev_queue *nq;
 511         struct xdp_frame *xdpf;
 512         int remain_len;
 513         int frag_len;
 514         int err = 0;
 515
 516         xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
 517         frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 518         xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
 519                          XDP_PACKET_HEADROOM, frag_len, false);
 520
 521         dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
 522                                       XDP_PACKET_HEADROOM, len,
 523                                       DMA_FROM_DEVICE);
 524
 525         prefetchw(&xdp_buf.data_hard_start);
 526
 527         /*  We limit MTU size to one buffer if !xdp_has_frags, so
 528          *  if the recv len is bigger than one buffer
 529          *     then we know we have frag info to gather
 530          */
 531         remain_len = len - frag_len;
 532         if (remain_len) {
 533                 struct skb_shared_info *sinfo;
 534                 struct ionic_buf_info *bi;
 535                 skb_frag_t *frag;
 536
 537                 bi = buf_info;
 538                 sinfo = xdp_get_shared_info_from_buff(&xdp_buf);
 539                 sinfo->nr_frags = 0;
 540                 sinfo->xdp_frags_size = 0;
 541                 xdp_buff_set_frags_flag(&xdp_buf);
 542
 543                 do {
 544                         if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
 545                                 err = -ENOSPC;
 546                                 goto out_xdp_abort;
 547                         }
 548
 549                         frag = &sinfo->frags[sinfo->nr_frags];
 550                         sinfo->nr_frags++;
 551                         bi++;
 552                         frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi));
 553                         dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi),
 554                                                       0, frag_len, DMA_FROM_DEVICE);
 555                         skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
 556                         sinfo->xdp_frags_size += frag_len;
 557                         remain_len -= frag_len;
 558
 559                         if (page_is_pfmemalloc(bi->page))
 560                                 xdp_buff_set_frag_pfmemalloc(&xdp_buf);
 561                 } while (remain_len > 0);
 562         }
 563
 564         xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
 565
 566         switch (xdp_action) {
 567         case XDP_PASS:
 568                 stats->xdp_pass++;
 569                 return false;  /* false = we didn't consume the packet */
 570
 571         case XDP_DROP:
 572                 ionic_rx_page_free(rxq, buf_info);
 573                 stats->xdp_drop++;
 574                 break;
 575
 576         case XDP_TX:
 577                 xdpf = xdp_convert_buff_to_frame(&xdp_buf);
 578                 if (!xdpf)
 579                         goto out_xdp_abort;
 580
 581                 txq = rxq->partner;
 582                 nq = netdev_get_tx_queue(netdev, txq->index);
 583                 __netif_tx_lock(nq, smp_processor_id());
 584                 txq_trans_cond_update(nq);
 585
 586                 if (netif_tx_queue_stopped(nq) ||
 587                     !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 588                                           ionic_q_space_avail(txq),
 589                                           1, 1)) {
 590                         __netif_tx_unlock(nq);
 591                         goto out_xdp_abort;
 592                 }
 593
 594                 dma_unmap_page(rxq->dev, buf_info->dma_addr,
 595                                IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 596
 597                 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX,
 598                                            buf_info->page,
 599                                            buf_info->page_offset,
 600                                            true);
 601                 __netif_tx_unlock(nq);
 602                 if (err) {
 603                         netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
 604                         goto out_xdp_abort;
 605                 }
 606                 stats->xdp_tx++;
 607
 608                 /* the Tx completion will free the buffers */
 609                 break;
 610
 611         case XDP_REDIRECT:
 612                 /* unmap the pages before handing them to a different device */
 613                 dma_unmap_page(rxq->dev, buf_info->dma_addr,
 614                                IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 615
 616                 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
 617                 if (err) {
 618                         netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
 619                         goto out_xdp_abort;
 620                 }
 621                 buf_info->page = NULL;
 622                 rxq->xdp_flush = true;
 623                 stats->xdp_redirect++;
 624                 break;
 625
 626         case XDP_ABORTED:
 627         default:
 628                 goto out_xdp_abort;
 629         }
 630
 631         return true;
 632
 633 out_xdp_abort:
 634         trace_xdp_exception(netdev, xdp_prog, xdp_action);
 635         ionic_rx_page_free(rxq, buf_info);
 636         stats->xdp_aborted++;
 637
 638         return true;
 639 }
 640
 641 static void ionic_rx_clean(struct ionic_queue *q,
 642                            struct ionic_rx_desc_info *desc_info,
 643                            struct ionic_rxq_comp *comp)
 644 {
 645         struct net_device *netdev = q->lif->netdev;
 646         struct ionic_qcq *qcq = q_to_qcq(q);
 647         struct ionic_rx_stats *stats;
 648         struct bpf_prog *xdp_prog;
 649         unsigned int headroom;
 650         struct sk_buff *skb;
 651         bool synced = false;
 652         bool use_copybreak;
 653         u16 len;
 654
 655         stats = q_to_rx_stats(q);
 656
 657         if (comp->status) {
 658                 stats->dropped++;
 659                 return;
 660         }
 661
 662         len = le16_to_cpu(comp->len);
 663         stats->pkts++;
 664         stats->bytes += len;
 665
 666         xdp_prog = READ_ONCE(q->lif->xdp_prog);
 667         if (xdp_prog) {
 668                 if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))
 669                         return;
 670                 synced = true;
 671         }
 672
 673         headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
 674         use_copybreak = len <= q->lif->rx_copybreak;
 675         if (use_copybreak)
 676                 skb = ionic_rx_copybreak(netdev, q, desc_info,
 677                                          headroom, len, synced);
 678         else
 679                 skb = ionic_rx_build_skb(q, desc_info, headroom, len,
 680                                          comp->num_sg_elems, synced);
 681
 682         if (unlikely(!skb)) {
 683                 stats->dropped++;
 684                 return;
 685         }
 686
 687         skb_record_rx_queue(skb, q->index);
 688
 689         if (likely(netdev->features & NETIF_F_RXHASH)) {
 690                 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 691                 case IONIC_PKT_TYPE_IPV4:
 692                 case IONIC_PKT_TYPE_IPV6:
 693                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 694                                      PKT_HASH_TYPE_L3);
 695                         break;
 696                 case IONIC_PKT_TYPE_IPV4_TCP:
 697                 case IONIC_PKT_TYPE_IPV6_TCP:
 698                 case IONIC_PKT_TYPE_IPV4_UDP:
 699                 case IONIC_PKT_TYPE_IPV6_UDP:
 700                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 701                                      PKT_HASH_TYPE_L4);
 702                         break;
 703                 }
 704         }
 705
 706         if (likely(netdev->features & NETIF_F_RXCSUM) &&
 707             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
 708                 skb->ip_summed = CHECKSUM_COMPLETE;
 709                 skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
 710                 stats->csum_complete++;
 711         } else {
 712                 stats->csum_none++;
 713         }
 714
 715         if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 716                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
 717                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
 718                 stats->csum_error++;
 719
 720         if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
 721             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
 722                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 723                                        le16_to_cpu(comp->vlan_tci));
 724                 stats->vlan_stripped++;
 725         }
 726
 727         if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
 728                 __le64 *cq_desc_hwstamp;
 729                 u64 hwstamp;
 730
 731                 cq_desc_hwstamp =
 732                         (void *)comp +
 733                         qcq->cq.desc_size -
 734                         sizeof(struct ionic_rxq_comp) -
 735                         IONIC_HWSTAMP_CQ_NEGOFFSET;
 736
 737                 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
 738
 739                 if (hwstamp != IONIC_HWSTAMP_INVALID) {
 740                         skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
 741                         stats->hwstamp_valid++;
 742                 } else {
 743                         stats->hwstamp_invalid++;
 744                 }
 745         }
 746
 747         if (use_copybreak)
 748                 napi_gro_receive(&qcq->napi, skb);
 749         else
 750                 napi_gro_frags(&qcq->napi);
 751 }
 752
 753 bool ionic_rx_service(struct ionic_cq *cq)
 754 {
 755         struct ionic_rx_desc_info *desc_info;
 756         struct ionic_queue *q = cq->bound_q;
 757         struct ionic_rxq_comp *comp;
 758
 759         comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx];
 760
 761         if (!color_match(comp->pkt_type_color, cq->done_color))
 762                 return false;
 763
 764         /* check for empty queue */
 765         if (q->tail_idx == q->head_idx)
 766                 return false;
 767
 768         if (q->tail_idx != le16_to_cpu(comp->comp_index))
 769                 return false;
 770
 771         desc_info = &q->rx_info[q->tail_idx];
 772         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 773
 774         /* clean the related q entry, only one per qc completion */
 775         ionic_rx_clean(q, desc_info, comp);
 776
 777         return true;
 778 }
 779
 780 static inline void ionic_write_cmb_desc(struct ionic_queue *q,
 781                                         void *desc)
 782 {
 783         /* Since Rx and Tx descriptors are the same size, we can
 784          * save an instruction or two and skip the qtype check.
 785          */
 786         if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS))
 787                 memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0]));
 788 }
 789
 790 void ionic_rx_fill(struct ionic_queue *q)
 791 {
 792         struct net_device *netdev = q->lif->netdev;
 793         struct ionic_rx_desc_info *desc_info;
 794         struct ionic_rxq_sg_elem *sg_elem;
 795         struct ionic_buf_info *buf_info;
 796         unsigned int fill_threshold;
 797         struct ionic_rxq_desc *desc;
 798         unsigned int remain_len;
 799         unsigned int frag_len;
 800         unsigned int nfrags;
 801         unsigned int n_fill;
 802         unsigned int len;
 803         unsigned int i;
 804         unsigned int j;
 805
 806         n_fill = ionic_q_space_avail(q);
 807
 808         fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
 809                                q->num_descs / IONIC_RX_FILL_DIV);
 810         if (n_fill < fill_threshold)
 811                 return;
 812
 813         len = netdev->mtu + VLAN_ETH_HLEN;
 814
 815         for (i = n_fill; i; i--) {
 816                 unsigned int headroom;
 817                 unsigned int buf_len;
 818
 819                 nfrags = 0;
 820                 remain_len = len;
 821                 desc = &q->rxq[q->head_idx];
 822                 desc_info = &q->rx_info[q->head_idx];
 823                 buf_info = &desc_info->bufs[0];
 824
 825                 if (!buf_info->page) { /* alloc a new buffer? */
 826                         if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
 827                                 desc->addr = 0;
 828                                 desc->len = 0;
 829                                 return;
 830                         }
 831                 }
 832
 833                 /* fill main descriptor - buf[0]
 834                  * XDP uses space in the first buffer, so account for
 835                  * head room, tail room, and ip header in the first frag size.
 836                  */
 837                 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
 838                 if (q->xdp_rxq_info)
 839                         buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN;
 840                 else
 841                         buf_len = ionic_rx_buf_size(buf_info);
 842                 frag_len = min_t(u16, len, buf_len);
 843
 844                 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
 845                 desc->len = cpu_to_le16(frag_len);
 846                 remain_len -= frag_len;
 847                 buf_info++;
 848                 nfrags++;
 849
 850                 /* fill sg descriptors - buf[1..n] */
 851                 sg_elem = q->rxq_sgl[q->head_idx].elems;
 852                 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) {
 853                         if (!buf_info->page) { /* alloc a new sg buffer? */
 854                                 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
 855                                         sg_elem->addr = 0;
 856                                         sg_elem->len = 0;
 857                                         return;
 858                                 }
 859                         }
 860
 861                         sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
 862                         frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info));
 863                         sg_elem->len = cpu_to_le16(frag_len);
 864                         remain_len -= frag_len;
 865                         buf_info++;
 866                         nfrags++;
 867                 }
 868
 869                 /* clear end sg element as a sentinel */
 870                 if (j < q->max_sg_elems)
 871                         memset(sg_elem, 0, sizeof(*sg_elem));
 872
 873                 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 874                                               IONIC_RXQ_DESC_OPCODE_SIMPLE;
 875                 desc_info->nbufs = nfrags;
 876
 877                 ionic_write_cmb_desc(q, desc);
 878
 879                 ionic_rxq_post(q, false);
 880         }
 881
 882         ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
 883                          q->dbval | q->head_idx);
 884
 885         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 886         q->dbell_jiffies = jiffies;
 887
 888         mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
 889                   jiffies + IONIC_NAPI_DEADLINE);
 890 }
 891
 892 void ionic_rx_empty(struct ionic_queue *q)
 893 {
 894         struct ionic_rx_desc_info *desc_info;
 895         struct ionic_buf_info *buf_info;
 896         unsigned int i, j;
 897
 898         for (i = 0; i < q->num_descs; i++) {
 899                 desc_info = &q->rx_info[i];
 900                 for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) {
 901                         buf_info = &desc_info->bufs[j];
 902                         if (buf_info->page)
 903                                 ionic_rx_page_free(q, buf_info);
 904                 }
 905
 906                 desc_info->nbufs = 0;
 907         }
 908
 909         q->head_idx = 0;
 910         q->tail_idx = 0;
 911 }
 912
 913 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
 914 {
 915         struct dim_sample dim_sample;
 916         struct ionic_lif *lif;
 917         unsigned int qi;
 918         u64 pkts, bytes;
 919
 920         if (!qcq->intr.dim_coal_hw)
 921                 return;
 922
 923         lif = qcq->q.lif;
 924         qi = qcq->cq.bound_q->index;
 925
 926         switch (napi_mode) {
 927         case IONIC_LIF_F_TX_DIM_INTR:
 928                 pkts = lif->txqstats[qi].pkts;
 929                 bytes = lif->txqstats[qi].bytes;
 930                 break;
 931         case IONIC_LIF_F_RX_DIM_INTR:
 932                 pkts = lif->rxqstats[qi].pkts;
 933                 bytes = lif->rxqstats[qi].bytes;
 934                 break;
 935         default:
 936                 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
 937                 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
 938                 break;
 939         }
 940
 941         dim_update_sample(qcq->cq.bound_intr->rearm_count,
 942                           pkts, bytes, &dim_sample);
 943
 944         net_dim(&qcq->dim, dim_sample);
 945 }
 946
 947 int ionic_tx_napi(struct napi_struct *napi, int budget)
 948 {
 949         struct ionic_qcq *qcq = napi_to_qcq(napi);
 950         struct ionic_cq *cq = napi_to_cq(napi);
 951         u32 work_done = 0;
 952         u32 flags = 0;
 953
 954         work_done = ionic_tx_cq_service(cq, budget);
 955
 956         if (unlikely(!budget))
 957                 return budget;
 958
 959         if (work_done < budget && napi_complete_done(napi, work_done)) {
 960                 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
 961                 flags |= IONIC_INTR_CRED_UNMASK;
 962                 cq->bound_intr->rearm_count++;
 963         }
 964
 965         if (work_done || flags) {
 966                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
 967                 ionic_intr_credits(cq->idev->intr_ctrl,
 968                                    cq->bound_intr->index,
 969                                    work_done, flags);
 970         }
 971
 972         if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
 973                 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
 974
 975         return work_done;
 976 }
 977
 978 static void ionic_xdp_do_flush(struct ionic_cq *cq)
 979 {
 980         if (cq->bound_q->xdp_flush) {
 981                 xdp_do_flush();
 982                 cq->bound_q->xdp_flush = false;
 983         }
 984 }
 985
 986 int ionic_rx_napi(struct napi_struct *napi, int budget)
 987 {
 988         struct ionic_qcq *qcq = napi_to_qcq(napi);
 989         struct ionic_cq *cq = napi_to_cq(napi);
 990         u32 work_done = 0;
 991         u32 flags = 0;
 992
 993         if (unlikely(!budget))
 994                 return budget;
 995
 996         work_done = ionic_cq_service(cq, budget,
 997                                      ionic_rx_service, NULL, NULL);
 998
 999         ionic_rx_fill(cq->bound_q);
1000
1001         ionic_xdp_do_flush(cq);
1002         if (work_done < budget && napi_complete_done(napi, work_done)) {
1003                 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
1004                 flags |= IONIC_INTR_CRED_UNMASK;
1005                 cq->bound_intr->rearm_count++;
1006         }
1007
1008         if (work_done || flags) {
1009                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1010                 ionic_intr_credits(cq->idev->intr_ctrl,
1011                                    cq->bound_intr->index,
1012                                    work_done, flags);
1013         }
1014
1015         if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
1016                 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
1017
1018         return work_done;
1019 }
1020
1021 int ionic_txrx_napi(struct napi_struct *napi, int budget)
1022 {
1023         struct ionic_qcq *rxqcq = napi_to_qcq(napi);
1024         struct ionic_cq *rxcq = napi_to_cq(napi);
1025         unsigned int qi = rxcq->bound_q->index;
1026         struct ionic_qcq *txqcq;
1027         struct ionic_lif *lif;
1028         struct ionic_cq *txcq;
1029         bool resched = false;
1030         u32 rx_work_done = 0;
1031         u32 tx_work_done = 0;
1032         u32 flags = 0;
1033
1034         lif = rxcq->bound_q->lif;
1035         txqcq = lif->txqcqs[qi];
1036         txcq = &lif->txqcqs[qi]->cq;
1037
1038         tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT);
1039
1040         if (unlikely(!budget))
1041                 return budget;
1042
1043         rx_work_done = ionic_cq_service(rxcq, budget,
1044                                         ionic_rx_service, NULL, NULL);
1045
1046         ionic_rx_fill(rxcq->bound_q);
1047
1048         ionic_xdp_do_flush(rxcq);
1049         if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
1050                 ionic_dim_update(rxqcq, 0);
1051                 flags |= IONIC_INTR_CRED_UNMASK;
1052                 rxcq->bound_intr->rearm_count++;
1053         }
1054
1055         if (rx_work_done || flags) {
1056                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1057                 ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index,
1058                                    tx_work_done + rx_work_done, flags);
1059         }
1060
1061         if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
1062                 resched = true;
1063         if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
1064                 resched = true;
1065         if (resched)
1066                 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
1067
1068         return rx_work_done;
1069 }
1070
1071 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
1072                                       void *data, size_t len)
1073 {
1074         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1075         struct device *dev = q->dev;
1076         dma_addr_t dma_addr;
1077
1078         dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
1079         if (dma_mapping_error(dev, dma_addr)) {
1080                 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
1081                                      dev_name(dev), q->name);
1082                 stats->dma_map_err++;
1083                 return 0;
1084         }
1085         return dma_addr;
1086 }
1087
1088 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
1089                                     const skb_frag_t *frag,
1090                                     size_t offset, size_t len)
1091 {
1092         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1093         struct device *dev = q->dev;
1094         dma_addr_t dma_addr;
1095
1096         dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
1097         if (dma_mapping_error(dev, dma_addr)) {
1098                 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
1099                                      dev_name(dev), q->name);
1100                 stats->dma_map_err++;
1101         }
1102         return dma_addr;
1103 }
1104
1105 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
1106                             struct ionic_tx_desc_info *desc_info)
1107 {
1108         struct ionic_buf_info *buf_info = desc_info->bufs;
1109         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1110         struct device *dev = q->dev;
1111         dma_addr_t dma_addr;
1112         unsigned int nfrags;
1113         skb_frag_t *frag;
1114         int frag_idx;
1115
1116         dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
1117         if (dma_mapping_error(dev, dma_addr)) {
1118                 stats->dma_map_err++;
1119                 return -EIO;
1120         }
1121         buf_info->dma_addr = dma_addr;
1122         buf_info->len = skb_headlen(skb);
1123         buf_info++;
1124
1125         frag = skb_shinfo(skb)->frags;
1126         nfrags = skb_shinfo(skb)->nr_frags;
1127         for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
1128                 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
1129                 if (dma_mapping_error(dev, dma_addr)) {
1130                         stats->dma_map_err++;
1131                         goto dma_fail;
1132                 }
1133                 buf_info->dma_addr = dma_addr;
1134                 buf_info->len = skb_frag_size(frag);
1135                 buf_info++;
1136         }
1137
1138         desc_info->nbufs = 1 + nfrags;
1139
1140         return 0;
1141
1142 dma_fail:
1143         /* unwind the frag mappings and the head mapping */
1144         while (frag_idx > 0) {
1145                 frag_idx--;
1146                 buf_info--;
1147                 dma_unmap_page(dev, buf_info->dma_addr,
1148                                buf_info->len, DMA_TO_DEVICE);
1149         }
1150         dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
1151         return -EIO;
1152 }
1153
1154 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
1155                                      struct ionic_tx_desc_info *desc_info)
1156 {
1157         struct ionic_buf_info *buf_info = desc_info->bufs;
1158         struct device *dev = q->dev;
1159         unsigned int i;
1160
1161         if (!desc_info->nbufs)
1162                 return;
1163
1164         dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
1165                          buf_info->len, DMA_TO_DEVICE);
1166         buf_info++;
1167         for (i = 1; i < desc_info->nbufs; i++, buf_info++)
1168                 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
1169                                buf_info->len, DMA_TO_DEVICE);
1170
1171         desc_info->nbufs = 0;
1172 }
1173
1174 static void ionic_tx_clean(struct ionic_queue *q,
1175                            struct ionic_tx_desc_info *desc_info,
1176                            struct ionic_txq_comp *comp)
1177 {
1178         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1179         struct ionic_qcq *qcq = q_to_qcq(q);
1180         struct sk_buff *skb;
1181
1182         if (desc_info->xdpf) {
1183                 ionic_xdp_tx_desc_clean(q->partner, desc_info);
1184                 stats->clean++;
1185
1186                 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
1187                         netif_wake_subqueue(q->lif->netdev, q->index);
1188
1189                 return;
1190         }
1191
1192         ionic_tx_desc_unmap_bufs(q, desc_info);
1193
1194         skb = desc_info->skb;
1195         if (!skb)
1196                 return;
1197
1198         if (unlikely(ionic_txq_hwstamp_enabled(q))) {
1199                 if (comp) {
1200                         struct skb_shared_hwtstamps hwts = {};
1201                         __le64 *cq_desc_hwstamp;
1202                         u64 hwstamp;
1203
1204                         cq_desc_hwstamp =
1205                                 (void *)comp +
1206                                 qcq->cq.desc_size -
1207                                 sizeof(struct ionic_txq_comp) -
1208                                 IONIC_HWSTAMP_CQ_NEGOFFSET;
1209
1210                         hwstamp = le64_to_cpu(*cq_desc_hwstamp);
1211
1212                         if (hwstamp != IONIC_HWSTAMP_INVALID) {
1213                                 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
1214
1215                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1216                                 skb_tstamp_tx(skb, &hwts);
1217
1218                                 stats->hwstamp_valid++;
1219                         } else {
1220                                 stats->hwstamp_invalid++;
1221                         }
1222                 }
1223         }
1224
1225         desc_info->bytes = skb->len;
1226         stats->clean++;
1227
1228         napi_consume_skb(skb, 1);
1229 }
1230
1231 static bool ionic_tx_service(struct ionic_cq *cq,
1232                              unsigned int *total_pkts, unsigned int *total_bytes)
1233 {
1234         struct ionic_tx_desc_info *desc_info;
1235         struct ionic_queue *q = cq->bound_q;
1236         struct ionic_txq_comp *comp;
1237         unsigned int bytes = 0;
1238         unsigned int pkts = 0;
1239         u16 index;
1240
1241         comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx];
1242
1243         if (!color_match(comp->color, cq->done_color))
1244                 return false;
1245
1246         /* clean the related q entries, there could be
1247          * several q entries completed for each cq completion
1248          */
1249         do {
1250                 desc_info = &q->tx_info[q->tail_idx];
1251                 desc_info->bytes = 0;
1252                 index = q->tail_idx;
1253                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1254                 ionic_tx_clean(q, desc_info, comp);
1255                 if (desc_info->skb) {
1256                         pkts++;
1257                         bytes += desc_info->bytes;
1258                         desc_info->skb = NULL;
1259                 }
1260         } while (index != le16_to_cpu(comp->comp_index));
1261
1262         (*total_pkts) += pkts;
1263         (*total_bytes) += bytes;
1264
1265         return true;
1266 }
1267
1268 unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do)
1269 {
1270         unsigned int work_done = 0;
1271         unsigned int bytes = 0;
1272         unsigned int pkts = 0;
1273
1274         if (work_to_do == 0)
1275                 return 0;
1276
1277         while (ionic_tx_service(cq, &pkts, &bytes)) {
1278                 if (cq->tail_idx == cq->num_descs - 1)
1279                         cq->done_color = !cq->done_color;
1280                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
1281
1282                 if (++work_done >= work_to_do)
1283                         break;
1284         }
1285
1286         if (work_done) {
1287                 struct ionic_queue *q = cq->bound_q;
1288
1289                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1290                         netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q),
1291                                                  pkts, bytes,
1292                                                  ionic_q_space_avail(q),
1293                                                  IONIC_TSO_DESCS_NEEDED);
1294         }
1295
1296         return work_done;
1297 }
1298
1299 void ionic_tx_flush(struct ionic_cq *cq)
1300 {
1301         u32 work_done;
1302
1303         work_done = ionic_tx_cq_service(cq, cq->num_descs);
1304         if (work_done)
1305                 ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index,
1306                                    work_done, IONIC_INTR_CRED_RESET_COALESCE);
1307 }
1308
1309 void ionic_tx_empty(struct ionic_queue *q)
1310 {
1311         struct ionic_tx_desc_info *desc_info;
1312         int bytes = 0;
1313         int pkts = 0;
1314
1315         /* walk the not completed tx entries, if any */
1316         while (q->head_idx != q->tail_idx) {
1317                 desc_info = &q->tx_info[q->tail_idx];
1318                 desc_info->bytes = 0;
1319                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1320                 ionic_tx_clean(q, desc_info, NULL);
1321                 if (desc_info->skb) {
1322                         pkts++;
1323                         bytes += desc_info->bytes;
1324                         desc_info->skb = NULL;
1325                 }
1326         }
1327
1328         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1329                 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q);
1330
1331                 netdev_tx_completed_queue(ndq, pkts, bytes);
1332                 netdev_tx_reset_queue(ndq);
1333         }
1334 }
1335
1336 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
1337 {
1338         int err;
1339
1340         err = skb_cow_head(skb, 0);
1341         if (err)
1342                 return err;
1343
1344         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1345                 inner_ip_hdr(skb)->check = 0;
1346                 inner_tcp_hdr(skb)->check =
1347                         ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
1348                                            inner_ip_hdr(skb)->daddr,
1349                                            0, IPPROTO_TCP, 0);
1350         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1351                 inner_tcp_hdr(skb)->check =
1352                         ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
1353                                          &inner_ipv6_hdr(skb)->daddr,
1354                                          0, IPPROTO_TCP, 0);
1355         }
1356
1357         return 0;
1358 }
1359
1360 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
1361 {
1362         int err;
1363
1364         err = skb_cow_head(skb, 0);
1365         if (err)
1366                 return err;
1367
1368         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1369                 ip_hdr(skb)->check = 0;
1370                 tcp_hdr(skb)->check =
1371                         ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1372                                            ip_hdr(skb)->daddr,
1373                                            0, IPPROTO_TCP, 0);
1374         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1375                 tcp_v6_gso_csum_prep(skb);
1376         }
1377
1378         return 0;
1379 }
1380
1381 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
1382                               struct ionic_tx_desc_info *desc_info,
1383                               struct sk_buff *skb,
1384                               dma_addr_t addr, u8 nsge, u16 len,
1385                               unsigned int hdrlen, unsigned int mss,
1386                               bool outer_csum,
1387                               u16 vlan_tci, bool has_vlan,
1388                               bool start, bool done)
1389 {
1390         struct ionic_txq_desc *desc = &q->txq[q->head_idx];
1391         u8 flags = 0;
1392         u64 cmd;
1393
1394         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1395         flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1396         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
1397         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
1398
1399         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
1400         desc->cmd = cpu_to_le64(cmd);
1401         desc->len = cpu_to_le16(len);
1402         desc->vlan_tci = cpu_to_le16(vlan_tci);
1403         desc->hdr_len = cpu_to_le16(hdrlen);
1404         desc->mss = cpu_to_le16(mss);
1405
1406         ionic_write_cmb_desc(q, desc);
1407
1408         if (start) {
1409                 skb_tx_timestamp(skb);
1410                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1411                         netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len);
1412                 ionic_txq_post(q, false);
1413         } else {
1414                 ionic_txq_post(q, done);
1415         }
1416 }
1417
1418 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
1419                         struct sk_buff *skb)
1420 {
1421         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1422         struct ionic_tx_desc_info *desc_info;
1423         struct ionic_buf_info *buf_info;
1424         struct ionic_txq_sg_elem *elem;
1425         struct ionic_txq_desc *desc;
1426         unsigned int chunk_len;
1427         unsigned int frag_rem;
1428         unsigned int tso_rem;
1429         unsigned int seg_rem;
1430         dma_addr_t desc_addr;
1431         dma_addr_t frag_addr;
1432         unsigned int hdrlen;
1433         unsigned int len;
1434         unsigned int mss;
1435         bool start, done;
1436         bool outer_csum;
1437         bool has_vlan;
1438         u16 desc_len;
1439         u8 desc_nsge;
1440         u16 vlan_tci;
1441         bool encap;
1442         int err;
1443
1444         desc_info = &q->tx_info[q->head_idx];
1445
1446         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1447                 return -EIO;
1448
1449         len = skb->len;
1450         mss = skb_shinfo(skb)->gso_size;
1451         outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1452                                                    SKB_GSO_GRE_CSUM |
1453                                                    SKB_GSO_IPXIP4 |
1454                                                    SKB_GSO_IPXIP6 |
1455                                                    SKB_GSO_UDP_TUNNEL |
1456                                                    SKB_GSO_UDP_TUNNEL_CSUM));
1457         has_vlan = !!skb_vlan_tag_present(skb);
1458         vlan_tci = skb_vlan_tag_get(skb);
1459         encap = skb->encapsulation;
1460
1461         /* Preload inner-most TCP csum field with IP pseudo hdr
1462          * calculated with IP length set to zero.  HW will later
1463          * add in length to each TCP segment resulting from the TSO.
1464          */
1465
1466         if (encap)
1467                 err = ionic_tx_tcp_inner_pseudo_csum(skb);
1468         else
1469                 err = ionic_tx_tcp_pseudo_csum(skb);
1470         if (err) {
1471                 /* clean up mapping from ionic_tx_map_skb */
1472                 ionic_tx_desc_unmap_bufs(q, desc_info);
1473                 return err;
1474         }
1475
1476         if (encap)
1477                 hdrlen = skb_inner_tcp_all_headers(skb);
1478         else
1479                 hdrlen = skb_tcp_all_headers(skb);
1480
1481         desc_info->skb = skb;
1482         buf_info = desc_info->bufs;
1483         tso_rem = len;
1484         seg_rem = min(tso_rem, hdrlen + mss);
1485
1486         frag_addr = 0;
1487         frag_rem = 0;
1488
1489         start = true;
1490
1491         while (tso_rem > 0) {
1492                 desc = NULL;
1493                 elem = NULL;
1494                 desc_addr = 0;
1495                 desc_len = 0;
1496                 desc_nsge = 0;
1497                 /* use fragments until we have enough to post a single descriptor */
1498                 while (seg_rem > 0) {
1499                         /* if the fragment is exhausted then move to the next one */
1500                         if (frag_rem == 0) {
1501                                 /* grab the next fragment */
1502                                 frag_addr = buf_info->dma_addr;
1503                                 frag_rem = buf_info->len;
1504                                 buf_info++;
1505                         }
1506                         chunk_len = min(frag_rem, seg_rem);
1507                         if (!desc) {
1508                                 /* fill main descriptor */
1509                                 desc = &q->txq[q->head_idx];
1510                                 elem = ionic_tx_sg_elems(q);
1511                                 desc_addr = frag_addr;
1512                                 desc_len = chunk_len;
1513                         } else {
1514                                 /* fill sg descriptor */
1515                                 elem->addr = cpu_to_le64(frag_addr);
1516                                 elem->len = cpu_to_le16(chunk_len);
1517                                 elem++;
1518                                 desc_nsge++;
1519                         }
1520                         frag_addr += chunk_len;
1521                         frag_rem -= chunk_len;
1522                         tso_rem -= chunk_len;
1523                         seg_rem -= chunk_len;
1524                 }
1525                 seg_rem = min(tso_rem, mss);
1526                 done = (tso_rem == 0);
1527                 /* post descriptor */
1528                 ionic_tx_tso_post(netdev, q, desc_info, skb,
1529                                   desc_addr, desc_nsge, desc_len,
1530                                   hdrlen, mss, outer_csum, vlan_tci, has_vlan,
1531                                   start, done);
1532                 start = false;
1533                 /* Buffer information is stored with the first tso descriptor */
1534                 desc_info = &q->tx_info[q->head_idx];
1535                 desc_info->nbufs = 0;
1536         }
1537
1538         stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
1539         stats->bytes += len;
1540         stats->tso++;
1541         stats->tso_bytes = len;
1542
1543         return 0;
1544 }
1545
1546 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
1547                                struct ionic_tx_desc_info *desc_info)
1548 {
1549         struct ionic_txq_desc *desc = &q->txq[q->head_idx];
1550         struct ionic_buf_info *buf_info = desc_info->bufs;
1551         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1552         bool has_vlan;
1553         u8 flags = 0;
1554         bool encap;
1555         u64 cmd;
1556
1557         has_vlan = !!skb_vlan_tag_present(skb);
1558         encap = skb->encapsulation;
1559
1560         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1561         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1562
1563         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
1564                                   flags, skb_shinfo(skb)->nr_frags,
1565                                   buf_info->dma_addr);
1566         desc->cmd = cpu_to_le64(cmd);
1567         desc->len = cpu_to_le16(buf_info->len);
1568         if (has_vlan) {
1569                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1570                 stats->vlan_inserted++;
1571         } else {
1572                 desc->vlan_tci = 0;
1573         }
1574         desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
1575         desc->csum_offset = cpu_to_le16(skb->csum_offset);
1576
1577         ionic_write_cmb_desc(q, desc);
1578
1579         if (skb_csum_is_sctp(skb))
1580                 stats->crc32_csum++;
1581         else
1582                 stats->csum++;
1583 }
1584
1585 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
1586                                   struct ionic_tx_desc_info *desc_info)
1587 {
1588         struct ionic_txq_desc *desc = &q->txq[q->head_idx];
1589         struct ionic_buf_info *buf_info = desc_info->bufs;
1590         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1591         bool has_vlan;
1592         u8 flags = 0;
1593         bool encap;
1594         u64 cmd;
1595
1596         has_vlan = !!skb_vlan_tag_present(skb);
1597         encap = skb->encapsulation;
1598
1599         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1600         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1601
1602         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
1603                                   flags, skb_shinfo(skb)->nr_frags,
1604                                   buf_info->dma_addr);
1605         desc->cmd = cpu_to_le64(cmd);
1606         desc->len = cpu_to_le16(buf_info->len);
1607         if (has_vlan) {
1608                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1609                 stats->vlan_inserted++;
1610         } else {
1611                 desc->vlan_tci = 0;
1612         }
1613         desc->csum_start = 0;
1614         desc->csum_offset = 0;
1615
1616         ionic_write_cmb_desc(q, desc);
1617
1618         stats->csum_none++;
1619 }
1620
1621 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
1622                                struct ionic_tx_desc_info *desc_info)
1623 {
1624         struct ionic_buf_info *buf_info = &desc_info->bufs[1];
1625         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1626         struct ionic_txq_sg_elem *elem;
1627         unsigned int i;
1628
1629         elem = ionic_tx_sg_elems(q);
1630         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
1631                 elem->addr = cpu_to_le64(buf_info->dma_addr);
1632                 elem->len = cpu_to_le16(buf_info->len);
1633         }
1634
1635         stats->frags += skb_shinfo(skb)->nr_frags;
1636 }
1637
1638 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q,
1639                     struct sk_buff *skb)
1640 {
1641         struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx];
1642         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1643         bool ring_dbell = true;
1644
1645         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1646                 return -EIO;
1647
1648         desc_info->skb = skb;
1649
1650         /* set up the initial descriptor */
1651         if (skb->ip_summed == CHECKSUM_PARTIAL)
1652                 ionic_tx_calc_csum(q, skb, desc_info);
1653         else
1654                 ionic_tx_calc_no_csum(q, skb, desc_info);
1655
1656         /* add frags */
1657         ionic_tx_skb_frags(q, skb, desc_info);
1658
1659         skb_tx_timestamp(skb);
1660         stats->pkts++;
1661         stats->bytes += skb->len;
1662
1663         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1664                 struct netdev_queue *ndq = q_to_ndq(netdev, q);
1665
1666                 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1)))
1667                         netif_tx_stop_queue(ndq);
1668                 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len,
1669                                                     netdev_xmit_more());
1670         }
1671         ionic_txq_post(q, ring_dbell);
1672
1673         return 0;
1674 }
1675
1676 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1677 {
1678         int nr_frags = skb_shinfo(skb)->nr_frags;
1679         bool too_many_frags = false;
1680         skb_frag_t *frag;
1681         int desc_bufs;
1682         int chunk_len;
1683         int frag_rem;
1684         int tso_rem;
1685         int seg_rem;
1686         bool encap;
1687         int hdrlen;
1688         int ndescs;
1689         int err;
1690
1691         /* Each desc is mss long max, so a descriptor for each gso_seg */
1692         if (skb_is_gso(skb)) {
1693                 ndescs = skb_shinfo(skb)->gso_segs;
1694                 if (!nr_frags)
1695                         return ndescs;
1696         } else {
1697                 ndescs = 1;
1698                 if (!nr_frags)
1699                         return ndescs;
1700
1701                 if (unlikely(nr_frags > q->max_sg_elems)) {
1702                         too_many_frags = true;
1703                         goto linearize;
1704                 }
1705
1706                 return ndescs;
1707         }
1708
1709         /* We need to scan the skb to be sure that none of the MTU sized
1710          * packets in the TSO will require more sgs per descriptor than we
1711          * can support.  We loop through the frags, add up the lengths for
1712          * a packet, and count the number of sgs used per packet.
1713          */
1714         tso_rem = skb->len;
1715         frag = skb_shinfo(skb)->frags;
1716         encap = skb->encapsulation;
1717
1718         /* start with just hdr in first part of first descriptor */
1719         if (encap)
1720                 hdrlen = skb_inner_tcp_all_headers(skb);
1721         else
1722                 hdrlen = skb_tcp_all_headers(skb);
1723         seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size);
1724         frag_rem = hdrlen;
1725
1726         while (tso_rem > 0) {
1727                 desc_bufs = 0;
1728                 while (seg_rem > 0) {
1729                         desc_bufs++;
1730
1731                         /* We add the +1 because we can take buffers for one
1732                          * more than we have SGs: one for the initial desc data
1733                          * in addition to the SG segments that might follow.
1734                          */
1735                         if (desc_bufs > q->max_sg_elems + 1) {
1736                                 too_many_frags = true;
1737                                 goto linearize;
1738                         }
1739
1740                         if (frag_rem == 0) {
1741                                 frag_rem = skb_frag_size(frag);
1742                                 frag++;
1743                         }
1744                         chunk_len = min(frag_rem, seg_rem);
1745                         frag_rem -= chunk_len;
1746                         tso_rem -= chunk_len;
1747                         seg_rem -= chunk_len;
1748                 }
1749
1750                 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size);
1751         }
1752
1753 linearize:
1754         if (too_many_frags) {
1755                 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1756
1757                 err = skb_linearize(skb);
1758                 if (err)
1759                         return err;
1760                 stats->linearize++;
1761         }
1762
1763         return ndescs;
1764 }
1765
1766 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1767                                             struct net_device *netdev)
1768 {
1769         struct ionic_lif *lif = netdev_priv(netdev);
1770         struct ionic_queue *q;
1771         int err, ndescs;
1772
1773         /* Does not stop/start txq, because we post to a separate tx queue
1774          * for timestamping, and if a packet can't be posted immediately to
1775          * the timestamping queue, it is dropped.
1776          */
1777
1778         q = &lif->hwstamp_txq->q;
1779         ndescs = ionic_tx_descs_needed(q, skb);
1780         if (unlikely(ndescs < 0))
1781                 goto err_out_drop;
1782
1783         if (unlikely(!ionic_q_has_space(q, ndescs)))
1784                 goto err_out_drop;
1785
1786         skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1787         if (skb_is_gso(skb))
1788                 err = ionic_tx_tso(netdev, q, skb);
1789         else
1790                 err = ionic_tx(netdev, q, skb);
1791
1792         if (err)
1793                 goto err_out_drop;
1794
1795         return NETDEV_TX_OK;
1796
1797 err_out_drop:
1798         q->drop++;
1799         dev_kfree_skb(skb);
1800         return NETDEV_TX_OK;
1801 }
1802
1803 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1804 {
1805         u16 queue_index = skb_get_queue_mapping(skb);
1806         struct ionic_lif *lif = netdev_priv(netdev);
1807         struct ionic_queue *q;
1808         int ndescs;
1809         int err;
1810
1811         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1812                 dev_kfree_skb(skb);
1813                 return NETDEV_TX_OK;
1814         }
1815
1816         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1817                 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1818                         return ionic_start_hwstamp_xmit(skb, netdev);
1819
1820         if (unlikely(queue_index >= lif->nxqs))
1821                 queue_index = 0;
1822         q = &lif->txqcqs[queue_index]->q;
1823
1824         ndescs = ionic_tx_descs_needed(q, skb);
1825         if (ndescs < 0)
1826                 goto err_out_drop;
1827
1828         if (!netif_txq_maybe_stop(q_to_ndq(netdev, q),
1829                                   ionic_q_space_avail(q),
1830                                   ndescs, ndescs))
1831                 return NETDEV_TX_BUSY;
1832
1833         if (skb_is_gso(skb))
1834                 err = ionic_tx_tso(netdev, q, skb);
1835         else
1836                 err = ionic_tx(netdev, q, skb);
1837
1838         if (err)
1839                 goto err_out_drop;
1840
1841         return NETDEV_TX_OK;
1842
1843 err_out_drop:
1844         q->drop++;
1845         dev_kfree_skb(skb);
1846         return NETDEV_TX_OK;
1847 }