drivers/net/ethernet/pensando/ionic/ionic_txrx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip6_checksum.h>
   8 #include <net/netdev_queues.h>
   9
  10 #include "ionic.h"
  11 #include "ionic_lif.h"
  12 #include "ionic_txrx.h"
  13
  14 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
  15                                       void *data, size_t len);
  16
  17 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
  18                                     const skb_frag_t *frag,
  19                                     size_t offset, size_t len);
  20
  21 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
  22                                      struct ionic_desc_info *desc_info);
  23
  24 static void ionic_tx_clean(struct ionic_queue *q,
  25                            struct ionic_desc_info *desc_info,
  26                            struct ionic_cq_info *cq_info,
  27                            void *cb_arg);
  28
  29 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
  30                                   ionic_desc_cb cb_func, void *cb_arg)
  31 {
  32         ionic_q_post(q, ring_dbell, cb_func, cb_arg);
  33 }
  34
  35 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
  36                                   ionic_desc_cb cb_func, void *cb_arg)
  37 {
  38         ionic_q_post(q, ring_dbell, cb_func, cb_arg);
  39 }
  40
  41 bool ionic_txq_poke_doorbell(struct ionic_queue *q)
  42 {
  43         struct netdev_queue *netdev_txq;
  44         unsigned long now, then, dif;
  45         struct net_device *netdev;
  46
  47         netdev = q->lif->netdev;
  48         netdev_txq = netdev_get_tx_queue(netdev, q->index);
  49
  50         HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
  51
  52         if (q->tail_idx == q->head_idx) {
  53                 HARD_TX_UNLOCK(netdev, netdev_txq);
  54                 return false;
  55         }
  56
  57         now = READ_ONCE(jiffies);
  58         then = q->dbell_jiffies;
  59         dif = now - then;
  60
  61         if (dif > q->dbell_deadline) {
  62                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  63                                  q->dbval | q->head_idx);
  64
  65                 q->dbell_jiffies = now;
  66         }
  67
  68         HARD_TX_UNLOCK(netdev, netdev_txq);
  69
  70         return true;
  71 }
  72
  73 bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
  74 {
  75         unsigned long now, then, dif;
  76
  77         /* no lock, called from rx napi or txrx napi, nothing else can fill */
  78
  79         if (q->tail_idx == q->head_idx)
  80                 return false;
  81
  82         now = READ_ONCE(jiffies);
  83         then = q->dbell_jiffies;
  84         dif = now - then;
  85
  86         if (dif > q->dbell_deadline) {
  87                 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
  88                                  q->dbval | q->head_idx);
  89
  90                 q->dbell_jiffies = now;
  91
  92                 dif = 2 * q->dbell_deadline;
  93                 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
  94                         dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
  95
  96                 q->dbell_deadline = dif;
  97         }
  98
  99         return true;
 100 }
 101
 102 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev,
 103                                             struct ionic_queue *q)
 104 {
 105         return netdev_get_tx_queue(netdev, q->index);
 106 }
 107
 108 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
 109 {
 110         return page_address(buf_info->page) + buf_info->page_offset;
 111 }
 112
 113 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
 114 {
 115         return buf_info->dma_addr + buf_info->page_offset;
 116 }
 117
 118 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info)
 119 {
 120         return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset);
 121 }
 122
 123 static int ionic_rx_page_alloc(struct ionic_queue *q,
 124                                struct ionic_buf_info *buf_info)
 125 {
 126         struct ionic_rx_stats *stats;
 127         struct device *dev;
 128         struct page *page;
 129
 130         dev = q->dev;
 131         stats = q_to_rx_stats(q);
 132
 133         if (unlikely(!buf_info)) {
 134                 net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
 135                                     dev_name(dev), q->name);
 136                 return -EINVAL;
 137         }
 138
 139         page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
 140         if (unlikely(!page)) {
 141                 net_err_ratelimited("%s: %s page alloc failed\n",
 142                                     dev_name(dev), q->name);
 143                 stats->alloc_err++;
 144                 return -ENOMEM;
 145         }
 146
 147         buf_info->dma_addr = dma_map_page(dev, page, 0,
 148                                           IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 149         if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
 150                 __free_pages(page, 0);
 151                 net_err_ratelimited("%s: %s dma map failed\n",
 152                                     dev_name(dev), q->name);
 153                 stats->dma_map_err++;
 154                 return -EIO;
 155         }
 156
 157         buf_info->page = page;
 158         buf_info->page_offset = 0;
 159
 160         return 0;
 161 }
 162
 163 static void ionic_rx_page_free(struct ionic_queue *q,
 164                                struct ionic_buf_info *buf_info)
 165 {
 166         struct device *dev = q->dev;
 167
 168         if (unlikely(!buf_info)) {
 169                 net_err_ratelimited("%s: %s invalid buf_info in free\n",
 170                                     dev_name(dev), q->name);
 171                 return;
 172         }
 173
 174         if (!buf_info->page)
 175                 return;
 176
 177         dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 178         __free_pages(buf_info->page, 0);
 179         buf_info->page = NULL;
 180 }
 181
 182 static bool ionic_rx_buf_recycle(struct ionic_queue *q,
 183                                  struct ionic_buf_info *buf_info, u32 used)
 184 {
 185         u32 size;
 186
 187         /* don't re-use pages allocated in low-mem condition */
 188         if (page_is_pfmemalloc(buf_info->page))
 189                 return false;
 190
 191         /* don't re-use buffers from non-local numa nodes */
 192         if (page_to_nid(buf_info->page) != numa_mem_id())
 193                 return false;
 194
 195         size = ALIGN(used, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ);
 196         buf_info->page_offset += size;
 197         if (buf_info->page_offset >= IONIC_PAGE_SIZE)
 198                 return false;
 199
 200         get_page(buf_info->page);
 201
 202         return true;
 203 }
 204
 205 static struct sk_buff *ionic_rx_frags(struct net_device *netdev,
 206                                       struct ionic_queue *q,
 207                                       struct ionic_desc_info *desc_info,
 208                                       unsigned int headroom,
 209                                       unsigned int len,
 210                                       unsigned int num_sg_elems,
 211                                       bool synced)
 212 {
 213         struct ionic_buf_info *buf_info;
 214         struct ionic_rx_stats *stats;
 215         struct device *dev = q->dev;
 216         struct sk_buff *skb;
 217         unsigned int i;
 218         u16 frag_len;
 219
 220         stats = q_to_rx_stats(q);
 221
 222         buf_info = &desc_info->bufs[0];
 223
 224         prefetchw(buf_info->page);
 225
 226         skb = napi_get_frags(&q_to_qcq(q)->napi);
 227         if (unlikely(!skb)) {
 228                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 229                                      dev_name(dev), q->name);
 230                 stats->alloc_err++;
 231                 return NULL;
 232         }
 233
 234         i = num_sg_elems + 1;
 235         do {
 236                 if (unlikely(!buf_info->page)) {
 237                         dev_kfree_skb(skb);
 238                         return NULL;
 239                 }
 240
 241                 if (headroom)
 242                         frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 243                 else
 244                         frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
 245                 len -= frag_len;
 246
 247                 if (!synced)
 248                         dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
 249                                                       headroom, frag_len, DMA_FROM_DEVICE);
 250
 251                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 252                                 buf_info->page, buf_info->page_offset + headroom,
 253                                 frag_len, IONIC_PAGE_SIZE);
 254
 255                 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
 256                         dma_unmap_page(dev, buf_info->dma_addr,
 257                                        IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 258                         buf_info->page = NULL;
 259                 }
 260
 261                 /* only needed on the first buffer */
 262                 if (headroom)
 263                         headroom = 0;
 264
 265                 buf_info++;
 266
 267                 i--;
 268         } while (i > 0);
 269
 270         return skb;
 271 }
 272
 273 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
 274                                           struct ionic_queue *q,
 275                                           struct ionic_desc_info *desc_info,
 276                                           unsigned int headroom,
 277                                           unsigned int len,
 278                                           bool synced)
 279 {
 280         struct ionic_buf_info *buf_info;
 281         struct ionic_rx_stats *stats;
 282         struct device *dev = q->dev;
 283         struct sk_buff *skb;
 284
 285         stats = q_to_rx_stats(q);
 286
 287         buf_info = &desc_info->bufs[0];
 288
 289         skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
 290         if (unlikely(!skb)) {
 291                 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
 292                                      dev_name(dev), q->name);
 293                 stats->alloc_err++;
 294                 return NULL;
 295         }
 296
 297         if (unlikely(!buf_info->page)) {
 298                 dev_kfree_skb(skb);
 299                 return NULL;
 300         }
 301
 302         if (!synced)
 303                 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
 304                                               headroom, len, DMA_FROM_DEVICE);
 305         skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
 306         dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info),
 307                                          headroom, len, DMA_FROM_DEVICE);
 308
 309         skb_put(skb, len);
 310         skb->protocol = eth_type_trans(skb, netdev);
 311
 312         return skb;
 313 }
 314
 315 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
 316                                     struct ionic_desc_info *desc_info)
 317 {
 318         unsigned int nbufs = desc_info->nbufs;
 319         struct ionic_buf_info *buf_info;
 320         struct device *dev = q->dev;
 321         int i;
 322
 323         if (!nbufs)
 324                 return;
 325
 326         buf_info = desc_info->bufs;
 327         dma_unmap_single(dev, buf_info->dma_addr,
 328                          buf_info->len, DMA_TO_DEVICE);
 329         if (desc_info->act == XDP_TX)
 330                 __free_pages(buf_info->page, 0);
 331         buf_info->page = NULL;
 332
 333         buf_info++;
 334         for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) {
 335                 dma_unmap_page(dev, buf_info->dma_addr,
 336                                buf_info->len, DMA_TO_DEVICE);
 337                 if (desc_info->act == XDP_TX)
 338                         __free_pages(buf_info->page, 0);
 339                 buf_info->page = NULL;
 340         }
 341
 342         if (desc_info->act == XDP_REDIRECT)
 343                 xdp_return_frame(desc_info->xdpf);
 344
 345         desc_info->nbufs = 0;
 346         desc_info->xdpf = NULL;
 347         desc_info->act = 0;
 348 }
 349
 350 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
 351                                 enum xdp_action act, struct page *page, int off,
 352                                 bool ring_doorbell)
 353 {
 354         struct ionic_desc_info *desc_info;
 355         struct ionic_buf_info *buf_info;
 356         struct ionic_tx_stats *stats;
 357         struct ionic_txq_desc *desc;
 358         size_t len = frame->len;
 359         dma_addr_t dma_addr;
 360         u64 cmd;
 361
 362         desc_info = &q->info[q->head_idx];
 363         desc = desc_info->txq_desc;
 364         buf_info = desc_info->bufs;
 365         stats = q_to_tx_stats(q);
 366
 367         dma_addr = ionic_tx_map_single(q, frame->data, len);
 368         if (dma_mapping_error(q->dev, dma_addr)) {
 369                 stats->dma_map_err++;
 370                 return -EIO;
 371         }
 372         buf_info->dma_addr = dma_addr;
 373         buf_info->len = len;
 374         buf_info->page = page;
 375         buf_info->page_offset = off;
 376
 377         desc_info->nbufs = 1;
 378         desc_info->xdpf = frame;
 379         desc_info->act = act;
 380
 381         if (xdp_frame_has_frags(frame)) {
 382                 struct ionic_txq_sg_elem *elem;
 383                 struct skb_shared_info *sinfo;
 384                 struct ionic_buf_info *bi;
 385                 skb_frag_t *frag;
 386                 int i;
 387
 388                 bi = &buf_info[1];
 389                 sinfo = xdp_get_shared_info_from_frame(frame);
 390                 frag = sinfo->frags;
 391                 elem = desc_info->txq_sg_desc->elems;
 392                 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
 393                         dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
 394                         if (dma_mapping_error(q->dev, dma_addr)) {
 395                                 stats->dma_map_err++;
 396                                 ionic_tx_desc_unmap_bufs(q, desc_info);
 397                                 return -EIO;
 398                         }
 399                         bi->dma_addr = dma_addr;
 400                         bi->len = skb_frag_size(frag);
 401                         bi->page = skb_frag_page(frag);
 402
 403                         elem->addr = cpu_to_le64(bi->dma_addr);
 404                         elem->len = cpu_to_le16(bi->len);
 405                         elem++;
 406
 407                         desc_info->nbufs++;
 408                 }
 409         }
 410
 411         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
 412                                   0, (desc_info->nbufs - 1), buf_info->dma_addr);
 413         desc->cmd = cpu_to_le64(cmd);
 414         desc->len = cpu_to_le16(len);
 415         desc->csum_start = 0;
 416         desc->csum_offset = 0;
 417
 418         stats->xdp_frames++;
 419         stats->pkts++;
 420         stats->bytes += len;
 421
 422         ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL);
 423
 424         return 0;
 425 }
 426
 427 int ionic_xdp_xmit(struct net_device *netdev, int n,
 428                    struct xdp_frame **xdp_frames, u32 flags)
 429 {
 430         struct ionic_lif *lif = netdev_priv(netdev);
 431         struct ionic_queue *txq;
 432         struct netdev_queue *nq;
 433         int nxmit;
 434         int space;
 435         int cpu;
 436         int qi;
 437
 438         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))
 439                 return -ENETDOWN;
 440
 441         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 442                 return -EINVAL;
 443
 444         /* AdminQ is assumed on cpu 0, while we attempt to affinitize the
 445          * TxRx queue pairs 0..n-1 on cpus 1..n.  We try to keep with that
 446          * affinitization here, but of course irqbalance and friends might
 447          * have juggled things anyway, so we have to check for the 0 case.
 448          */
 449         cpu = smp_processor_id();
 450         qi = cpu ? (cpu - 1) % lif->nxqs : cpu;
 451
 452         txq = &lif->txqcqs[qi]->q;
 453         nq = netdev_get_tx_queue(netdev, txq->index);
 454         __netif_tx_lock(nq, cpu);
 455         txq_trans_cond_update(nq);
 456
 457         if (netif_tx_queue_stopped(nq) ||
 458             !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 459                                   ionic_q_space_avail(txq),
 460                                   1, 1)) {
 461                 __netif_tx_unlock(nq);
 462                 return -EIO;
 463         }
 464
 465         space = min_t(int, n, ionic_q_space_avail(txq));
 466         for (nxmit = 0; nxmit < space ; nxmit++) {
 467                 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit],
 468                                          XDP_REDIRECT,
 469                                          virt_to_page(xdp_frames[nxmit]->data),
 470                                          0, false)) {
 471                         nxmit--;
 472                         break;
 473                 }
 474         }
 475
 476         if (flags & XDP_XMIT_FLUSH)
 477                 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,
 478                                  txq->dbval | txq->head_idx);
 479
 480         netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 481                              ionic_q_space_avail(txq),
 482                              4, 4);
 483         __netif_tx_unlock(nq);
 484
 485         return nxmit;
 486 }
 487
 488 static bool ionic_run_xdp(struct ionic_rx_stats *stats,
 489                           struct net_device *netdev,
 490                           struct bpf_prog *xdp_prog,
 491                           struct ionic_queue *rxq,
 492                           struct ionic_buf_info *buf_info,
 493                           int len)
 494 {
 495         u32 xdp_action = XDP_ABORTED;
 496         struct xdp_buff xdp_buf;
 497         struct ionic_queue *txq;
 498         struct netdev_queue *nq;
 499         struct xdp_frame *xdpf;
 500         int remain_len;
 501         int frag_len;
 502         int err = 0;
 503
 504         xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
 505         frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
 506         xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
 507                          XDP_PACKET_HEADROOM, frag_len, false);
 508
 509         dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
 510                                       XDP_PACKET_HEADROOM, len,
 511                                       DMA_FROM_DEVICE);
 512
 513         prefetchw(&xdp_buf.data_hard_start);
 514
 515         /*  We limit MTU size to one buffer if !xdp_has_frags, so
 516          *  if the recv len is bigger than one buffer
 517          *     then we know we have frag info to gather
 518          */
 519         remain_len = len - frag_len;
 520         if (remain_len) {
 521                 struct skb_shared_info *sinfo;
 522                 struct ionic_buf_info *bi;
 523                 skb_frag_t *frag;
 524
 525                 bi = buf_info;
 526                 sinfo = xdp_get_shared_info_from_buff(&xdp_buf);
 527                 sinfo->nr_frags = 0;
 528                 sinfo->xdp_frags_size = 0;
 529                 xdp_buff_set_frags_flag(&xdp_buf);
 530
 531                 do {
 532                         if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
 533                                 err = -ENOSPC;
 534                                 goto out_xdp_abort;
 535                         }
 536
 537                         frag = &sinfo->frags[sinfo->nr_frags];
 538                         sinfo->nr_frags++;
 539                         bi++;
 540                         frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi));
 541                         dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi),
 542                                                       0, frag_len, DMA_FROM_DEVICE);
 543                         skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
 544                         sinfo->xdp_frags_size += frag_len;
 545                         remain_len -= frag_len;
 546
 547                         if (page_is_pfmemalloc(bi->page))
 548                                 xdp_buff_set_frag_pfmemalloc(&xdp_buf);
 549                 } while (remain_len > 0);
 550         }
 551
 552         xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
 553
 554         switch (xdp_action) {
 555         case XDP_PASS:
 556                 stats->xdp_pass++;
 557                 return false;  /* false = we didn't consume the packet */
 558
 559         case XDP_DROP:
 560                 ionic_rx_page_free(rxq, buf_info);
 561                 stats->xdp_drop++;
 562                 break;
 563
 564         case XDP_TX:
 565                 xdpf = xdp_convert_buff_to_frame(&xdp_buf);
 566                 if (!xdpf)
 567                         goto out_xdp_abort;
 568
 569                 txq = rxq->partner;
 570                 nq = netdev_get_tx_queue(netdev, txq->index);
 571                 __netif_tx_lock(nq, smp_processor_id());
 572                 txq_trans_cond_update(nq);
 573
 574                 if (netif_tx_queue_stopped(nq) ||
 575                     !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
 576                                           ionic_q_space_avail(txq),
 577                                           1, 1)) {
 578                         __netif_tx_unlock(nq);
 579                         goto out_xdp_abort;
 580                 }
 581
 582                 dma_unmap_page(rxq->dev, buf_info->dma_addr,
 583                                IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 584
 585                 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX,
 586                                            buf_info->page,
 587                                            buf_info->page_offset,
 588                                            true);
 589                 __netif_tx_unlock(nq);
 590                 if (err) {
 591                         netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
 592                         goto out_xdp_abort;
 593                 }
 594                 stats->xdp_tx++;
 595
 596                 /* the Tx completion will free the buffers */
 597                 break;
 598
 599         case XDP_REDIRECT:
 600                 /* unmap the pages before handing them to a different device */
 601                 dma_unmap_page(rxq->dev, buf_info->dma_addr,
 602                                IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
 603
 604                 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
 605                 if (err) {
 606                         netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
 607                         goto out_xdp_abort;
 608                 }
 609                 buf_info->page = NULL;
 610                 rxq->xdp_flush = true;
 611                 stats->xdp_redirect++;
 612                 break;
 613
 614         case XDP_ABORTED:
 615         default:
 616                 goto out_xdp_abort;
 617         }
 618
 619         return true;
 620
 621 out_xdp_abort:
 622         trace_xdp_exception(netdev, xdp_prog, xdp_action);
 623         ionic_rx_page_free(rxq, buf_info);
 624         stats->xdp_aborted++;
 625
 626         return true;
 627 }
 628
 629 static void ionic_rx_clean(struct ionic_queue *q,
 630                            struct ionic_desc_info *desc_info,
 631                            struct ionic_cq_info *cq_info,
 632                            void *cb_arg)
 633 {
 634         struct net_device *netdev = q->lif->netdev;
 635         struct ionic_qcq *qcq = q_to_qcq(q);
 636         struct ionic_rx_stats *stats;
 637         struct ionic_rxq_comp *comp;
 638         struct bpf_prog *xdp_prog;
 639         unsigned int headroom;
 640         struct sk_buff *skb;
 641         u16 len;
 642
 643         comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
 644
 645         stats = q_to_rx_stats(q);
 646
 647         if (comp->status) {
 648                 stats->dropped++;
 649                 return;
 650         }
 651
 652         len = le16_to_cpu(comp->len);
 653         stats->pkts++;
 654         stats->bytes += len;
 655
 656         xdp_prog = READ_ONCE(q->lif->xdp_prog);
 657         if (xdp_prog &&
 658             ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))
 659                 return;
 660
 661         headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
 662         if (len <= q->lif->rx_copybreak)
 663                 skb = ionic_rx_copybreak(netdev, q, desc_info,
 664                                          headroom, len, !!xdp_prog);
 665         else
 666                 skb = ionic_rx_frags(netdev, q, desc_info, headroom, len,
 667                                      comp->num_sg_elems, !!xdp_prog);
 668
 669         if (unlikely(!skb)) {
 670                 stats->dropped++;
 671                 return;
 672         }
 673
 674         skb_record_rx_queue(skb, q->index);
 675
 676         if (likely(netdev->features & NETIF_F_RXHASH)) {
 677                 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 678                 case IONIC_PKT_TYPE_IPV4:
 679                 case IONIC_PKT_TYPE_IPV6:
 680                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 681                                      PKT_HASH_TYPE_L3);
 682                         break;
 683                 case IONIC_PKT_TYPE_IPV4_TCP:
 684                 case IONIC_PKT_TYPE_IPV6_TCP:
 685                 case IONIC_PKT_TYPE_IPV4_UDP:
 686                 case IONIC_PKT_TYPE_IPV6_UDP:
 687                         skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
 688                                      PKT_HASH_TYPE_L4);
 689                         break;
 690                 }
 691         }
 692
 693         if (likely(netdev->features & NETIF_F_RXCSUM) &&
 694             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
 695                 skb->ip_summed = CHECKSUM_COMPLETE;
 696                 skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
 697                 stats->csum_complete++;
 698         } else {
 699                 stats->csum_none++;
 700         }
 701
 702         if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 703                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
 704                      (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
 705                 stats->csum_error++;
 706
 707         if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
 708             (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
 709                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 710                                        le16_to_cpu(comp->vlan_tci));
 711                 stats->vlan_stripped++;
 712         }
 713
 714         if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
 715                 __le64 *cq_desc_hwstamp;
 716                 u64 hwstamp;
 717
 718                 cq_desc_hwstamp =
 719                         cq_info->cq_desc +
 720                         qcq->cq.desc_size -
 721                         sizeof(struct ionic_rxq_comp) -
 722                         IONIC_HWSTAMP_CQ_NEGOFFSET;
 723
 724                 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
 725
 726                 if (hwstamp != IONIC_HWSTAMP_INVALID) {
 727                         skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
 728                         stats->hwstamp_valid++;
 729                 } else {
 730                         stats->hwstamp_invalid++;
 731                 }
 732         }
 733
 734         if (len <= q->lif->rx_copybreak)
 735                 napi_gro_receive(&qcq->napi, skb);
 736         else
 737                 napi_gro_frags(&qcq->napi);
 738 }
 739
 740 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
 741 {
 742         struct ionic_queue *q = cq->bound_q;
 743         struct ionic_desc_info *desc_info;
 744         struct ionic_rxq_comp *comp;
 745
 746         comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
 747
 748         if (!color_match(comp->pkt_type_color, cq->done_color))
 749                 return false;
 750
 751         /* check for empty queue */
 752         if (q->tail_idx == q->head_idx)
 753                 return false;
 754
 755         if (q->tail_idx != le16_to_cpu(comp->comp_index))
 756                 return false;
 757
 758         desc_info = &q->info[q->tail_idx];
 759         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 760
 761         /* clean the related q entry, only one per qc completion */
 762         ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
 763
 764         desc_info->cb = NULL;
 765         desc_info->cb_arg = NULL;
 766
 767         return true;
 768 }
 769
 770 static inline void ionic_write_cmb_desc(struct ionic_queue *q,
 771                                         void __iomem *cmb_desc,
 772                                         void *desc)
 773 {
 774         if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)
 775                 memcpy_toio(cmb_desc, desc, q->desc_size);
 776 }
 777
 778 void ionic_rx_fill(struct ionic_queue *q)
 779 {
 780         struct net_device *netdev = q->lif->netdev;
 781         struct ionic_desc_info *desc_info;
 782         struct ionic_rxq_sg_desc *sg_desc;
 783         struct ionic_rxq_sg_elem *sg_elem;
 784         struct ionic_buf_info *buf_info;
 785         unsigned int fill_threshold;
 786         struct ionic_rxq_desc *desc;
 787         unsigned int remain_len;
 788         unsigned int frag_len;
 789         unsigned int nfrags;
 790         unsigned int n_fill;
 791         unsigned int len;
 792         unsigned int i;
 793         unsigned int j;
 794
 795         n_fill = ionic_q_space_avail(q);
 796
 797         fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
 798                                q->num_descs / IONIC_RX_FILL_DIV);
 799         if (n_fill < fill_threshold)
 800                 return;
 801
 802         len = netdev->mtu + VLAN_ETH_HLEN;
 803
 804         for (i = n_fill; i; i--) {
 805                 unsigned int headroom;
 806                 unsigned int buf_len;
 807
 808                 nfrags = 0;
 809                 remain_len = len;
 810                 desc_info = &q->info[q->head_idx];
 811                 desc = desc_info->desc;
 812                 buf_info = &desc_info->bufs[0];
 813
 814                 if (!buf_info->page) { /* alloc a new buffer? */
 815                         if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
 816                                 desc->addr = 0;
 817                                 desc->len = 0;
 818                                 return;
 819                         }
 820                 }
 821
 822                 /* fill main descriptor - buf[0]
 823                  * XDP uses space in the first buffer, so account for
 824                  * head room, tail room, and ip header in the first frag size.
 825                  */
 826                 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
 827                 if (q->xdp_rxq_info)
 828                         buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN;
 829                 else
 830                         buf_len = ionic_rx_buf_size(buf_info);
 831                 frag_len = min_t(u16, len, buf_len);
 832
 833                 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
 834                 desc->len = cpu_to_le16(frag_len);
 835                 remain_len -= frag_len;
 836                 buf_info++;
 837                 nfrags++;
 838
 839                 /* fill sg descriptors - buf[1..n] */
 840                 sg_desc = desc_info->sg_desc;
 841                 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
 842                         sg_elem = &sg_desc->elems[j];
 843                         if (!buf_info->page) { /* alloc a new sg buffer? */
 844                                 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
 845                                         sg_elem->addr = 0;
 846                                         sg_elem->len = 0;
 847                                         return;
 848                                 }
 849                         }
 850
 851                         sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
 852                         frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info));
 853                         sg_elem->len = cpu_to_le16(frag_len);
 854                         remain_len -= frag_len;
 855                         buf_info++;
 856                         nfrags++;
 857                 }
 858
 859                 /* clear end sg element as a sentinel */
 860                 if (j < q->max_sg_elems) {
 861                         sg_elem = &sg_desc->elems[j];
 862                         memset(sg_elem, 0, sizeof(*sg_elem));
 863                 }
 864
 865                 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 866                                               IONIC_RXQ_DESC_OPCODE_SIMPLE;
 867                 desc_info->nbufs = nfrags;
 868
 869                 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
 870
 871                 ionic_rxq_post(q, false, ionic_rx_clean, NULL);
 872         }
 873
 874         ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
 875                          q->dbval | q->head_idx);
 876
 877         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 878         q->dbell_jiffies = jiffies;
 879
 880         mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
 881                   jiffies + IONIC_NAPI_DEADLINE);
 882 }
 883
 884 void ionic_rx_empty(struct ionic_queue *q)
 885 {
 886         struct ionic_desc_info *desc_info;
 887         struct ionic_buf_info *buf_info;
 888         unsigned int i, j;
 889
 890         for (i = 0; i < q->num_descs; i++) {
 891                 desc_info = &q->info[i];
 892                 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
 893                         buf_info = &desc_info->bufs[j];
 894                         if (buf_info->page)
 895                                 ionic_rx_page_free(q, buf_info);
 896                 }
 897
 898                 desc_info->nbufs = 0;
 899                 desc_info->cb = NULL;
 900                 desc_info->cb_arg = NULL;
 901         }
 902
 903         q->head_idx = 0;
 904         q->tail_idx = 0;
 905 }
 906
 907 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
 908 {
 909         struct dim_sample dim_sample;
 910         struct ionic_lif *lif;
 911         unsigned int qi;
 912         u64 pkts, bytes;
 913
 914         if (!qcq->intr.dim_coal_hw)
 915                 return;
 916
 917         lif = qcq->q.lif;
 918         qi = qcq->cq.bound_q->index;
 919
 920         switch (napi_mode) {
 921         case IONIC_LIF_F_TX_DIM_INTR:
 922                 pkts = lif->txqstats[qi].pkts;
 923                 bytes = lif->txqstats[qi].bytes;
 924                 break;
 925         case IONIC_LIF_F_RX_DIM_INTR:
 926                 pkts = lif->rxqstats[qi].pkts;
 927                 bytes = lif->rxqstats[qi].bytes;
 928                 break;
 929         default:
 930                 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
 931                 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
 932                 break;
 933         }
 934
 935         dim_update_sample(qcq->cq.bound_intr->rearm_count,
 936                           pkts, bytes, &dim_sample);
 937
 938         net_dim(&qcq->dim, dim_sample);
 939 }
 940
 941 int ionic_tx_napi(struct napi_struct *napi, int budget)
 942 {
 943         struct ionic_qcq *qcq = napi_to_qcq(napi);
 944         struct ionic_cq *cq = napi_to_cq(napi);
 945         struct ionic_dev *idev;
 946         struct ionic_lif *lif;
 947         u32 work_done = 0;
 948         u32 flags = 0;
 949
 950         lif = cq->bound_q->lif;
 951         idev = &lif->ionic->idev;
 952
 953         work_done = ionic_tx_cq_service(cq, budget);
 954
 955         if (unlikely(!budget))
 956                 return budget;
 957
 958         if (work_done < budget && napi_complete_done(napi, work_done)) {
 959                 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
 960                 flags |= IONIC_INTR_CRED_UNMASK;
 961                 cq->bound_intr->rearm_count++;
 962         }
 963
 964         if (work_done || flags) {
 965                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
 966                 ionic_intr_credits(idev->intr_ctrl,
 967                                    cq->bound_intr->index,
 968                                    work_done, flags);
 969         }
 970
 971         if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
 972                 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
 973
 974         return work_done;
 975 }
 976
 977 static void ionic_xdp_do_flush(struct ionic_cq *cq)
 978 {
 979         if (cq->bound_q->xdp_flush) {
 980                 xdp_do_flush();
 981                 cq->bound_q->xdp_flush = false;
 982         }
 983 }
 984
 985 int ionic_rx_napi(struct napi_struct *napi, int budget)
 986 {
 987         struct ionic_qcq *qcq = napi_to_qcq(napi);
 988         struct ionic_cq *cq = napi_to_cq(napi);
 989         struct ionic_dev *idev;
 990         struct ionic_lif *lif;
 991         u32 work_done = 0;
 992         u32 flags = 0;
 993
 994         if (unlikely(!budget))
 995                 return budget;
 996
 997         lif = cq->bound_q->lif;
 998         idev = &lif->ionic->idev;
 999
1000         work_done = ionic_cq_service(cq, budget,
1001                                      ionic_rx_service, NULL, NULL);
1002
1003         ionic_rx_fill(cq->bound_q);
1004
1005         ionic_xdp_do_flush(cq);
1006         if (work_done < budget && napi_complete_done(napi, work_done)) {
1007                 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
1008                 flags |= IONIC_INTR_CRED_UNMASK;
1009                 cq->bound_intr->rearm_count++;
1010         }
1011
1012         if (work_done || flags) {
1013                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1014                 ionic_intr_credits(idev->intr_ctrl,
1015                                    cq->bound_intr->index,
1016                                    work_done, flags);
1017         }
1018
1019         if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
1020                 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
1021
1022         return work_done;
1023 }
1024
1025 int ionic_txrx_napi(struct napi_struct *napi, int budget)
1026 {
1027         struct ionic_qcq *rxqcq = napi_to_qcq(napi);
1028         struct ionic_cq *rxcq = napi_to_cq(napi);
1029         unsigned int qi = rxcq->bound_q->index;
1030         struct ionic_qcq *txqcq;
1031         struct ionic_dev *idev;
1032         struct ionic_lif *lif;
1033         struct ionic_cq *txcq;
1034         bool resched = false;
1035         u32 rx_work_done = 0;
1036         u32 tx_work_done = 0;
1037         u32 flags = 0;
1038
1039         lif = rxcq->bound_q->lif;
1040         idev = &lif->ionic->idev;
1041         txqcq = lif->txqcqs[qi];
1042         txcq = &lif->txqcqs[qi]->cq;
1043
1044         tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT);
1045
1046         if (unlikely(!budget))
1047                 return budget;
1048
1049         rx_work_done = ionic_cq_service(rxcq, budget,
1050                                         ionic_rx_service, NULL, NULL);
1051
1052         ionic_rx_fill(rxcq->bound_q);
1053
1054         ionic_xdp_do_flush(rxcq);
1055         if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
1056                 ionic_dim_update(rxqcq, 0);
1057                 flags |= IONIC_INTR_CRED_UNMASK;
1058                 rxcq->bound_intr->rearm_count++;
1059         }
1060
1061         if (rx_work_done || flags) {
1062                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1063                 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
1064                                    tx_work_done + rx_work_done, flags);
1065         }
1066
1067         if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
1068                 resched = true;
1069         if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
1070                 resched = true;
1071         if (resched)
1072                 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
1073
1074         return rx_work_done;
1075 }
1076
1077 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
1078                                       void *data, size_t len)
1079 {
1080         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1081         struct device *dev = q->dev;
1082         dma_addr_t dma_addr;
1083
1084         dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
1085         if (dma_mapping_error(dev, dma_addr)) {
1086                 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
1087                                      dev_name(dev), q->name);
1088                 stats->dma_map_err++;
1089                 return 0;
1090         }
1091         return dma_addr;
1092 }
1093
1094 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
1095                                     const skb_frag_t *frag,
1096                                     size_t offset, size_t len)
1097 {
1098         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1099         struct device *dev = q->dev;
1100         dma_addr_t dma_addr;
1101
1102         dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
1103         if (dma_mapping_error(dev, dma_addr)) {
1104                 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
1105                                      dev_name(dev), q->name);
1106                 stats->dma_map_err++;
1107         }
1108         return dma_addr;
1109 }
1110
1111 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
1112                             struct ionic_desc_info *desc_info)
1113 {
1114         struct ionic_buf_info *buf_info = desc_info->bufs;
1115         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1116         struct device *dev = q->dev;
1117         dma_addr_t dma_addr;
1118         unsigned int nfrags;
1119         skb_frag_t *frag;
1120         int frag_idx;
1121
1122         dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
1123         if (dma_mapping_error(dev, dma_addr)) {
1124                 stats->dma_map_err++;
1125                 return -EIO;
1126         }
1127         buf_info->dma_addr = dma_addr;
1128         buf_info->len = skb_headlen(skb);
1129         buf_info++;
1130
1131         frag = skb_shinfo(skb)->frags;
1132         nfrags = skb_shinfo(skb)->nr_frags;
1133         for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
1134                 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
1135                 if (dma_mapping_error(dev, dma_addr)) {
1136                         stats->dma_map_err++;
1137                         goto dma_fail;
1138                 }
1139                 buf_info->dma_addr = dma_addr;
1140                 buf_info->len = skb_frag_size(frag);
1141                 buf_info++;
1142         }
1143
1144         desc_info->nbufs = 1 + nfrags;
1145
1146         return 0;
1147
1148 dma_fail:
1149         /* unwind the frag mappings and the head mapping */
1150         while (frag_idx > 0) {
1151                 frag_idx--;
1152                 buf_info--;
1153                 dma_unmap_page(dev, buf_info->dma_addr,
1154                                buf_info->len, DMA_TO_DEVICE);
1155         }
1156         dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
1157         return -EIO;
1158 }
1159
1160 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
1161                                      struct ionic_desc_info *desc_info)
1162 {
1163         struct ionic_buf_info *buf_info = desc_info->bufs;
1164         struct device *dev = q->dev;
1165         unsigned int i;
1166
1167         if (!desc_info->nbufs)
1168                 return;
1169
1170         dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
1171                          buf_info->len, DMA_TO_DEVICE);
1172         buf_info++;
1173         for (i = 1; i < desc_info->nbufs; i++, buf_info++)
1174                 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
1175                                buf_info->len, DMA_TO_DEVICE);
1176
1177         desc_info->nbufs = 0;
1178 }
1179
1180 static void ionic_tx_clean(struct ionic_queue *q,
1181                            struct ionic_desc_info *desc_info,
1182                            struct ionic_cq_info *cq_info,
1183                            void *cb_arg)
1184 {
1185         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1186         struct ionic_qcq *qcq = q_to_qcq(q);
1187         struct sk_buff *skb = cb_arg;
1188
1189         if (desc_info->xdpf) {
1190                 ionic_xdp_tx_desc_clean(q->partner, desc_info);
1191                 stats->clean++;
1192
1193                 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
1194                         netif_wake_subqueue(q->lif->netdev, q->index);
1195
1196                 return;
1197         }
1198
1199         ionic_tx_desc_unmap_bufs(q, desc_info);
1200
1201         if (!skb)
1202                 return;
1203
1204         if (unlikely(ionic_txq_hwstamp_enabled(q))) {
1205                 if (cq_info) {
1206                         struct skb_shared_hwtstamps hwts = {};
1207                         __le64 *cq_desc_hwstamp;
1208                         u64 hwstamp;
1209
1210                         cq_desc_hwstamp =
1211                                 cq_info->cq_desc +
1212                                 qcq->cq.desc_size -
1213                                 sizeof(struct ionic_txq_comp) -
1214                                 IONIC_HWSTAMP_CQ_NEGOFFSET;
1215
1216                         hwstamp = le64_to_cpu(*cq_desc_hwstamp);
1217
1218                         if (hwstamp != IONIC_HWSTAMP_INVALID) {
1219                                 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
1220
1221                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1222                                 skb_tstamp_tx(skb, &hwts);
1223
1224                                 stats->hwstamp_valid++;
1225                         } else {
1226                                 stats->hwstamp_invalid++;
1227                         }
1228                 }
1229         }
1230
1231         desc_info->bytes = skb->len;
1232         stats->clean++;
1233
1234         napi_consume_skb(skb, 1);
1235 }
1236
1237 static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info,
1238                              unsigned int *total_pkts, unsigned int *total_bytes)
1239 {
1240         struct ionic_queue *q = cq->bound_q;
1241         struct ionic_desc_info *desc_info;
1242         struct ionic_txq_comp *comp;
1243         unsigned int bytes = 0;
1244         unsigned int pkts = 0;
1245         u16 index;
1246
1247         comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
1248
1249         if (!color_match(comp->color, cq->done_color))
1250                 return false;
1251
1252         /* clean the related q entries, there could be
1253          * several q entries completed for each cq completion
1254          */
1255         do {
1256                 desc_info = &q->info[q->tail_idx];
1257                 desc_info->bytes = 0;
1258                 index = q->tail_idx;
1259                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1260                 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
1261                 if (desc_info->cb_arg) {
1262                         pkts++;
1263                         bytes += desc_info->bytes;
1264                 }
1265                 desc_info->cb = NULL;
1266                 desc_info->cb_arg = NULL;
1267         } while (index != le16_to_cpu(comp->comp_index));
1268
1269         (*total_pkts) += pkts;
1270         (*total_bytes) += bytes;
1271
1272         return true;
1273 }
1274
1275 unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do)
1276 {
1277         struct ionic_cq_info *cq_info;
1278         unsigned int work_done = 0;
1279         unsigned int bytes = 0;
1280         unsigned int pkts = 0;
1281
1282         if (work_to_do == 0)
1283                 return 0;
1284
1285         cq_info = &cq->info[cq->tail_idx];
1286         while (ionic_tx_service(cq, cq_info, &pkts, &bytes)) {
1287                 if (cq->tail_idx == cq->num_descs - 1)
1288                         cq->done_color = !cq->done_color;
1289                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
1290                 cq_info = &cq->info[cq->tail_idx];
1291
1292                 if (++work_done >= work_to_do)
1293                         break;
1294         }
1295
1296         if (work_done) {
1297                 struct ionic_queue *q = cq->bound_q;
1298
1299                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1300                         netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q),
1301                                                  pkts, bytes,
1302                                                  ionic_q_space_avail(q),
1303                                                  IONIC_TSO_DESCS_NEEDED);
1304         }
1305
1306         return work_done;
1307 }
1308
1309 void ionic_tx_flush(struct ionic_cq *cq)
1310 {
1311         struct ionic_dev *idev = &cq->lif->ionic->idev;
1312         u32 work_done;
1313
1314         work_done = ionic_tx_cq_service(cq, cq->num_descs);
1315         if (work_done)
1316                 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
1317                                    work_done, IONIC_INTR_CRED_RESET_COALESCE);
1318 }
1319
1320 void ionic_tx_empty(struct ionic_queue *q)
1321 {
1322         struct ionic_desc_info *desc_info;
1323         int bytes = 0;
1324         int pkts = 0;
1325
1326         /* walk the not completed tx entries, if any */
1327         while (q->head_idx != q->tail_idx) {
1328                 desc_info = &q->info[q->tail_idx];
1329                 desc_info->bytes = 0;
1330                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1331                 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
1332                 if (desc_info->cb_arg) {
1333                         pkts++;
1334                         bytes += desc_info->bytes;
1335                 }
1336                 desc_info->cb = NULL;
1337                 desc_info->cb_arg = NULL;
1338         }
1339
1340         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1341                 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q);
1342
1343                 netdev_tx_completed_queue(ndq, pkts, bytes);
1344                 netdev_tx_reset_queue(ndq);
1345         }
1346 }
1347
1348 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
1349 {
1350         int err;
1351
1352         err = skb_cow_head(skb, 0);
1353         if (err)
1354                 return err;
1355
1356         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1357                 inner_ip_hdr(skb)->check = 0;
1358                 inner_tcp_hdr(skb)->check =
1359                         ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
1360                                            inner_ip_hdr(skb)->daddr,
1361                                            0, IPPROTO_TCP, 0);
1362         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1363                 inner_tcp_hdr(skb)->check =
1364                         ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
1365                                          &inner_ipv6_hdr(skb)->daddr,
1366                                          0, IPPROTO_TCP, 0);
1367         }
1368
1369         return 0;
1370 }
1371
1372 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
1373 {
1374         int err;
1375
1376         err = skb_cow_head(skb, 0);
1377         if (err)
1378                 return err;
1379
1380         if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1381                 ip_hdr(skb)->check = 0;
1382                 tcp_hdr(skb)->check =
1383                         ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1384                                            ip_hdr(skb)->daddr,
1385                                            0, IPPROTO_TCP, 0);
1386         } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
1387                 tcp_v6_gso_csum_prep(skb);
1388         }
1389
1390         return 0;
1391 }
1392
1393 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
1394                               struct ionic_desc_info *desc_info,
1395                               struct sk_buff *skb,
1396                               dma_addr_t addr, u8 nsge, u16 len,
1397                               unsigned int hdrlen, unsigned int mss,
1398                               bool outer_csum,
1399                               u16 vlan_tci, bool has_vlan,
1400                               bool start, bool done)
1401 {
1402         struct ionic_txq_desc *desc = desc_info->desc;
1403         u8 flags = 0;
1404         u64 cmd;
1405
1406         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1407         flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1408         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
1409         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
1410
1411         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
1412         desc->cmd = cpu_to_le64(cmd);
1413         desc->len = cpu_to_le16(len);
1414         desc->vlan_tci = cpu_to_le16(vlan_tci);
1415         desc->hdr_len = cpu_to_le16(hdrlen);
1416         desc->mss = cpu_to_le16(mss);
1417
1418         ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
1419
1420         if (start) {
1421                 skb_tx_timestamp(skb);
1422                 if (likely(!ionic_txq_hwstamp_enabled(q)))
1423                         netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len);
1424                 ionic_txq_post(q, false, ionic_tx_clean, skb);
1425         } else {
1426                 ionic_txq_post(q, done, NULL, NULL);
1427         }
1428 }
1429
1430 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
1431                         struct sk_buff *skb)
1432 {
1433         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1434         struct ionic_desc_info *desc_info;
1435         struct ionic_buf_info *buf_info;
1436         struct ionic_txq_sg_elem *elem;
1437         struct ionic_txq_desc *desc;
1438         unsigned int chunk_len;
1439         unsigned int frag_rem;
1440         unsigned int tso_rem;
1441         unsigned int seg_rem;
1442         dma_addr_t desc_addr;
1443         dma_addr_t frag_addr;
1444         unsigned int hdrlen;
1445         unsigned int len;
1446         unsigned int mss;
1447         bool start, done;
1448         bool outer_csum;
1449         bool has_vlan;
1450         u16 desc_len;
1451         u8 desc_nsge;
1452         u16 vlan_tci;
1453         bool encap;
1454         int err;
1455
1456         desc_info = &q->info[q->head_idx];
1457         buf_info = desc_info->bufs;
1458
1459         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1460                 return -EIO;
1461
1462         len = skb->len;
1463         mss = skb_shinfo(skb)->gso_size;
1464         outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1465                                                    SKB_GSO_GRE_CSUM |
1466                                                    SKB_GSO_IPXIP4 |
1467                                                    SKB_GSO_IPXIP6 |
1468                                                    SKB_GSO_UDP_TUNNEL |
1469                                                    SKB_GSO_UDP_TUNNEL_CSUM));
1470         has_vlan = !!skb_vlan_tag_present(skb);
1471         vlan_tci = skb_vlan_tag_get(skb);
1472         encap = skb->encapsulation;
1473
1474         /* Preload inner-most TCP csum field with IP pseudo hdr
1475          * calculated with IP length set to zero.  HW will later
1476          * add in length to each TCP segment resulting from the TSO.
1477          */
1478
1479         if (encap)
1480                 err = ionic_tx_tcp_inner_pseudo_csum(skb);
1481         else
1482                 err = ionic_tx_tcp_pseudo_csum(skb);
1483         if (err) {
1484                 /* clean up mapping from ionic_tx_map_skb */
1485                 ionic_tx_desc_unmap_bufs(q, desc_info);
1486                 return err;
1487         }
1488
1489         if (encap)
1490                 hdrlen = skb_inner_tcp_all_headers(skb);
1491         else
1492                 hdrlen = skb_tcp_all_headers(skb);
1493
1494         tso_rem = len;
1495         seg_rem = min(tso_rem, hdrlen + mss);
1496
1497         frag_addr = 0;
1498         frag_rem = 0;
1499
1500         start = true;
1501
1502         while (tso_rem > 0) {
1503                 desc = NULL;
1504                 elem = NULL;
1505                 desc_addr = 0;
1506                 desc_len = 0;
1507                 desc_nsge = 0;
1508                 /* use fragments until we have enough to post a single descriptor */
1509                 while (seg_rem > 0) {
1510                         /* if the fragment is exhausted then move to the next one */
1511                         if (frag_rem == 0) {
1512                                 /* grab the next fragment */
1513                                 frag_addr = buf_info->dma_addr;
1514                                 frag_rem = buf_info->len;
1515                                 buf_info++;
1516                         }
1517                         chunk_len = min(frag_rem, seg_rem);
1518                         if (!desc) {
1519                                 /* fill main descriptor */
1520                                 desc = desc_info->txq_desc;
1521                                 elem = desc_info->txq_sg_desc->elems;
1522                                 desc_addr = frag_addr;
1523                                 desc_len = chunk_len;
1524                         } else {
1525                                 /* fill sg descriptor */
1526                                 elem->addr = cpu_to_le64(frag_addr);
1527                                 elem->len = cpu_to_le16(chunk_len);
1528                                 elem++;
1529                                 desc_nsge++;
1530                         }
1531                         frag_addr += chunk_len;
1532                         frag_rem -= chunk_len;
1533                         tso_rem -= chunk_len;
1534                         seg_rem -= chunk_len;
1535                 }
1536                 seg_rem = min(tso_rem, mss);
1537                 done = (tso_rem == 0);
1538                 /* post descriptor */
1539                 ionic_tx_tso_post(netdev, q, desc_info, skb,
1540                                   desc_addr, desc_nsge, desc_len,
1541                                   hdrlen, mss, outer_csum, vlan_tci, has_vlan,
1542                                   start, done);
1543                 start = false;
1544                 /* Buffer information is stored with the first tso descriptor */
1545                 desc_info = &q->info[q->head_idx];
1546                 desc_info->nbufs = 0;
1547         }
1548
1549         stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
1550         stats->bytes += len;
1551         stats->tso++;
1552         stats->tso_bytes = len;
1553
1554         return 0;
1555 }
1556
1557 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
1558                                struct ionic_desc_info *desc_info)
1559 {
1560         struct ionic_txq_desc *desc = desc_info->txq_desc;
1561         struct ionic_buf_info *buf_info = desc_info->bufs;
1562         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1563         bool has_vlan;
1564         u8 flags = 0;
1565         bool encap;
1566         u64 cmd;
1567
1568         has_vlan = !!skb_vlan_tag_present(skb);
1569         encap = skb->encapsulation;
1570
1571         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1572         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1573
1574         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
1575                                   flags, skb_shinfo(skb)->nr_frags,
1576                                   buf_info->dma_addr);
1577         desc->cmd = cpu_to_le64(cmd);
1578         desc->len = cpu_to_le16(buf_info->len);
1579         if (has_vlan) {
1580                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1581                 stats->vlan_inserted++;
1582         } else {
1583                 desc->vlan_tci = 0;
1584         }
1585         desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
1586         desc->csum_offset = cpu_to_le16(skb->csum_offset);
1587
1588         ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
1589
1590         if (skb_csum_is_sctp(skb))
1591                 stats->crc32_csum++;
1592         else
1593                 stats->csum++;
1594 }
1595
1596 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
1597                                   struct ionic_desc_info *desc_info)
1598 {
1599         struct ionic_txq_desc *desc = desc_info->txq_desc;
1600         struct ionic_buf_info *buf_info = desc_info->bufs;
1601         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1602         bool has_vlan;
1603         u8 flags = 0;
1604         bool encap;
1605         u64 cmd;
1606
1607         has_vlan = !!skb_vlan_tag_present(skb);
1608         encap = skb->encapsulation;
1609
1610         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1611         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1612
1613         cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
1614                                   flags, skb_shinfo(skb)->nr_frags,
1615                                   buf_info->dma_addr);
1616         desc->cmd = cpu_to_le64(cmd);
1617         desc->len = cpu_to_le16(buf_info->len);
1618         if (has_vlan) {
1619                 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1620                 stats->vlan_inserted++;
1621         } else {
1622                 desc->vlan_tci = 0;
1623         }
1624         desc->csum_start = 0;
1625         desc->csum_offset = 0;
1626
1627         ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
1628
1629         stats->csum_none++;
1630 }
1631
1632 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
1633                                struct ionic_desc_info *desc_info)
1634 {
1635         struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
1636         struct ionic_buf_info *buf_info = &desc_info->bufs[1];
1637         struct ionic_txq_sg_elem *elem = sg_desc->elems;
1638         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1639         unsigned int i;
1640
1641         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
1642                 elem->addr = cpu_to_le64(buf_info->dma_addr);
1643                 elem->len = cpu_to_le16(buf_info->len);
1644         }
1645
1646         stats->frags += skb_shinfo(skb)->nr_frags;
1647 }
1648
1649 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q,
1650                     struct sk_buff *skb)
1651 {
1652         struct ionic_desc_info *desc_info = &q->info[q->head_idx];
1653         struct ionic_tx_stats *stats = q_to_tx_stats(q);
1654         bool ring_dbell = true;
1655
1656         if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1657                 return -EIO;
1658
1659         /* set up the initial descriptor */
1660         if (skb->ip_summed == CHECKSUM_PARTIAL)
1661                 ionic_tx_calc_csum(q, skb, desc_info);
1662         else
1663                 ionic_tx_calc_no_csum(q, skb, desc_info);
1664
1665         /* add frags */
1666         ionic_tx_skb_frags(q, skb, desc_info);
1667
1668         skb_tx_timestamp(skb);
1669         stats->pkts++;
1670         stats->bytes += skb->len;
1671
1672         if (likely(!ionic_txq_hwstamp_enabled(q))) {
1673                 struct netdev_queue *ndq = q_to_ndq(netdev, q);
1674
1675                 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1)))
1676                         netif_tx_stop_queue(ndq);
1677                 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len,
1678                                                     netdev_xmit_more());
1679         }
1680         ionic_txq_post(q, ring_dbell, ionic_tx_clean, skb);
1681
1682         return 0;
1683 }
1684
1685 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1686 {
1687         int nr_frags = skb_shinfo(skb)->nr_frags;
1688         bool too_many_frags = false;
1689         skb_frag_t *frag;
1690         int desc_bufs;
1691         int chunk_len;
1692         int frag_rem;
1693         int tso_rem;
1694         int seg_rem;
1695         bool encap;
1696         int hdrlen;
1697         int ndescs;
1698         int err;
1699
1700         /* Each desc is mss long max, so a descriptor for each gso_seg */
1701         if (skb_is_gso(skb)) {
1702                 ndescs = skb_shinfo(skb)->gso_segs;
1703                 if (!nr_frags)
1704                         return ndescs;
1705         } else {
1706                 ndescs = 1;
1707                 if (!nr_frags)
1708                         return ndescs;
1709
1710                 if (unlikely(nr_frags > q->max_sg_elems)) {
1711                         too_many_frags = true;
1712                         goto linearize;
1713                 }
1714
1715                 return ndescs;
1716         }
1717
1718         /* We need to scan the skb to be sure that none of the MTU sized
1719          * packets in the TSO will require more sgs per descriptor than we
1720          * can support.  We loop through the frags, add up the lengths for
1721          * a packet, and count the number of sgs used per packet.
1722          */
1723         tso_rem = skb->len;
1724         frag = skb_shinfo(skb)->frags;
1725         encap = skb->encapsulation;
1726
1727         /* start with just hdr in first part of first descriptor */
1728         if (encap)
1729                 hdrlen = skb_inner_tcp_all_headers(skb);
1730         else
1731                 hdrlen = skb_tcp_all_headers(skb);
1732         seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size);
1733         frag_rem = hdrlen;
1734
1735         while (tso_rem > 0) {
1736                 desc_bufs = 0;
1737                 while (seg_rem > 0) {
1738                         desc_bufs++;
1739
1740                         /* We add the +1 because we can take buffers for one
1741                          * more than we have SGs: one for the initial desc data
1742                          * in addition to the SG segments that might follow.
1743                          */
1744                         if (desc_bufs > q->max_sg_elems + 1) {
1745                                 too_many_frags = true;
1746                                 goto linearize;
1747                         }
1748
1749                         if (frag_rem == 0) {
1750                                 frag_rem = skb_frag_size(frag);
1751                                 frag++;
1752                         }
1753                         chunk_len = min(frag_rem, seg_rem);
1754                         frag_rem -= chunk_len;
1755                         tso_rem -= chunk_len;
1756                         seg_rem -= chunk_len;
1757                 }
1758
1759                 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size);
1760         }
1761
1762 linearize:
1763         if (too_many_frags) {
1764                 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1765
1766                 err = skb_linearize(skb);
1767                 if (err)
1768                         return err;
1769                 stats->linearize++;
1770         }
1771
1772         return ndescs;
1773 }
1774
1775 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1776                                             struct net_device *netdev)
1777 {
1778         struct ionic_lif *lif = netdev_priv(netdev);
1779         struct ionic_queue *q;
1780         int err, ndescs;
1781
1782         /* Does not stop/start txq, because we post to a separate tx queue
1783          * for timestamping, and if a packet can't be posted immediately to
1784          * the timestamping queue, it is dropped.
1785          */
1786
1787         q = &lif->hwstamp_txq->q;
1788         ndescs = ionic_tx_descs_needed(q, skb);
1789         if (unlikely(ndescs < 0))
1790                 goto err_out_drop;
1791
1792         if (unlikely(!ionic_q_has_space(q, ndescs)))
1793                 goto err_out_drop;
1794
1795         skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1796         if (skb_is_gso(skb))
1797                 err = ionic_tx_tso(netdev, q, skb);
1798         else
1799                 err = ionic_tx(netdev, q, skb);
1800
1801         if (err)
1802                 goto err_out_drop;
1803
1804         return NETDEV_TX_OK;
1805
1806 err_out_drop:
1807         q->drop++;
1808         dev_kfree_skb(skb);
1809         return NETDEV_TX_OK;
1810 }
1811
1812 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1813 {
1814         u16 queue_index = skb_get_queue_mapping(skb);
1815         struct ionic_lif *lif = netdev_priv(netdev);
1816         struct ionic_queue *q;
1817         int ndescs;
1818         int err;
1819
1820         if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1821                 dev_kfree_skb(skb);
1822                 return NETDEV_TX_OK;
1823         }
1824
1825         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1826                 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1827                         return ionic_start_hwstamp_xmit(skb, netdev);
1828
1829         if (unlikely(queue_index >= lif->nxqs))
1830                 queue_index = 0;
1831         q = &lif->txqcqs[queue_index]->q;
1832
1833         ndescs = ionic_tx_descs_needed(q, skb);
1834         if (ndescs < 0)
1835                 goto err_out_drop;
1836
1837         if (!netif_txq_maybe_stop(q_to_ndq(netdev, q),
1838                                   ionic_q_space_avail(q),
1839                                   ndescs, ndescs))
1840                 return NETDEV_TX_BUSY;
1841
1842         if (skb_is_gso(skb))
1843                 err = ionic_tx_tso(netdev, q, skb);
1844         else
1845                 err = ionic_tx(netdev, q, skb);
1846
1847         if (err)
1848                 goto err_out_drop;
1849
1850         return NETDEV_TX_OK;
1851
1852 err_out_drop:
1853         q->drop++;
1854         dev_kfree_skb(skb);
1855         return NETDEV_TX_OK;
1856 }