drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
   3
   4 #include <generated/utsrelease.h>
   5 #include <linux/crash_dump.h>
   6 #include <linux/if_bridge.h>
   7 #include <linux/if_macvlan.h>
   8 #include <linux/module.h>
   9 #include <net/pkt_cls.h>
  10 #include <net/xdp_sock_drv.h>
  11
  12 /* Local includes */
  13 #include "i40e.h"
  14 #include "i40e_devids.h"
  15 #include "i40e_diag.h"
  16 #include "i40e_lan_hmc.h"
  17 #include "i40e_virtchnl_pf.h"
  18 #include "i40e_xsk.h"
  19
  20 /* All i40e tracepoints are defined by the include below, which
  21  * must be included exactly once across the whole kernel with
  22  * CREATE_TRACE_POINTS defined
  23  */
  24 #define CREATE_TRACE_POINTS
  25 #include "i40e_trace.h"
  26
  27 const char i40e_driver_name[] = "i40e";
  28 static const char i40e_driver_string[] =
  29                         "Intel(R) Ethernet Connection XL710 Network Driver";
  30
  31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
  32
  33 /* a bit of forward declarations */
  34 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  35 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  36 static int i40e_add_vsi(struct i40e_vsi *vsi);
  37 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  38 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  39 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  40 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  41 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  42 static void i40e_prep_for_reset(struct i40e_pf *pf);
  43 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
  44                                    bool lock_acquired);
  45 static int i40e_reset(struct i40e_pf *pf);
  46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  47 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
  48 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
  49 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
  50 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
  51 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  52 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  53 static int i40e_get_capabilities(struct i40e_pf *pf,
  54                                  enum i40e_admin_queue_opc list_type);
  55 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
  56
  57 /* i40e_pci_tbl - PCI Device ID Table
  58  *
  59  * Last entry must be all 0s
  60  *
  61  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  62  *   Class, Class Mask, private data (not used) }
  63  */
  64 static const struct pci_device_id i40e_pci_tbl[] = {
  65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  91         /* required last entry */
  92         {0, }
  93 };
  94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  95
  96 #define I40E_MAX_VF_COUNT 128
  97 static int debug = -1;
  98 module_param(debug, uint, 0);
  99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 100
 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 103 MODULE_LICENSE("GPL v2");
 104
 105 static struct workqueue_struct *i40e_wq;
 106
 107 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
 108                                   struct net_device *netdev, int delta)
 109 {
 110         struct netdev_hw_addr_list *ha_list;
 111         struct netdev_hw_addr *ha;
 112
 113         if (!f || !netdev)
 114                 return;
 115
 116         if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
 117                 ha_list = &netdev->uc;
 118         else
 119                 ha_list = &netdev->mc;
 120
 121         netdev_hw_addr_list_for_each(ha, ha_list) {
 122                 if (ether_addr_equal(ha->addr, f->macaddr)) {
 123                         ha->refcount += delta;
 124                         if (ha->refcount <= 0)
 125                                 ha->refcount = 1;
 126                         break;
 127                 }
 128         }
 129 }
 130
 131 /**
 132  * i40e_hw_to_dev - get device pointer from the hardware structure
 133  * @hw: pointer to the device HW structure
 134  **/
 135 struct device *i40e_hw_to_dev(struct i40e_hw *hw)
 136 {
 137         struct i40e_pf *pf = i40e_hw_to_pf(hw);
 138
 139         return &pf->pdev->dev;
 140 }
 141
 142 /**
 143  * i40e_allocate_dma_mem - OS specific memory alloc for shared code
 144  * @hw:   pointer to the HW structure
 145  * @mem:  ptr to mem struct to fill out
 146  * @size: size of memory requested
 147  * @alignment: what to align the allocation to
 148  **/
 149 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 150                           u64 size, u32 alignment)
 151 {
 152         struct i40e_pf *pf = i40e_hw_to_pf(hw);
 153
 154         mem->size = ALIGN(size, alignment);
 155         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
 156                                      GFP_KERNEL);
 157         if (!mem->va)
 158                 return -ENOMEM;
 159
 160         return 0;
 161 }
 162
 163 /**
 164  * i40e_free_dma_mem - OS specific memory free for shared code
 165  * @hw:   pointer to the HW structure
 166  * @mem:  ptr to mem struct to free
 167  **/
 168 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 169 {
 170         struct i40e_pf *pf = i40e_hw_to_pf(hw);
 171
 172         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 173         mem->va = NULL;
 174         mem->pa = 0;
 175         mem->size = 0;
 176
 177         return 0;
 178 }
 179
 180 /**
 181  * i40e_allocate_virt_mem - OS specific memory alloc for shared code
 182  * @hw:   pointer to the HW structure
 183  * @mem:  ptr to mem struct to fill out
 184  * @size: size of memory requested
 185  **/
 186 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 187                            u32 size)
 188 {
 189         mem->size = size;
 190         mem->va = kzalloc(size, GFP_KERNEL);
 191
 192         if (!mem->va)
 193                 return -ENOMEM;
 194
 195         return 0;
 196 }
 197
 198 /**
 199  * i40e_free_virt_mem - OS specific memory free for shared code
 200  * @hw:   pointer to the HW structure
 201  * @mem:  ptr to mem struct to free
 202  **/
 203 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 204 {
 205         /* it's ok to kfree a NULL pointer */
 206         kfree(mem->va);
 207         mem->va = NULL;
 208         mem->size = 0;
 209
 210         return 0;
 211 }
 212
 213 /**
 214  * i40e_get_lump - find a lump of free generic resource
 215  * @pf: board private structure
 216  * @pile: the pile of resource to search
 217  * @needed: the number of items needed
 218  * @id: an owner id to stick on the items assigned
 219  *
 220  * Returns the base item index of the lump, or negative for error
 221  **/
 222 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 223                          u16 needed, u16 id)
 224 {
 225         int ret = -ENOMEM;
 226         int i, j;
 227
 228         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 229                 dev_info(&pf->pdev->dev,
 230                          "param err: pile=%s needed=%d id=0x%04x\n",
 231                          pile ? "<valid>" : "<null>", needed, id);
 232                 return -EINVAL;
 233         }
 234
 235         /* Allocate last queue in the pile for FDIR VSI queue
 236          * so it doesn't fragment the qp_pile
 237          */
 238         if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
 239                 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
 240                         dev_err(&pf->pdev->dev,
 241                                 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
 242                                 pile->num_entries - 1);
 243                         return -ENOMEM;
 244                 }
 245                 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
 246                 return pile->num_entries - 1;
 247         }
 248
 249         i = 0;
 250         while (i < pile->num_entries) {
 251                 /* skip already allocated entries */
 252                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 253                         i++;
 254                         continue;
 255                 }
 256
 257                 /* do we have enough in this lump? */
 258                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 259                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 260                                 break;
 261                 }
 262
 263                 if (j == needed) {
 264                         /* there was enough, so assign it to the requestor */
 265                         for (j = 0; j < needed; j++)
 266                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 267                         ret = i;
 268                         break;
 269                 }
 270
 271                 /* not enough, so skip over it and continue looking */
 272                 i += j;
 273         }
 274
 275         return ret;
 276 }
 277
 278 /**
 279  * i40e_put_lump - return a lump of generic resource
 280  * @pile: the pile of resource to search
 281  * @index: the base item index
 282  * @id: the owner id of the items assigned
 283  *
 284  * Returns the count of items in the lump
 285  **/
 286 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 287 {
 288         int valid_id = (id | I40E_PILE_VALID_BIT);
 289         int count = 0;
 290         u16 i;
 291
 292         if (!pile || index >= pile->num_entries)
 293                 return -EINVAL;
 294
 295         for (i = index;
 296              i < pile->num_entries && pile->list[i] == valid_id;
 297              i++) {
 298                 pile->list[i] = 0;
 299                 count++;
 300         }
 301
 302
 303         return count;
 304 }
 305
 306 /**
 307  * i40e_find_vsi_from_id - searches for the vsi with the given id
 308  * @pf: the pf structure to search for the vsi
 309  * @id: id of the vsi it is searching for
 310  **/
 311 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 312 {
 313         struct i40e_vsi *vsi;
 314         int i;
 315
 316         i40e_pf_for_each_vsi(pf, i, vsi)
 317                 if (vsi->id == id)
 318                         return vsi;
 319
 320         return NULL;
 321 }
 322
 323 /**
 324  * i40e_service_event_schedule - Schedule the service task to wake up
 325  * @pf: board private structure
 326  *
 327  * If not already scheduled, this puts the task into the work queue
 328  **/
 329 void i40e_service_event_schedule(struct i40e_pf *pf)
 330 {
 331         if ((!test_bit(__I40E_DOWN, pf->state) &&
 332              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
 333               test_bit(__I40E_RECOVERY_MODE, pf->state))
 334                 queue_work(i40e_wq, &pf->service_task);
 335 }
 336
 337 /**
 338  * i40e_tx_timeout - Respond to a Tx Hang
 339  * @netdev: network interface device structure
 340  * @txqueue: queue number timing out
 341  *
 342  * If any port has noticed a Tx timeout, it is likely that the whole
 343  * device is munged, not just the one netdev port, so go for the full
 344  * reset.
 345  **/
 346 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 347 {
 348         struct i40e_netdev_priv *np = netdev_priv(netdev);
 349         struct i40e_vsi *vsi = np->vsi;
 350         struct i40e_pf *pf = vsi->back;
 351         struct i40e_ring *tx_ring = NULL;
 352         unsigned int i;
 353         u32 head, val;
 354
 355         pf->tx_timeout_count++;
 356
 357         /* with txqueue index, find the tx_ring struct */
 358         for (i = 0; i < vsi->num_queue_pairs; i++) {
 359                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 360                         if (txqueue ==
 361                             vsi->tx_rings[i]->queue_index) {
 362                                 tx_ring = vsi->tx_rings[i];
 363                                 break;
 364                         }
 365                 }
 366         }
 367
 368         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 369                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 370         else if (time_before(jiffies,
 371                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 372                 return;   /* don't do any new action before the next timeout */
 373
 374         /* don't kick off another recovery if one is already pending */
 375         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 376                 return;
 377
 378         if (tx_ring) {
 379                 head = i40e_get_head(tx_ring);
 380                 /* Read interrupt register */
 381                 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 382                         val = rd32(&pf->hw,
 383                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 384                                                 tx_ring->vsi->base_vector - 1));
 385                 else
 386                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 387
 388                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 389                             vsi->seid, txqueue, tx_ring->next_to_clean,
 390                             head, tx_ring->next_to_use,
 391                             readl(tx_ring->tail), val);
 392         }
 393
 394         pf->tx_timeout_last_recovery = jiffies;
 395         netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
 396                     pf->tx_timeout_recovery_level, txqueue);
 397
 398         switch (pf->tx_timeout_recovery_level) {
 399         case 1:
 400                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 401                 break;
 402         case 2:
 403                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 404                 break;
 405         case 3:
 406                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 407                 break;
 408         default:
 409                 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
 410                 set_bit(__I40E_DOWN_REQUESTED, pf->state);
 411                 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
 412                 break;
 413         }
 414
 415         i40e_service_event_schedule(pf);
 416         pf->tx_timeout_recovery_level++;
 417 }
 418
 419 /**
 420  * i40e_get_vsi_stats_struct - Get System Network Statistics
 421  * @vsi: the VSI we care about
 422  *
 423  * Returns the address of the device statistics structure.
 424  * The statistics are actually updated from the service task.
 425  **/
 426 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 427 {
 428         return &vsi->net_stats;
 429 }
 430
 431 /**
 432  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 433  * @ring: Tx ring to get statistics from
 434  * @stats: statistics entry to be updated
 435  **/
 436 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 437                                             struct rtnl_link_stats64 *stats)
 438 {
 439         u64 bytes, packets;
 440         unsigned int start;
 441
 442         do {
 443                 start = u64_stats_fetch_begin(&ring->syncp);
 444                 packets = ring->stats.packets;
 445                 bytes   = ring->stats.bytes;
 446         } while (u64_stats_fetch_retry(&ring->syncp, start));
 447
 448         stats->tx_packets += packets;
 449         stats->tx_bytes   += bytes;
 450 }
 451
 452 /**
 453  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 454  * @netdev: network interface device structure
 455  * @stats: data structure to store statistics
 456  *
 457  * Returns the address of the device statistics structure.
 458  * The statistics are actually updated from the service task.
 459  **/
 460 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 461                                   struct rtnl_link_stats64 *stats)
 462 {
 463         struct i40e_netdev_priv *np = netdev_priv(netdev);
 464         struct i40e_vsi *vsi = np->vsi;
 465         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 466         struct i40e_ring *ring;
 467         int i;
 468
 469         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 470                 return;
 471
 472         if (!vsi->tx_rings)
 473                 return;
 474
 475         rcu_read_lock();
 476         for (i = 0; i < vsi->num_queue_pairs; i++) {
 477                 u64 bytes, packets;
 478                 unsigned int start;
 479
 480                 ring = READ_ONCE(vsi->tx_rings[i]);
 481                 if (!ring)
 482                         continue;
 483                 i40e_get_netdev_stats_struct_tx(ring, stats);
 484
 485                 if (i40e_enabled_xdp_vsi(vsi)) {
 486                         ring = READ_ONCE(vsi->xdp_rings[i]);
 487                         if (!ring)
 488                                 continue;
 489                         i40e_get_netdev_stats_struct_tx(ring, stats);
 490                 }
 491
 492                 ring = READ_ONCE(vsi->rx_rings[i]);
 493                 if (!ring)
 494                         continue;
 495                 do {
 496                         start   = u64_stats_fetch_begin(&ring->syncp);
 497                         packets = ring->stats.packets;
 498                         bytes   = ring->stats.bytes;
 499                 } while (u64_stats_fetch_retry(&ring->syncp, start));
 500
 501                 stats->rx_packets += packets;
 502                 stats->rx_bytes   += bytes;
 503
 504         }
 505         rcu_read_unlock();
 506
 507         /* following stats updated by i40e_watchdog_subtask() */
 508         stats->multicast        = vsi_stats->multicast;
 509         stats->tx_errors        = vsi_stats->tx_errors;
 510         stats->tx_dropped       = vsi_stats->tx_dropped;
 511         stats->rx_errors        = vsi_stats->rx_errors;
 512         stats->rx_dropped       = vsi_stats->rx_dropped;
 513         stats->rx_missed_errors = vsi_stats->rx_missed_errors;
 514         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 515         stats->rx_length_errors = vsi_stats->rx_length_errors;
 516 }
 517
 518 /**
 519  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 520  * @vsi: the VSI to have its stats reset
 521  **/
 522 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 523 {
 524         struct rtnl_link_stats64 *ns;
 525         int i;
 526
 527         if (!vsi)
 528                 return;
 529
 530         ns = i40e_get_vsi_stats_struct(vsi);
 531         memset(ns, 0, sizeof(*ns));
 532         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 533         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 534         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 535         if (vsi->rx_rings && vsi->rx_rings[0]) {
 536                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 537                         memset(&vsi->rx_rings[i]->stats, 0,
 538                                sizeof(vsi->rx_rings[i]->stats));
 539                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 540                                sizeof(vsi->rx_rings[i]->rx_stats));
 541                         memset(&vsi->tx_rings[i]->stats, 0,
 542                                sizeof(vsi->tx_rings[i]->stats));
 543                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 544                                sizeof(vsi->tx_rings[i]->tx_stats));
 545                 }
 546         }
 547         vsi->stat_offsets_loaded = false;
 548 }
 549
 550 /**
 551  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 552  * @pf: the PF to be reset
 553  **/
 554 void i40e_pf_reset_stats(struct i40e_pf *pf)
 555 {
 556         struct i40e_veb *veb;
 557         int i;
 558
 559         memset(&pf->stats, 0, sizeof(pf->stats));
 560         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 561         pf->stat_offsets_loaded = false;
 562
 563         i40e_pf_for_each_veb(pf, i, veb) {
 564                 memset(&veb->stats, 0, sizeof(veb->stats));
 565                 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));
 566                 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));
 567                 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));
 568                 veb->stat_offsets_loaded = false;
 569         }
 570         pf->hw_csum_rx_error = 0;
 571 }
 572
 573 /**
 574  * i40e_compute_pci_to_hw_id - compute index form PCI function.
 575  * @vsi: ptr to the VSI to read from.
 576  * @hw: ptr to the hardware info.
 577  **/
 578 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
 579 {
 580         int pf_count = i40e_get_pf_count(hw);
 581
 582         if (vsi->type == I40E_VSI_SRIOV)
 583                 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
 584
 585         return hw->port + BIT(7);
 586 }
 587
 588 /**
 589  * i40e_stat_update64 - read and update a 64 bit stat from the chip.
 590  * @hw: ptr to the hardware info.
 591  * @hireg: the high 32 bit reg to read.
 592  * @loreg: the low 32 bit reg to read.
 593  * @offset_loaded: has the initial offset been loaded yet.
 594  * @offset: ptr to current offset value.
 595  * @stat: ptr to the stat.
 596  *
 597  * Since the device stats are not reset at PFReset, they will not
 598  * be zeroed when the driver starts.  We'll save the first values read
 599  * and use them as offsets to be subtracted from the raw values in order
 600  * to report stats that count from zero.
 601  **/
 602 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
 603                                bool offset_loaded, u64 *offset, u64 *stat)
 604 {
 605         u64 new_data;
 606
 607         new_data = rd64(hw, loreg);
 608
 609         if (!offset_loaded || new_data < *offset)
 610                 *offset = new_data;
 611         *stat = new_data - *offset;
 612 }
 613
 614 /**
 615  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 616  * @hw: ptr to the hardware info
 617  * @hireg: the high 32 bit reg to read
 618  * @loreg: the low 32 bit reg to read
 619  * @offset_loaded: has the initial offset been loaded yet
 620  * @offset: ptr to current offset value
 621  * @stat: ptr to the stat
 622  *
 623  * Since the device stats are not reset at PFReset, they likely will not
 624  * be zeroed when the driver starts.  We'll save the first values read
 625  * and use them as offsets to be subtracted from the raw values in order
 626  * to report stats that count from zero.  In the process, we also manage
 627  * the potential roll-over.
 628  **/
 629 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 630                                bool offset_loaded, u64 *offset, u64 *stat)
 631 {
 632         u64 new_data;
 633
 634         if (hw->device_id == I40E_DEV_ID_QEMU) {
 635                 new_data = rd32(hw, loreg);
 636                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 637         } else {
 638                 new_data = rd64(hw, loreg);
 639         }
 640         if (!offset_loaded)
 641                 *offset = new_data;
 642         if (likely(new_data >= *offset))
 643                 *stat = new_data - *offset;
 644         else
 645                 *stat = (new_data + BIT_ULL(48)) - *offset;
 646         *stat &= 0xFFFFFFFFFFFFULL;
 647 }
 648
 649 /**
 650  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 651  * @hw: ptr to the hardware info
 652  * @reg: the hw reg to read
 653  * @offset_loaded: has the initial offset been loaded yet
 654  * @offset: ptr to current offset value
 655  * @stat: ptr to the stat
 656  **/
 657 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 658                                bool offset_loaded, u64 *offset, u64 *stat)
 659 {
 660         u32 new_data;
 661
 662         new_data = rd32(hw, reg);
 663         if (!offset_loaded)
 664                 *offset = new_data;
 665         if (likely(new_data >= *offset))
 666                 *stat = (u32)(new_data - *offset);
 667         else
 668                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 669 }
 670
 671 /**
 672  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 673  * @hw: ptr to the hardware info
 674  * @reg: the hw reg to read and clear
 675  * @stat: ptr to the stat
 676  **/
 677 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 678 {
 679         u32 new_data = rd32(hw, reg);
 680
 681         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 682         *stat += new_data;
 683 }
 684
 685 /**
 686  * i40e_stats_update_rx_discards - update rx_discards.
 687  * @vsi: ptr to the VSI to be updated.
 688  * @hw: ptr to the hardware info.
 689  * @stat_idx: VSI's stat_counter_idx.
 690  * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
 691  * @stat_offset: ptr to stat_offset to store first read of specific register.
 692  * @stat: ptr to VSI's stat to be updated.
 693  **/
 694 static void
 695 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
 696                               int stat_idx, bool offset_loaded,
 697                               struct i40e_eth_stats *stat_offset,
 698                               struct i40e_eth_stats *stat)
 699 {
 700         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
 701                            &stat_offset->rx_discards, &stat->rx_discards);
 702         i40e_stat_update64(hw,
 703                            I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
 704                            I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
 705                            offset_loaded, &stat_offset->rx_discards_other,
 706                            &stat->rx_discards_other);
 707 }
 708
 709 /**
 710  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 711  * @vsi: the VSI to be updated
 712  **/
 713 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 714 {
 715         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 716         struct i40e_pf *pf = vsi->back;
 717         struct i40e_hw *hw = &pf->hw;
 718         struct i40e_eth_stats *oes;
 719         struct i40e_eth_stats *es;     /* device's eth stats */
 720
 721         es = &vsi->eth_stats;
 722         oes = &vsi->eth_stats_offsets;
 723
 724         /* Gather up the stats that the hw collects */
 725         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 726                            vsi->stat_offsets_loaded,
 727                            &oes->tx_errors, &es->tx_errors);
 728         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 729                            vsi->stat_offsets_loaded,
 730                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 731
 732         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 733                            I40E_GLV_GORCL(stat_idx),
 734                            vsi->stat_offsets_loaded,
 735                            &oes->rx_bytes, &es->rx_bytes);
 736         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 737                            I40E_GLV_UPRCL(stat_idx),
 738                            vsi->stat_offsets_loaded,
 739                            &oes->rx_unicast, &es->rx_unicast);
 740         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 741                            I40E_GLV_MPRCL(stat_idx),
 742                            vsi->stat_offsets_loaded,
 743                            &oes->rx_multicast, &es->rx_multicast);
 744         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 745                            I40E_GLV_BPRCL(stat_idx),
 746                            vsi->stat_offsets_loaded,
 747                            &oes->rx_broadcast, &es->rx_broadcast);
 748
 749         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 750                            I40E_GLV_GOTCL(stat_idx),
 751                            vsi->stat_offsets_loaded,
 752                            &oes->tx_bytes, &es->tx_bytes);
 753         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 754                            I40E_GLV_UPTCL(stat_idx),
 755                            vsi->stat_offsets_loaded,
 756                            &oes->tx_unicast, &es->tx_unicast);
 757         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 758                            I40E_GLV_MPTCL(stat_idx),
 759                            vsi->stat_offsets_loaded,
 760                            &oes->tx_multicast, &es->tx_multicast);
 761         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 762                            I40E_GLV_BPTCL(stat_idx),
 763                            vsi->stat_offsets_loaded,
 764                            &oes->tx_broadcast, &es->tx_broadcast);
 765
 766         i40e_stats_update_rx_discards(vsi, hw, stat_idx,
 767                                       vsi->stat_offsets_loaded, oes, es);
 768
 769         vsi->stat_offsets_loaded = true;
 770 }
 771
 772 /**
 773  * i40e_update_veb_stats - Update Switch component statistics
 774  * @veb: the VEB being updated
 775  **/
 776 void i40e_update_veb_stats(struct i40e_veb *veb)
 777 {
 778         struct i40e_pf *pf = veb->pf;
 779         struct i40e_hw *hw = &pf->hw;
 780         struct i40e_eth_stats *oes;
 781         struct i40e_eth_stats *es;     /* device's eth stats */
 782         struct i40e_veb_tc_stats *veb_oes;
 783         struct i40e_veb_tc_stats *veb_es;
 784         int i, idx = 0;
 785
 786         idx = veb->stats_idx;
 787         es = &veb->stats;
 788         oes = &veb->stats_offsets;
 789         veb_es = &veb->tc_stats;
 790         veb_oes = &veb->tc_stats_offsets;
 791
 792         /* Gather up the stats that the hw collects */
 793         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 794                            veb->stat_offsets_loaded,
 795                            &oes->tx_discards, &es->tx_discards);
 796         if (hw->revision_id > 0)
 797                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 798                                    veb->stat_offsets_loaded,
 799                                    &oes->rx_unknown_protocol,
 800                                    &es->rx_unknown_protocol);
 801         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 802                            veb->stat_offsets_loaded,
 803                            &oes->rx_bytes, &es->rx_bytes);
 804         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 805                            veb->stat_offsets_loaded,
 806                            &oes->rx_unicast, &es->rx_unicast);
 807         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 808                            veb->stat_offsets_loaded,
 809                            &oes->rx_multicast, &es->rx_multicast);
 810         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 811                            veb->stat_offsets_loaded,
 812                            &oes->rx_broadcast, &es->rx_broadcast);
 813
 814         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 815                            veb->stat_offsets_loaded,
 816                            &oes->tx_bytes, &es->tx_bytes);
 817         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 818                            veb->stat_offsets_loaded,
 819                            &oes->tx_unicast, &es->tx_unicast);
 820         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 821                            veb->stat_offsets_loaded,
 822                            &oes->tx_multicast, &es->tx_multicast);
 823         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 824                            veb->stat_offsets_loaded,
 825                            &oes->tx_broadcast, &es->tx_broadcast);
 826         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 827                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 828                                    I40E_GLVEBTC_RPCL(i, idx),
 829                                    veb->stat_offsets_loaded,
 830                                    &veb_oes->tc_rx_packets[i],
 831                                    &veb_es->tc_rx_packets[i]);
 832                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 833                                    I40E_GLVEBTC_RBCL(i, idx),
 834                                    veb->stat_offsets_loaded,
 835                                    &veb_oes->tc_rx_bytes[i],
 836                                    &veb_es->tc_rx_bytes[i]);
 837                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 838                                    I40E_GLVEBTC_TPCL(i, idx),
 839                                    veb->stat_offsets_loaded,
 840                                    &veb_oes->tc_tx_packets[i],
 841                                    &veb_es->tc_tx_packets[i]);
 842                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 843                                    I40E_GLVEBTC_TBCL(i, idx),
 844                                    veb->stat_offsets_loaded,
 845                                    &veb_oes->tc_tx_bytes[i],
 846                                    &veb_es->tc_tx_bytes[i]);
 847         }
 848         veb->stat_offsets_loaded = true;
 849 }
 850
 851 /**
 852  * i40e_update_vsi_stats - Update the vsi statistics counters.
 853  * @vsi: the VSI to be updated
 854  *
 855  * There are a few instances where we store the same stat in a
 856  * couple of different structs.  This is partly because we have
 857  * the netdev stats that need to be filled out, which is slightly
 858  * different from the "eth_stats" defined by the chip and used in
 859  * VF communications.  We sort it out here.
 860  **/
 861 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 862 {
 863         u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
 864         struct i40e_pf *pf = vsi->back;
 865         struct rtnl_link_stats64 *ons;
 866         struct rtnl_link_stats64 *ns;   /* netdev stats */
 867         struct i40e_eth_stats *oes;
 868         struct i40e_eth_stats *es;     /* device's eth stats */
 869         u64 tx_restart, tx_busy;
 870         struct i40e_ring *p;
 871         u64 bytes, packets;
 872         unsigned int start;
 873         u64 tx_linearize;
 874         u64 tx_force_wb;
 875         u64 tx_stopped;
 876         u64 rx_p, rx_b;
 877         u64 tx_p, tx_b;
 878         u16 q;
 879
 880         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 881             test_bit(__I40E_CONFIG_BUSY, pf->state))
 882                 return;
 883
 884         ns = i40e_get_vsi_stats_struct(vsi);
 885         ons = &vsi->net_stats_offsets;
 886         es = &vsi->eth_stats;
 887         oes = &vsi->eth_stats_offsets;
 888
 889         /* Gather up the netdev and vsi stats that the driver collects
 890          * on the fly during packet processing
 891          */
 892         rx_b = rx_p = 0;
 893         tx_b = tx_p = 0;
 894         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 895         tx_stopped = 0;
 896         rx_page = 0;
 897         rx_buf = 0;
 898         rx_reuse = 0;
 899         rx_alloc = 0;
 900         rx_waive = 0;
 901         rx_busy = 0;
 902         rcu_read_lock();
 903         for (q = 0; q < vsi->num_queue_pairs; q++) {
 904                 /* locate Tx ring */
 905                 p = READ_ONCE(vsi->tx_rings[q]);
 906                 if (!p)
 907                         continue;
 908
 909                 do {
 910                         start = u64_stats_fetch_begin(&p->syncp);
 911                         packets = p->stats.packets;
 912                         bytes = p->stats.bytes;
 913                 } while (u64_stats_fetch_retry(&p->syncp, start));
 914                 tx_b += bytes;
 915                 tx_p += packets;
 916                 tx_restart += p->tx_stats.restart_queue;
 917                 tx_busy += p->tx_stats.tx_busy;
 918                 tx_linearize += p->tx_stats.tx_linearize;
 919                 tx_force_wb += p->tx_stats.tx_force_wb;
 920                 tx_stopped += p->tx_stats.tx_stopped;
 921
 922                 /* locate Rx ring */
 923                 p = READ_ONCE(vsi->rx_rings[q]);
 924                 if (!p)
 925                         continue;
 926
 927                 do {
 928                         start = u64_stats_fetch_begin(&p->syncp);
 929                         packets = p->stats.packets;
 930                         bytes = p->stats.bytes;
 931                 } while (u64_stats_fetch_retry(&p->syncp, start));
 932                 rx_b += bytes;
 933                 rx_p += packets;
 934                 rx_buf += p->rx_stats.alloc_buff_failed;
 935                 rx_page += p->rx_stats.alloc_page_failed;
 936                 rx_reuse += p->rx_stats.page_reuse_count;
 937                 rx_alloc += p->rx_stats.page_alloc_count;
 938                 rx_waive += p->rx_stats.page_waive_count;
 939                 rx_busy += p->rx_stats.page_busy_count;
 940
 941                 if (i40e_enabled_xdp_vsi(vsi)) {
 942                         /* locate XDP ring */
 943                         p = READ_ONCE(vsi->xdp_rings[q]);
 944                         if (!p)
 945                                 continue;
 946
 947                         do {
 948                                 start = u64_stats_fetch_begin(&p->syncp);
 949                                 packets = p->stats.packets;
 950                                 bytes = p->stats.bytes;
 951                         } while (u64_stats_fetch_retry(&p->syncp, start));
 952                         tx_b += bytes;
 953                         tx_p += packets;
 954                         tx_restart += p->tx_stats.restart_queue;
 955                         tx_busy += p->tx_stats.tx_busy;
 956                         tx_linearize += p->tx_stats.tx_linearize;
 957                         tx_force_wb += p->tx_stats.tx_force_wb;
 958                 }
 959         }
 960         rcu_read_unlock();
 961         vsi->tx_restart = tx_restart;
 962         vsi->tx_busy = tx_busy;
 963         vsi->tx_linearize = tx_linearize;
 964         vsi->tx_force_wb = tx_force_wb;
 965         vsi->tx_stopped = tx_stopped;
 966         vsi->rx_page_failed = rx_page;
 967         vsi->rx_buf_failed = rx_buf;
 968         vsi->rx_page_reuse = rx_reuse;
 969         vsi->rx_page_alloc = rx_alloc;
 970         vsi->rx_page_waive = rx_waive;
 971         vsi->rx_page_busy = rx_busy;
 972
 973         ns->rx_packets = rx_p;
 974         ns->rx_bytes = rx_b;
 975         ns->tx_packets = tx_p;
 976         ns->tx_bytes = tx_b;
 977
 978         /* update netdev stats from eth stats */
 979         i40e_update_eth_stats(vsi);
 980         ons->tx_errors = oes->tx_errors;
 981         ns->tx_errors = es->tx_errors;
 982         ons->multicast = oes->rx_multicast;
 983         ns->multicast = es->rx_multicast;
 984         ons->rx_dropped = oes->rx_discards_other;
 985         ns->rx_dropped = es->rx_discards_other;
 986         ons->rx_missed_errors = oes->rx_discards;
 987         ns->rx_missed_errors = es->rx_discards;
 988         ons->tx_dropped = oes->tx_discards;
 989         ns->tx_dropped = es->tx_discards;
 990
 991         /* pull in a couple PF stats if this is the main vsi */
 992         if (vsi == pf->vsi[pf->lan_vsi]) {
 993                 ns->rx_crc_errors = pf->stats.crc_errors;
 994                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 995                 ns->rx_length_errors = pf->stats.rx_length_errors;
 996         }
 997 }
 998
 999 /**
1000  * i40e_update_pf_stats - Update the PF statistics counters.
1001  * @pf: the PF to be updated
1002  **/
1003 static void i40e_update_pf_stats(struct i40e_pf *pf)
1004 {
1005         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
1006         struct i40e_hw_port_stats *nsd = &pf->stats;
1007         struct i40e_hw *hw = &pf->hw;
1008         u32 val;
1009         int i;
1010
1011         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1012                            I40E_GLPRT_GORCL(hw->port),
1013                            pf->stat_offsets_loaded,
1014                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
1015         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1016                            I40E_GLPRT_GOTCL(hw->port),
1017                            pf->stat_offsets_loaded,
1018                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
1019         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->eth.rx_discards,
1022                            &nsd->eth.rx_discards);
1023         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1024                            I40E_GLPRT_UPRCL(hw->port),
1025                            pf->stat_offsets_loaded,
1026                            &osd->eth.rx_unicast,
1027                            &nsd->eth.rx_unicast);
1028         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1029                            I40E_GLPRT_MPRCL(hw->port),
1030                            pf->stat_offsets_loaded,
1031                            &osd->eth.rx_multicast,
1032                            &nsd->eth.rx_multicast);
1033         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1034                            I40E_GLPRT_BPRCL(hw->port),
1035                            pf->stat_offsets_loaded,
1036                            &osd->eth.rx_broadcast,
1037                            &nsd->eth.rx_broadcast);
1038         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1039                            I40E_GLPRT_UPTCL(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->eth.tx_unicast,
1042                            &nsd->eth.tx_unicast);
1043         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1044                            I40E_GLPRT_MPTCL(hw->port),
1045                            pf->stat_offsets_loaded,
1046                            &osd->eth.tx_multicast,
1047                            &nsd->eth.tx_multicast);
1048         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1049                            I40E_GLPRT_BPTCL(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->eth.tx_broadcast,
1052                            &nsd->eth.tx_broadcast);
1053
1054         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1055                            pf->stat_offsets_loaded,
1056                            &osd->tx_dropped_link_down,
1057                            &nsd->tx_dropped_link_down);
1058
1059         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1060                            pf->stat_offsets_loaded,
1061                            &osd->crc_errors, &nsd->crc_errors);
1062
1063         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1064                            pf->stat_offsets_loaded,
1065                            &osd->illegal_bytes, &nsd->illegal_bytes);
1066
1067         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1068                            pf->stat_offsets_loaded,
1069                            &osd->mac_local_faults,
1070                            &nsd->mac_local_faults);
1071         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1072                            pf->stat_offsets_loaded,
1073                            &osd->mac_remote_faults,
1074                            &nsd->mac_remote_faults);
1075
1076         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1077                            pf->stat_offsets_loaded,
1078                            &osd->rx_length_errors,
1079                            &nsd->rx_length_errors);
1080
1081         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1082                            pf->stat_offsets_loaded,
1083                            &osd->link_xon_rx, &nsd->link_xon_rx);
1084         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1085                            pf->stat_offsets_loaded,
1086                            &osd->link_xon_tx, &nsd->link_xon_tx);
1087         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1088                            pf->stat_offsets_loaded,
1089                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
1090         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1091                            pf->stat_offsets_loaded,
1092                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
1093
1094         for (i = 0; i < 8; i++) {
1095                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1096                                    pf->stat_offsets_loaded,
1097                                    &osd->priority_xoff_rx[i],
1098                                    &nsd->priority_xoff_rx[i]);
1099                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1100                                    pf->stat_offsets_loaded,
1101                                    &osd->priority_xon_rx[i],
1102                                    &nsd->priority_xon_rx[i]);
1103                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1104                                    pf->stat_offsets_loaded,
1105                                    &osd->priority_xon_tx[i],
1106                                    &nsd->priority_xon_tx[i]);
1107                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1108                                    pf->stat_offsets_loaded,
1109                                    &osd->priority_xoff_tx[i],
1110                                    &nsd->priority_xoff_tx[i]);
1111                 i40e_stat_update32(hw,
1112                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1113                                    pf->stat_offsets_loaded,
1114                                    &osd->priority_xon_2_xoff[i],
1115                                    &nsd->priority_xon_2_xoff[i]);
1116         }
1117
1118         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1119                            I40E_GLPRT_PRC64L(hw->port),
1120                            pf->stat_offsets_loaded,
1121                            &osd->rx_size_64, &nsd->rx_size_64);
1122         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1123                            I40E_GLPRT_PRC127L(hw->port),
1124                            pf->stat_offsets_loaded,
1125                            &osd->rx_size_127, &nsd->rx_size_127);
1126         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1127                            I40E_GLPRT_PRC255L(hw->port),
1128                            pf->stat_offsets_loaded,
1129                            &osd->rx_size_255, &nsd->rx_size_255);
1130         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1131                            I40E_GLPRT_PRC511L(hw->port),
1132                            pf->stat_offsets_loaded,
1133                            &osd->rx_size_511, &nsd->rx_size_511);
1134         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1135                            I40E_GLPRT_PRC1023L(hw->port),
1136                            pf->stat_offsets_loaded,
1137                            &osd->rx_size_1023, &nsd->rx_size_1023);
1138         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1139                            I40E_GLPRT_PRC1522L(hw->port),
1140                            pf->stat_offsets_loaded,
1141                            &osd->rx_size_1522, &nsd->rx_size_1522);
1142         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1143                            I40E_GLPRT_PRC9522L(hw->port),
1144                            pf->stat_offsets_loaded,
1145                            &osd->rx_size_big, &nsd->rx_size_big);
1146
1147         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1148                            I40E_GLPRT_PTC64L(hw->port),
1149                            pf->stat_offsets_loaded,
1150                            &osd->tx_size_64, &nsd->tx_size_64);
1151         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1152                            I40E_GLPRT_PTC127L(hw->port),
1153                            pf->stat_offsets_loaded,
1154                            &osd->tx_size_127, &nsd->tx_size_127);
1155         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1156                            I40E_GLPRT_PTC255L(hw->port),
1157                            pf->stat_offsets_loaded,
1158                            &osd->tx_size_255, &nsd->tx_size_255);
1159         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1160                            I40E_GLPRT_PTC511L(hw->port),
1161                            pf->stat_offsets_loaded,
1162                            &osd->tx_size_511, &nsd->tx_size_511);
1163         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1164                            I40E_GLPRT_PTC1023L(hw->port),
1165                            pf->stat_offsets_loaded,
1166                            &osd->tx_size_1023, &nsd->tx_size_1023);
1167         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1168                            I40E_GLPRT_PTC1522L(hw->port),
1169                            pf->stat_offsets_loaded,
1170                            &osd->tx_size_1522, &nsd->tx_size_1522);
1171         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1172                            I40E_GLPRT_PTC9522L(hw->port),
1173                            pf->stat_offsets_loaded,
1174                            &osd->tx_size_big, &nsd->tx_size_big);
1175
1176         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1177                            pf->stat_offsets_loaded,
1178                            &osd->rx_undersize, &nsd->rx_undersize);
1179         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1180                            pf->stat_offsets_loaded,
1181                            &osd->rx_fragments, &nsd->rx_fragments);
1182         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1183                            pf->stat_offsets_loaded,
1184                            &osd->rx_oversize, &nsd->rx_oversize);
1185         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1186                            pf->stat_offsets_loaded,
1187                            &osd->rx_jabber, &nsd->rx_jabber);
1188
1189         /* FDIR stats */
1190         i40e_stat_update_and_clear32(hw,
1191                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1192                         &nsd->fd_atr_match);
1193         i40e_stat_update_and_clear32(hw,
1194                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1195                         &nsd->fd_sb_match);
1196         i40e_stat_update_and_clear32(hw,
1197                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1198                         &nsd->fd_atr_tunnel_match);
1199
1200         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1201         nsd->tx_lpi_status =
1202                        FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);
1203         nsd->rx_lpi_status =
1204                        FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);
1205         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1206                            pf->stat_offsets_loaded,
1207                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1208         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1209                            pf->stat_offsets_loaded,
1210                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1211
1212         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
1213             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1214                 nsd->fd_sb_status = true;
1215         else
1216                 nsd->fd_sb_status = false;
1217
1218         if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
1219             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1220                 nsd->fd_atr_status = true;
1221         else
1222                 nsd->fd_atr_status = false;
1223
1224         pf->stat_offsets_loaded = true;
1225 }
1226
1227 /**
1228  * i40e_update_stats - Update the various statistics counters.
1229  * @vsi: the VSI to be updated
1230  *
1231  * Update the various stats for this VSI and its related entities.
1232  **/
1233 void i40e_update_stats(struct i40e_vsi *vsi)
1234 {
1235         struct i40e_pf *pf = vsi->back;
1236
1237         if (vsi == pf->vsi[pf->lan_vsi])
1238                 i40e_update_pf_stats(pf);
1239
1240         i40e_update_vsi_stats(vsi);
1241 }
1242
1243 /**
1244  * i40e_count_filters - counts VSI mac filters
1245  * @vsi: the VSI to be searched
1246  *
1247  * Returns count of mac filters
1248  **/
1249 int i40e_count_filters(struct i40e_vsi *vsi)
1250 {
1251         struct i40e_mac_filter *f;
1252         struct hlist_node *h;
1253         int bkt;
1254         int cnt = 0;
1255
1256         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1257                 if (f->state == I40E_FILTER_NEW ||
1258                     f->state == I40E_FILTER_ACTIVE)
1259                         ++cnt;
1260         }
1261
1262         return cnt;
1263 }
1264
1265 /**
1266  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1267  * @vsi: the VSI to be searched
1268  * @macaddr: the MAC address
1269  * @vlan: the vlan
1270  *
1271  * Returns ptr to the filter object or NULL
1272  **/
1273 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1274                                                 const u8 *macaddr, s16 vlan)
1275 {
1276         struct i40e_mac_filter *f;
1277         u64 key;
1278
1279         if (!vsi || !macaddr)
1280                 return NULL;
1281
1282         key = i40e_addr_to_hkey(macaddr);
1283         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1284                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1285                     (vlan == f->vlan))
1286                         return f;
1287         }
1288         return NULL;
1289 }
1290
1291 /**
1292  * i40e_find_mac - Find a mac addr in the macvlan filters list
1293  * @vsi: the VSI to be searched
1294  * @macaddr: the MAC address we are searching for
1295  *
1296  * Returns the first filter with the provided MAC address or NULL if
1297  * MAC address was not found
1298  **/
1299 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1300 {
1301         struct i40e_mac_filter *f;
1302         u64 key;
1303
1304         if (!vsi || !macaddr)
1305                 return NULL;
1306
1307         key = i40e_addr_to_hkey(macaddr);
1308         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1309                 if ((ether_addr_equal(macaddr, f->macaddr)))
1310                         return f;
1311         }
1312         return NULL;
1313 }
1314
1315 /**
1316  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1317  * @vsi: the VSI to be searched
1318  *
1319  * Returns true if VSI is in vlan mode or false otherwise
1320  **/
1321 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1322 {
1323         /* If we have a PVID, always operate in VLAN mode */
1324         if (vsi->info.pvid)
1325                 return true;
1326
1327         /* We need to operate in VLAN mode whenever we have any filters with
1328          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1329          * time, incurring search cost repeatedly. However, we can notice two
1330          * things:
1331          *
1332          * 1) the only place where we can gain a VLAN filter is in
1333          *    i40e_add_filter.
1334          *
1335          * 2) the only place where filters are actually removed is in
1336          *    i40e_sync_filters_subtask.
1337          *
1338          * Thus, we can simply use a boolean value, has_vlan_filters which we
1339          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1340          * we have to perform the full search after deleting filters in
1341          * i40e_sync_filters_subtask, but we already have to search
1342          * filters here and can perform the check at the same time. This
1343          * results in avoiding embedding a loop for VLAN mode inside another
1344          * loop over all the filters, and should maintain correctness as noted
1345          * above.
1346          */
1347         return vsi->has_vlan_filter;
1348 }
1349
1350 /**
1351  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1352  * @vsi: the VSI to configure
1353  * @tmp_add_list: list of filters ready to be added
1354  * @tmp_del_list: list of filters ready to be deleted
1355  * @vlan_filters: the number of active VLAN filters
1356  *
1357  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1358  * behave as expected. If we have any active VLAN filters remaining or about
1359  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1360  * so that they only match against untagged traffic. If we no longer have any
1361  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1362  * so that they match against both tagged and untagged traffic. In this way,
1363  * we ensure that we correctly receive the desired traffic. This ensures that
1364  * when we have an active VLAN we will receive only untagged traffic and
1365  * traffic matching active VLANs. If we have no active VLANs then we will
1366  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1367  *
1368  * Finally, in a similar fashion, this function also corrects filters when
1369  * there is an active PVID assigned to this VSI.
1370  *
1371  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1372  *
1373  * This function is only expected to be called from within
1374  * i40e_sync_vsi_filters.
1375  *
1376  * NOTE: This function expects to be called while under the
1377  * mac_filter_hash_lock
1378  */
1379 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1380                                          struct hlist_head *tmp_add_list,
1381                                          struct hlist_head *tmp_del_list,
1382                                          int vlan_filters)
1383 {
1384         s16 pvid = le16_to_cpu(vsi->info.pvid);
1385         struct i40e_mac_filter *f, *add_head;
1386         struct i40e_new_mac_filter *new;
1387         struct hlist_node *h;
1388         int bkt, new_vlan;
1389
1390         /* To determine if a particular filter needs to be replaced we
1391          * have the three following conditions:
1392          *
1393          * a) if we have a PVID assigned, then all filters which are
1394          *    not marked as VLAN=PVID must be replaced with filters that
1395          *    are.
1396          * b) otherwise, if we have any active VLANS, all filters
1397          *    which are marked as VLAN=-1 must be replaced with
1398          *    filters marked as VLAN=0
1399          * c) finally, if we do not have any active VLANS, all filters
1400          *    which are marked as VLAN=0 must be replaced with filters
1401          *    marked as VLAN=-1
1402          */
1403
1404         /* Update the filters about to be added in place */
1405         hlist_for_each_entry(new, tmp_add_list, hlist) {
1406                 if (pvid && new->f->vlan != pvid)
1407                         new->f->vlan = pvid;
1408                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1409                         new->f->vlan = 0;
1410                 else if (!vlan_filters && new->f->vlan == 0)
1411                         new->f->vlan = I40E_VLAN_ANY;
1412         }
1413
1414         /* Update the remaining active filters */
1415         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1416                 /* Combine the checks for whether a filter needs to be changed
1417                  * and then determine the new VLAN inside the if block, in
1418                  * order to avoid duplicating code for adding the new filter
1419                  * then deleting the old filter.
1420                  */
1421                 if ((pvid && f->vlan != pvid) ||
1422                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1423                     (!vlan_filters && f->vlan == 0)) {
1424                         /* Determine the new vlan we will be adding */
1425                         if (pvid)
1426                                 new_vlan = pvid;
1427                         else if (vlan_filters)
1428                                 new_vlan = 0;
1429                         else
1430                                 new_vlan = I40E_VLAN_ANY;
1431
1432                         /* Create the new filter */
1433                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1434                         if (!add_head)
1435                                 return -ENOMEM;
1436
1437                         /* Create a temporary i40e_new_mac_filter */
1438                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1439                         if (!new)
1440                                 return -ENOMEM;
1441
1442                         new->f = add_head;
1443                         new->state = add_head->state;
1444
1445                         /* Add the new filter to the tmp list */
1446                         hlist_add_head(&new->hlist, tmp_add_list);
1447
1448                         /* Put the original filter into the delete list */
1449                         f->state = I40E_FILTER_REMOVE;
1450                         hash_del(&f->hlist);
1451                         hlist_add_head(&f->hlist, tmp_del_list);
1452                 }
1453         }
1454
1455         vsi->has_vlan_filter = !!vlan_filters;
1456
1457         return 0;
1458 }
1459
1460 /**
1461  * i40e_get_vf_new_vlan - Get new vlan id on a vf
1462  * @vsi: the vsi to configure
1463  * @new_mac: new mac filter to be added
1464  * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1465  * @vlan_filters: the number of active VLAN filters
1466  * @trusted: flag if the VF is trusted
1467  *
1468  * Get new VLAN id based on current VLAN filters, trust, PVID
1469  * and vf-vlan-prune-disable flag.
1470  *
1471  * Returns the value of the new vlan filter or
1472  * the old value if no new filter is needed.
1473  */
1474 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1475                                 struct i40e_new_mac_filter *new_mac,
1476                                 struct i40e_mac_filter *f,
1477                                 int vlan_filters,
1478                                 bool trusted)
1479 {
1480         s16 pvid = le16_to_cpu(vsi->info.pvid);
1481         struct i40e_pf *pf = vsi->back;
1482         bool is_any;
1483
1484         if (new_mac)
1485                 f = new_mac->f;
1486
1487         if (pvid && f->vlan != pvid)
1488                 return pvid;
1489
1490         is_any = (trusted ||
1491                   !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
1492
1493         if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1494             (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1495             (is_any && !vlan_filters && f->vlan == 0)) {
1496                 if (is_any)
1497                         return I40E_VLAN_ANY;
1498                 else
1499                         return 0;
1500         }
1501
1502         return f->vlan;
1503 }
1504
1505 /**
1506  * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1507  * @vsi: the vsi to configure
1508  * @tmp_add_list: list of filters ready to be added
1509  * @tmp_del_list: list of filters ready to be deleted
1510  * @vlan_filters: the number of active VLAN filters
1511  * @trusted: flag if the VF is trusted
1512  *
1513  * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1514  * and vf-vlan-prune-disable flag.
1515  *
1516  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1517  *
1518  * This function is only expected to be called from within
1519  * i40e_sync_vsi_filters.
1520  *
1521  * NOTE: This function expects to be called while under the
1522  * mac_filter_hash_lock
1523  */
1524 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1525                                             struct hlist_head *tmp_add_list,
1526                                             struct hlist_head *tmp_del_list,
1527                                             int vlan_filters,
1528                                             bool trusted)
1529 {
1530         struct i40e_mac_filter *f, *add_head;
1531         struct i40e_new_mac_filter *new_mac;
1532         struct hlist_node *h;
1533         int bkt, new_vlan;
1534
1535         hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1536                 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1537                                                         vlan_filters, trusted);
1538         }
1539
1540         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1541                 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1542                                                 trusted);
1543                 if (new_vlan != f->vlan) {
1544                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1545                         if (!add_head)
1546                                 return -ENOMEM;
1547                         /* Create a temporary i40e_new_mac_filter */
1548                         new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1549                         if (!new_mac)
1550                                 return -ENOMEM;
1551                         new_mac->f = add_head;
1552                         new_mac->state = add_head->state;
1553
1554                         /* Add the new filter to the tmp list */
1555                         hlist_add_head(&new_mac->hlist, tmp_add_list);
1556
1557                         /* Put the original filter into the delete list */
1558                         f->state = I40E_FILTER_REMOVE;
1559                         hash_del(&f->hlist);
1560                         hlist_add_head(&f->hlist, tmp_del_list);
1561                 }
1562         }
1563
1564         vsi->has_vlan_filter = !!vlan_filters;
1565         return 0;
1566 }
1567
1568 /**
1569  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1570  * @vsi: the PF Main VSI - inappropriate for any other VSI
1571  * @macaddr: the MAC address
1572  *
1573  * Remove whatever filter the firmware set up so the driver can manage
1574  * its own filtering intelligently.
1575  **/
1576 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1577 {
1578         struct i40e_aqc_remove_macvlan_element_data element;
1579         struct i40e_pf *pf = vsi->back;
1580
1581         /* Only appropriate for the PF main VSI */
1582         if (vsi->type != I40E_VSI_MAIN)
1583                 return;
1584
1585         memset(&element, 0, sizeof(element));
1586         ether_addr_copy(element.mac_addr, macaddr);
1587         element.vlan_tag = 0;
1588         /* Ignore error returns, some firmware does it this way... */
1589         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1590         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1591
1592         memset(&element, 0, sizeof(element));
1593         ether_addr_copy(element.mac_addr, macaddr);
1594         element.vlan_tag = 0;
1595         /* ...and some firmware does it this way. */
1596         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1597                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1598         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1599 }
1600
1601 /**
1602  * i40e_add_filter - Add a mac/vlan filter to the VSI
1603  * @vsi: the VSI to be searched
1604  * @macaddr: the MAC address
1605  * @vlan: the vlan
1606  *
1607  * Returns ptr to the filter object or NULL when no memory available.
1608  *
1609  * NOTE: This function is expected to be called with mac_filter_hash_lock
1610  * being held.
1611  **/
1612 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1613                                         const u8 *macaddr, s16 vlan)
1614 {
1615         struct i40e_mac_filter *f;
1616         u64 key;
1617
1618         if (!vsi || !macaddr)
1619                 return NULL;
1620
1621         f = i40e_find_filter(vsi, macaddr, vlan);
1622         if (!f) {
1623                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1624                 if (!f)
1625                         return NULL;
1626
1627                 /* Update the boolean indicating if we need to function in
1628                  * VLAN mode.
1629                  */
1630                 if (vlan >= 0)
1631                         vsi->has_vlan_filter = true;
1632
1633                 ether_addr_copy(f->macaddr, macaddr);
1634                 f->vlan = vlan;
1635                 f->state = I40E_FILTER_NEW;
1636                 INIT_HLIST_NODE(&f->hlist);
1637
1638                 key = i40e_addr_to_hkey(macaddr);
1639                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1640
1641                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1642                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1643         }
1644
1645         /* If we're asked to add a filter that has been marked for removal, it
1646          * is safe to simply restore it to active state. __i40e_del_filter
1647          * will have simply deleted any filters which were previously marked
1648          * NEW or FAILED, so if it is currently marked REMOVE it must have
1649          * previously been ACTIVE. Since we haven't yet run the sync filters
1650          * task, just restore this filter to the ACTIVE state so that the
1651          * sync task leaves it in place
1652          */
1653         if (f->state == I40E_FILTER_REMOVE)
1654                 f->state = I40E_FILTER_ACTIVE;
1655
1656         return f;
1657 }
1658
1659 /**
1660  * __i40e_del_filter - Remove a specific filter from the VSI
1661  * @vsi: VSI to remove from
1662  * @f: the filter to remove from the list
1663  *
1664  * This function should be called instead of i40e_del_filter only if you know
1665  * the exact filter you will remove already, such as via i40e_find_filter or
1666  * i40e_find_mac.
1667  *
1668  * NOTE: This function is expected to be called with mac_filter_hash_lock
1669  * being held.
1670  * ANOTHER NOTE: This function MUST be called from within the context of
1671  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1672  * instead of list_for_each_entry().
1673  **/
1674 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1675 {
1676         if (!f)
1677                 return;
1678
1679         /* If the filter was never added to firmware then we can just delete it
1680          * directly and we don't want to set the status to remove or else an
1681          * admin queue command will unnecessarily fire.
1682          */
1683         if ((f->state == I40E_FILTER_FAILED) ||
1684             (f->state == I40E_FILTER_NEW)) {
1685                 hash_del(&f->hlist);
1686                 kfree(f);
1687         } else {
1688                 f->state = I40E_FILTER_REMOVE;
1689         }
1690
1691         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1692         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1693 }
1694
1695 /**
1696  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1697  * @vsi: the VSI to be searched
1698  * @macaddr: the MAC address
1699  * @vlan: the VLAN
1700  *
1701  * NOTE: This function is expected to be called with mac_filter_hash_lock
1702  * being held.
1703  * ANOTHER NOTE: This function MUST be called from within the context of
1704  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1705  * instead of list_for_each_entry().
1706  **/
1707 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1708 {
1709         struct i40e_mac_filter *f;
1710
1711         if (!vsi || !macaddr)
1712                 return;
1713
1714         f = i40e_find_filter(vsi, macaddr, vlan);
1715         __i40e_del_filter(vsi, f);
1716 }
1717
1718 /**
1719  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1720  * @vsi: the VSI to be searched
1721  * @macaddr: the mac address to be filtered
1722  *
1723  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1724  * go through all the macvlan filters and add a macvlan filter for each
1725  * unique vlan that already exists. If a PVID has been assigned, instead only
1726  * add the macaddr to that VLAN.
1727  *
1728  * Returns last filter added on success, else NULL
1729  **/
1730 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1731                                             const u8 *macaddr)
1732 {
1733         struct i40e_mac_filter *f, *add = NULL;
1734         struct hlist_node *h;
1735         int bkt;
1736
1737         if (vsi->info.pvid)
1738                 return i40e_add_filter(vsi, macaddr,
1739                                        le16_to_cpu(vsi->info.pvid));
1740
1741         if (!i40e_is_vsi_in_vlan(vsi))
1742                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1743
1744         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1745                 if (f->state == I40E_FILTER_REMOVE)
1746                         continue;
1747                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1748                 if (!add)
1749                         return NULL;
1750         }
1751
1752         return add;
1753 }
1754
1755 /**
1756  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1757  * @vsi: the VSI to be searched
1758  * @macaddr: the mac address to be removed
1759  *
1760  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1761  * associated with.
1762  *
1763  * Returns 0 for success, or error
1764  **/
1765 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1766 {
1767         struct i40e_mac_filter *f;
1768         struct hlist_node *h;
1769         bool found = false;
1770         int bkt;
1771
1772         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1773         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1774                 if (ether_addr_equal(macaddr, f->macaddr)) {
1775                         __i40e_del_filter(vsi, f);
1776                         found = true;
1777                 }
1778         }
1779
1780         if (found)
1781                 return 0;
1782         else
1783                 return -ENOENT;
1784 }
1785
1786 /**
1787  * i40e_set_mac - NDO callback to set mac address
1788  * @netdev: network interface device structure
1789  * @p: pointer to an address structure
1790  *
1791  * Returns 0 on success, negative on failure
1792  **/
1793 static int i40e_set_mac(struct net_device *netdev, void *p)
1794 {
1795         struct i40e_netdev_priv *np = netdev_priv(netdev);
1796         struct i40e_vsi *vsi = np->vsi;
1797         struct i40e_pf *pf = vsi->back;
1798         struct i40e_hw *hw = &pf->hw;
1799         struct sockaddr *addr = p;
1800
1801         if (!is_valid_ether_addr(addr->sa_data))
1802                 return -EADDRNOTAVAIL;
1803
1804         if (test_bit(__I40E_DOWN, pf->state) ||
1805             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1806                 return -EADDRNOTAVAIL;
1807
1808         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1809                 netdev_info(netdev, "returning to hw mac address %pM\n",
1810                             hw->mac.addr);
1811         else
1812                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1813
1814         /* Copy the address first, so that we avoid a possible race with
1815          * .set_rx_mode().
1816          * - Remove old address from MAC filter
1817          * - Copy new address
1818          * - Add new address to MAC filter
1819          */
1820         spin_lock_bh(&vsi->mac_filter_hash_lock);
1821         i40e_del_mac_filter(vsi, netdev->dev_addr);
1822         eth_hw_addr_set(netdev, addr->sa_data);
1823         i40e_add_mac_filter(vsi, netdev->dev_addr);
1824         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1825
1826         if (vsi->type == I40E_VSI_MAIN) {
1827                 int ret;
1828
1829                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1830                                                 addr->sa_data, NULL);
1831                 if (ret)
1832                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1833                                     ERR_PTR(ret),
1834                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1835         }
1836
1837         /* schedule our worker thread which will take care of
1838          * applying the new filter changes
1839          */
1840         i40e_service_event_schedule(pf);
1841         return 0;
1842 }
1843
1844 /**
1845  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1846  * @vsi: vsi structure
1847  * @seed: RSS hash seed
1848  * @lut: pointer to lookup table of lut_size
1849  * @lut_size: size of the lookup table
1850  **/
1851 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1852                               u8 *lut, u16 lut_size)
1853 {
1854         struct i40e_pf *pf = vsi->back;
1855         struct i40e_hw *hw = &pf->hw;
1856         int ret = 0;
1857
1858         if (seed) {
1859                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1860                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1861                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1862                 if (ret) {
1863                         dev_info(&pf->pdev->dev,
1864                                  "Cannot set RSS key, err %pe aq_err %s\n",
1865                                  ERR_PTR(ret),
1866                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1867                         return ret;
1868                 }
1869         }
1870         if (lut) {
1871                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1872
1873                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1874                 if (ret) {
1875                         dev_info(&pf->pdev->dev,
1876                                  "Cannot set RSS lut, err %pe aq_err %s\n",
1877                                  ERR_PTR(ret),
1878                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1879                         return ret;
1880                 }
1881         }
1882         return ret;
1883 }
1884
1885 /**
1886  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1887  * @vsi: VSI structure
1888  **/
1889 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1890 {
1891         struct i40e_pf *pf = vsi->back;
1892         u8 seed[I40E_HKEY_ARRAY_SIZE];
1893         u8 *lut;
1894         int ret;
1895
1896         if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
1897                 return 0;
1898         if (!vsi->rss_size)
1899                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1900                                       vsi->num_queue_pairs);
1901         if (!vsi->rss_size)
1902                 return -EINVAL;
1903         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1904         if (!lut)
1905                 return -ENOMEM;
1906
1907         /* Use the user configured hash keys and lookup table if there is one,
1908          * otherwise use default
1909          */
1910         if (vsi->rss_lut_user)
1911                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1912         else
1913                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1914         if (vsi->rss_hkey_user)
1915                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1916         else
1917                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1918         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1919         kfree(lut);
1920         return ret;
1921 }
1922
1923 /**
1924  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1925  * @vsi: the VSI being configured,
1926  * @ctxt: VSI context structure
1927  * @enabled_tc: number of traffic classes to enable
1928  *
1929  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1930  **/
1931 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1932                                            struct i40e_vsi_context *ctxt,
1933                                            u8 enabled_tc)
1934 {
1935         u16 qcount = 0, max_qcount, qmap, sections = 0;
1936         int i, override_q, pow, num_qps, ret;
1937         u8 netdev_tc = 0, offset = 0;
1938
1939         if (vsi->type != I40E_VSI_MAIN)
1940                 return -EINVAL;
1941         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1942         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1943         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1944         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1945         num_qps = vsi->mqprio_qopt.qopt.count[0];
1946
1947         /* find the next higher power-of-2 of num queue pairs */
1948         pow = ilog2(num_qps);
1949         if (!is_power_of_2(num_qps))
1950                 pow++;
1951         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1952                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1953
1954         /* Setup queue offset/count for all TCs for given VSI */
1955         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1956         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1957                 /* See if the given TC is enabled for the given VSI */
1958                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1959                         offset = vsi->mqprio_qopt.qopt.offset[i];
1960                         qcount = vsi->mqprio_qopt.qopt.count[i];
1961                         if (qcount > max_qcount)
1962                                 max_qcount = qcount;
1963                         vsi->tc_config.tc_info[i].qoffset = offset;
1964                         vsi->tc_config.tc_info[i].qcount = qcount;
1965                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1966                 } else {
1967                         /* TC is not enabled so set the offset to
1968                          * default queue and allocate one queue
1969                          * for the given TC.
1970                          */
1971                         vsi->tc_config.tc_info[i].qoffset = 0;
1972                         vsi->tc_config.tc_info[i].qcount = 1;
1973                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1974                 }
1975         }
1976
1977         /* Set actual Tx/Rx queue pairs */
1978         vsi->num_queue_pairs = offset + qcount;
1979
1980         /* Setup queue TC[0].qmap for given VSI context */
1981         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1982         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1983         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1984         ctxt->info.valid_sections |= cpu_to_le16(sections);
1985
1986         /* Reconfigure RSS for main VSI with max queue count */
1987         vsi->rss_size = max_qcount;
1988         ret = i40e_vsi_config_rss(vsi);
1989         if (ret) {
1990                 dev_info(&vsi->back->pdev->dev,
1991                          "Failed to reconfig rss for num_queues (%u)\n",
1992                          max_qcount);
1993                 return ret;
1994         }
1995         vsi->reconfig_rss = true;
1996         dev_dbg(&vsi->back->pdev->dev,
1997                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1998
1999         /* Find queue count available for channel VSIs and starting offset
2000          * for channel VSIs
2001          */
2002         override_q = vsi->mqprio_qopt.qopt.count[0];
2003         if (override_q && override_q < vsi->num_queue_pairs) {
2004                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
2005                 vsi->next_base_queue = override_q;
2006         }
2007         return 0;
2008 }
2009
2010 /**
2011  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
2012  * @vsi: the VSI being setup
2013  * @ctxt: VSI context structure
2014  * @enabled_tc: Enabled TCs bitmap
2015  * @is_add: True if called before Add VSI
2016  *
2017  * Setup VSI queue mapping for enabled traffic classes.
2018  **/
2019 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2020                                      struct i40e_vsi_context *ctxt,
2021                                      u8 enabled_tc,
2022                                      bool is_add)
2023 {
2024         struct i40e_pf *pf = vsi->back;
2025         u16 num_tc_qps = 0;
2026         u16 sections = 0;
2027         u8 netdev_tc = 0;
2028         u16 numtc = 1;
2029         u16 qcount;
2030         u8 offset;
2031         u16 qmap;
2032         int i;
2033
2034         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2035         offset = 0;
2036         /* zero out queue mapping, it will get updated on the end of the function */
2037         memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2038
2039         if (vsi->type == I40E_VSI_MAIN) {
2040                 /* This code helps add more queue to the VSI if we have
2041                  * more cores than RSS can support, the higher cores will
2042                  * be served by ATR or other filters. Furthermore, the
2043                  * non-zero req_queue_pairs says that user requested a new
2044                  * queue count via ethtool's set_channels, so use this
2045                  * value for queues distribution across traffic classes
2046                  * We need at least one queue pair for the interface
2047                  * to be usable as we see in else statement.
2048                  */
2049                 if (vsi->req_queue_pairs > 0)
2050                         vsi->num_queue_pairs = vsi->req_queue_pairs;
2051                 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2052                         vsi->num_queue_pairs = pf->num_lan_msix;
2053                 else
2054                         vsi->num_queue_pairs = 1;
2055         }
2056
2057         /* Number of queues per enabled TC */
2058         if (vsi->type == I40E_VSI_MAIN ||
2059             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2060                 num_tc_qps = vsi->num_queue_pairs;
2061         else
2062                 num_tc_qps = vsi->alloc_queue_pairs;
2063
2064         if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
2065                 /* Find numtc from enabled TC bitmap */
2066                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2067                         if (enabled_tc & BIT(i)) /* TC is enabled */
2068                                 numtc++;
2069                 }
2070                 if (!numtc) {
2071                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2072                         numtc = 1;
2073                 }
2074                 num_tc_qps = num_tc_qps / numtc;
2075                 num_tc_qps = min_t(int, num_tc_qps,
2076                                    i40e_pf_get_max_q_per_tc(pf));
2077         }
2078
2079         vsi->tc_config.numtc = numtc;
2080         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2081
2082         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
2083         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2084                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2085
2086         /* Setup queue offset/count for all TCs for given VSI */
2087         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2088                 /* See if the given TC is enabled for the given VSI */
2089                 if (vsi->tc_config.enabled_tc & BIT(i)) {
2090                         /* TC is enabled */
2091                         int pow, num_qps;
2092
2093                         switch (vsi->type) {
2094                         case I40E_VSI_MAIN:
2095                                 if ((!test_bit(I40E_FLAG_FD_SB_ENA,
2096                                                pf->flags) &&
2097                                      !test_bit(I40E_FLAG_FD_ATR_ENA,
2098                                                pf->flags)) ||
2099                                     vsi->tc_config.enabled_tc != 1) {
2100                                         qcount = min_t(int, pf->alloc_rss_size,
2101                                                        num_tc_qps);
2102                                         break;
2103                                 }
2104                                 fallthrough;
2105                         case I40E_VSI_FDIR:
2106                         case I40E_VSI_SRIOV:
2107                         case I40E_VSI_VMDQ2:
2108                         default:
2109                                 qcount = num_tc_qps;
2110                                 WARN_ON(i != 0);
2111                                 break;
2112                         }
2113                         vsi->tc_config.tc_info[i].qoffset = offset;
2114                         vsi->tc_config.tc_info[i].qcount = qcount;
2115
2116                         /* find the next higher power-of-2 of num queue pairs */
2117                         num_qps = qcount;
2118                         pow = 0;
2119                         while (num_qps && (BIT_ULL(pow) < qcount)) {
2120                                 pow++;
2121                                 num_qps >>= 1;
2122                         }
2123
2124                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2125                         qmap =
2126                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2127                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2128
2129                         offset += qcount;
2130                 } else {
2131                         /* TC is not enabled so set the offset to
2132                          * default queue and allocate one queue
2133                          * for the given TC.
2134                          */
2135                         vsi->tc_config.tc_info[i].qoffset = 0;
2136                         vsi->tc_config.tc_info[i].qcount = 1;
2137                         vsi->tc_config.tc_info[i].netdev_tc = 0;
2138
2139                         qmap = 0;
2140                 }
2141                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2142         }
2143         /* Do not change previously set num_queue_pairs for PFs and VFs*/
2144         if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2145             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2146             (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2147                 vsi->num_queue_pairs = offset;
2148
2149         /* Scheduler section valid can only be set for ADD VSI */
2150         if (is_add) {
2151                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2152
2153                 ctxt->info.up_enable_bits = enabled_tc;
2154         }
2155         if (vsi->type == I40E_VSI_SRIOV) {
2156                 ctxt->info.mapping_flags |=
2157                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2158                 for (i = 0; i < vsi->num_queue_pairs; i++)
2159                         ctxt->info.queue_mapping[i] =
2160                                                cpu_to_le16(vsi->base_queue + i);
2161         } else {
2162                 ctxt->info.mapping_flags |=
2163                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2164                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2165         }
2166         ctxt->info.valid_sections |= cpu_to_le16(sections);
2167 }
2168
2169 /**
2170  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2171  * @netdev: the netdevice
2172  * @addr: address to add
2173  *
2174  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2175  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2176  */
2177 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2178 {
2179         struct i40e_netdev_priv *np = netdev_priv(netdev);
2180         struct i40e_vsi *vsi = np->vsi;
2181
2182         if (i40e_add_mac_filter(vsi, addr))
2183                 return 0;
2184         else
2185                 return -ENOMEM;
2186 }
2187
2188 /**
2189  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2190  * @netdev: the netdevice
2191  * @addr: address to add
2192  *
2193  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2194  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2195  */
2196 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2197 {
2198         struct i40e_netdev_priv *np = netdev_priv(netdev);
2199         struct i40e_vsi *vsi = np->vsi;
2200
2201         /* Under some circumstances, we might receive a request to delete
2202          * our own device address from our uc list. Because we store the
2203          * device address in the VSI's MAC/VLAN filter list, we need to ignore
2204          * such requests and not delete our device address from this list.
2205          */
2206         if (ether_addr_equal(addr, netdev->dev_addr))
2207                 return 0;
2208
2209         i40e_del_mac_filter(vsi, addr);
2210
2211         return 0;
2212 }
2213
2214 /**
2215  * i40e_set_rx_mode - NDO callback to set the netdev filters
2216  * @netdev: network interface device structure
2217  **/
2218 static void i40e_set_rx_mode(struct net_device *netdev)
2219 {
2220         struct i40e_netdev_priv *np = netdev_priv(netdev);
2221         struct i40e_vsi *vsi = np->vsi;
2222
2223         spin_lock_bh(&vsi->mac_filter_hash_lock);
2224
2225         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2226         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2227
2228         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2229
2230         /* check for other flag changes */
2231         if (vsi->current_netdev_flags != vsi->netdev->flags) {
2232                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2233                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2234         }
2235 }
2236
2237 /**
2238  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2239  * @vsi: Pointer to VSI struct
2240  * @from: Pointer to list which contains MAC filter entries - changes to
2241  *        those entries needs to be undone.
2242  *
2243  * MAC filter entries from this list were slated for deletion.
2244  **/
2245 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2246                                          struct hlist_head *from)
2247 {
2248         struct i40e_mac_filter *f;
2249         struct hlist_node *h;
2250
2251         hlist_for_each_entry_safe(f, h, from, hlist) {
2252                 u64 key = i40e_addr_to_hkey(f->macaddr);
2253
2254                 /* Move the element back into MAC filter list*/
2255                 hlist_del(&f->hlist);
2256                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2257         }
2258 }
2259
2260 /**
2261  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2262  * @vsi: Pointer to vsi struct
2263  * @from: Pointer to list which contains MAC filter entries - changes to
2264  *        those entries needs to be undone.
2265  *
2266  * MAC filter entries from this list were slated for addition.
2267  **/
2268 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2269                                          struct hlist_head *from)
2270 {
2271         struct i40e_new_mac_filter *new;
2272         struct hlist_node *h;
2273
2274         hlist_for_each_entry_safe(new, h, from, hlist) {
2275                 /* We can simply free the wrapper structure */
2276                 hlist_del(&new->hlist);
2277                 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2278                 kfree(new);
2279         }
2280 }
2281
2282 /**
2283  * i40e_next_filter - Get the next non-broadcast filter from a list
2284  * @next: pointer to filter in list
2285  *
2286  * Returns the next non-broadcast filter in the list. Required so that we
2287  * ignore broadcast filters within the list, since these are not handled via
2288  * the normal firmware update path.
2289  */
2290 static
2291 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2292 {
2293         hlist_for_each_entry_continue(next, hlist) {
2294                 if (!is_broadcast_ether_addr(next->f->macaddr))
2295                         return next;
2296         }
2297
2298         return NULL;
2299 }
2300
2301 /**
2302  * i40e_update_filter_state - Update filter state based on return data
2303  * from firmware
2304  * @count: Number of filters added
2305  * @add_list: return data from fw
2306  * @add_head: pointer to first filter in current batch
2307  *
2308  * MAC filter entries from list were slated to be added to device. Returns
2309  * number of successful filters. Note that 0 does NOT mean success!
2310  **/
2311 static int
2312 i40e_update_filter_state(int count,
2313                          struct i40e_aqc_add_macvlan_element_data *add_list,
2314                          struct i40e_new_mac_filter *add_head)
2315 {
2316         int retval = 0;
2317         int i;
2318
2319         for (i = 0; i < count; i++) {
2320                 /* Always check status of each filter. We don't need to check
2321                  * the firmware return status because we pre-set the filter
2322                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2323                  * request to the adminq. Thus, if it no longer matches then
2324                  * we know the filter is active.
2325                  */
2326                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2327                         add_head->state = I40E_FILTER_FAILED;
2328                 } else {
2329                         add_head->state = I40E_FILTER_ACTIVE;
2330                         retval++;
2331                 }
2332
2333                 add_head = i40e_next_filter(add_head);
2334                 if (!add_head)
2335                         break;
2336         }
2337
2338         return retval;
2339 }
2340
2341 /**
2342  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2343  * @vsi: ptr to the VSI
2344  * @vsi_name: name to display in messages
2345  * @list: the list of filters to send to firmware
2346  * @num_del: the number of filters to delete
2347  * @retval: Set to -EIO on failure to delete
2348  *
2349  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2350  * *retval instead of a return value so that success does not force ret_val to
2351  * be set to 0. This ensures that a sequence of calls to this function
2352  * preserve the previous value of *retval on successful delete.
2353  */
2354 static
2355 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2356                           struct i40e_aqc_remove_macvlan_element_data *list,
2357                           int num_del, int *retval)
2358 {
2359         struct i40e_hw *hw = &vsi->back->hw;
2360         enum i40e_admin_queue_err aq_status;
2361         int aq_ret;
2362
2363         aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,
2364                                            &aq_status);
2365
2366         /* Explicitly ignore and do not report when firmware returns ENOENT */
2367         if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2368                 *retval = -EIO;
2369                 dev_info(&vsi->back->pdev->dev,
2370                          "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2371                          vsi_name, ERR_PTR(aq_ret),
2372                          i40e_aq_str(hw, aq_status));
2373         }
2374 }
2375
2376 /**
2377  * i40e_aqc_add_filters - Request firmware to add a set of filters
2378  * @vsi: ptr to the VSI
2379  * @vsi_name: name to display in messages
2380  * @list: the list of filters to send to firmware
2381  * @add_head: Position in the add hlist
2382  * @num_add: the number of filters to add
2383  *
2384  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2385  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2386  * space for more filters.
2387  */
2388 static
2389 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2390                           struct i40e_aqc_add_macvlan_element_data *list,
2391                           struct i40e_new_mac_filter *add_head,
2392                           int num_add)
2393 {
2394         struct i40e_hw *hw = &vsi->back->hw;
2395         enum i40e_admin_queue_err aq_status;
2396         int fcnt;
2397
2398         i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);
2399         fcnt = i40e_update_filter_state(num_add, list, add_head);
2400
2401         if (fcnt != num_add) {
2402                 if (vsi->type == I40E_VSI_MAIN) {
2403                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2404                         dev_warn(&vsi->back->pdev->dev,
2405                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2406                                  i40e_aq_str(hw, aq_status), vsi_name);
2407                 } else if (vsi->type == I40E_VSI_SRIOV ||
2408                            vsi->type == I40E_VSI_VMDQ1 ||
2409                            vsi->type == I40E_VSI_VMDQ2) {
2410                         dev_warn(&vsi->back->pdev->dev,
2411                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2412                                  i40e_aq_str(hw, aq_status), vsi_name,
2413                                              vsi_name);
2414                 } else {
2415                         dev_warn(&vsi->back->pdev->dev,
2416                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2417                                  i40e_aq_str(hw, aq_status), vsi_name,
2418                                              vsi->type);
2419                 }
2420         }
2421 }
2422
2423 /**
2424  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2425  * @vsi: pointer to the VSI
2426  * @vsi_name: the VSI name
2427  * @f: filter data
2428  *
2429  * This function sets or clears the promiscuous broadcast flags for VLAN
2430  * filters in order to properly receive broadcast frames. Assumes that only
2431  * broadcast filters are passed.
2432  *
2433  * Returns status indicating success or failure;
2434  **/
2435 static int
2436 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2437                           struct i40e_mac_filter *f)
2438 {
2439         bool enable = f->state == I40E_FILTER_NEW;
2440         struct i40e_hw *hw = &vsi->back->hw;
2441         int aq_ret;
2442
2443         if (f->vlan == I40E_VLAN_ANY) {
2444                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2445                                                    vsi->seid,
2446                                                    enable,
2447                                                    NULL);
2448         } else {
2449                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2450                                                             vsi->seid,
2451                                                             enable,
2452                                                             f->vlan,
2453                                                             NULL);
2454         }
2455
2456         if (aq_ret) {
2457                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2458                 dev_warn(&vsi->back->pdev->dev,
2459                          "Error %s, forcing overflow promiscuous on %s\n",
2460                          i40e_aq_str(hw, hw->aq.asq_last_status),
2461                          vsi_name);
2462         }
2463
2464         return aq_ret;
2465 }
2466
2467 /**
2468  * i40e_set_promiscuous - set promiscuous mode
2469  * @pf: board private structure
2470  * @promisc: promisc on or off
2471  *
2472  * There are different ways of setting promiscuous mode on a PF depending on
2473  * what state/environment we're in.  This identifies and sets it appropriately.
2474  * Returns 0 on success.
2475  **/
2476 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2477 {
2478         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2479         struct i40e_hw *hw = &pf->hw;
2480         int aq_ret;
2481
2482         if (vsi->type == I40E_VSI_MAIN &&
2483             pf->lan_veb != I40E_NO_VEB &&
2484             !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
2485                 /* set defport ON for Main VSI instead of true promisc
2486                  * this way we will get all unicast/multicast and VLAN
2487                  * promisc behavior but will not get VF or VMDq traffic
2488                  * replicated on the Main VSI.
2489                  */
2490                 if (promisc)
2491                         aq_ret = i40e_aq_set_default_vsi(hw,
2492                                                          vsi->seid,
2493                                                          NULL);
2494                 else
2495                         aq_ret = i40e_aq_clear_default_vsi(hw,
2496                                                            vsi->seid,
2497                                                            NULL);
2498                 if (aq_ret) {
2499                         dev_info(&pf->pdev->dev,
2500                                  "Set default VSI failed, err %pe, aq_err %s\n",
2501                                  ERR_PTR(aq_ret),
2502                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2503                 }
2504         } else {
2505                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2506                                                   hw,
2507                                                   vsi->seid,
2508                                                   promisc, NULL,
2509                                                   true);
2510                 if (aq_ret) {
2511                         dev_info(&pf->pdev->dev,
2512                                  "set unicast promisc failed, err %pe, aq_err %s\n",
2513                                  ERR_PTR(aq_ret),
2514                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2515                 }
2516                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2517                                                   hw,
2518                                                   vsi->seid,
2519                                                   promisc, NULL);
2520                 if (aq_ret) {
2521                         dev_info(&pf->pdev->dev,
2522                                  "set multicast promisc failed, err %pe, aq_err %s\n",
2523                                  ERR_PTR(aq_ret),
2524                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2525                 }
2526         }
2527
2528         if (!aq_ret)
2529                 pf->cur_promisc = promisc;
2530
2531         return aq_ret;
2532 }
2533
2534 /**
2535  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2536  * @vsi: ptr to the VSI
2537  *
2538  * Push any outstanding VSI filter changes through the AdminQ.
2539  *
2540  * Returns 0 or error value
2541  **/
2542 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2543 {
2544         struct hlist_head tmp_add_list, tmp_del_list;
2545         struct i40e_mac_filter *f;
2546         struct i40e_new_mac_filter *new, *add_head = NULL;
2547         struct i40e_hw *hw = &vsi->back->hw;
2548         bool old_overflow, new_overflow;
2549         unsigned int failed_filters = 0;
2550         unsigned int vlan_filters = 0;
2551         char vsi_name[16] = "PF";
2552         int filter_list_len = 0;
2553         u32 changed_flags = 0;
2554         struct hlist_node *h;
2555         struct i40e_pf *pf;
2556         int num_add = 0;
2557         int num_del = 0;
2558         int aq_ret = 0;
2559         int retval = 0;
2560         u16 cmd_flags;
2561         int list_size;
2562         int bkt;
2563
2564         /* empty array typed pointers, kcalloc later */
2565         struct i40e_aqc_add_macvlan_element_data *add_list;
2566         struct i40e_aqc_remove_macvlan_element_data *del_list;
2567
2568         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2569                 usleep_range(1000, 2000);
2570         pf = vsi->back;
2571
2572         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2573
2574         if (vsi->netdev) {
2575                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2576                 vsi->current_netdev_flags = vsi->netdev->flags;
2577         }
2578
2579         INIT_HLIST_HEAD(&tmp_add_list);
2580         INIT_HLIST_HEAD(&tmp_del_list);
2581
2582         if (vsi->type == I40E_VSI_SRIOV)
2583                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2584         else if (vsi->type != I40E_VSI_MAIN)
2585                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2586
2587         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2588                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2589
2590                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2591                 /* Create a list of filters to delete. */
2592                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2593                         if (f->state == I40E_FILTER_REMOVE) {
2594                                 /* Move the element into temporary del_list */
2595                                 hash_del(&f->hlist);
2596                                 hlist_add_head(&f->hlist, &tmp_del_list);
2597
2598                                 /* Avoid counting removed filters */
2599                                 continue;
2600                         }
2601                         if (f->state == I40E_FILTER_NEW) {
2602                                 /* Create a temporary i40e_new_mac_filter */
2603                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2604                                 if (!new)
2605                                         goto err_no_memory_locked;
2606
2607                                 /* Store pointer to the real filter */
2608                                 new->f = f;
2609                                 new->state = f->state;
2610
2611                                 /* Add it to the hash list */
2612                                 hlist_add_head(&new->hlist, &tmp_add_list);
2613                         }
2614
2615                         /* Count the number of active (current and new) VLAN
2616                          * filters we have now. Does not count filters which
2617                          * are marked for deletion.
2618                          */
2619                         if (f->vlan > 0)
2620                                 vlan_filters++;
2621                 }
2622
2623                 if (vsi->type != I40E_VSI_SRIOV)
2624                         retval = i40e_correct_mac_vlan_filters
2625                                 (vsi, &tmp_add_list, &tmp_del_list,
2626                                  vlan_filters);
2627                 else if (pf->vf)
2628                         retval = i40e_correct_vf_mac_vlan_filters
2629                                 (vsi, &tmp_add_list, &tmp_del_list,
2630                                  vlan_filters, pf->vf[vsi->vf_id].trusted);
2631
2632                 hlist_for_each_entry(new, &tmp_add_list, hlist)
2633                         netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2634
2635                 if (retval)
2636                         goto err_no_memory_locked;
2637
2638                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2639         }
2640
2641         /* Now process 'del_list' outside the lock */
2642         if (!hlist_empty(&tmp_del_list)) {
2643                 filter_list_len = hw->aq.asq_buf_size /
2644                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2645                 list_size = filter_list_len *
2646                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2647                 del_list = kzalloc(list_size, GFP_ATOMIC);
2648                 if (!del_list)
2649                         goto err_no_memory;
2650
2651                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2652                         cmd_flags = 0;
2653
2654                         /* handle broadcast filters by updating the broadcast
2655                          * promiscuous flag and release filter list.
2656                          */
2657                         if (is_broadcast_ether_addr(f->macaddr)) {
2658                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2659
2660                                 hlist_del(&f->hlist);
2661                                 kfree(f);
2662                                 continue;
2663                         }
2664
2665                         /* add to delete list */
2666                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2667                         if (f->vlan == I40E_VLAN_ANY) {
2668                                 del_list[num_del].vlan_tag = 0;
2669                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2670                         } else {
2671                                 del_list[num_del].vlan_tag =
2672                                         cpu_to_le16((u16)(f->vlan));
2673                         }
2674
2675                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2676                         del_list[num_del].flags = cmd_flags;
2677                         num_del++;
2678
2679                         /* flush a full buffer */
2680                         if (num_del == filter_list_len) {
2681                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2682                                                      num_del, &retval);
2683                                 memset(del_list, 0, list_size);
2684                                 num_del = 0;
2685                         }
2686                         /* Release memory for MAC filter entries which were
2687                          * synced up with HW.
2688                          */
2689                         hlist_del(&f->hlist);
2690                         kfree(f);
2691                 }
2692
2693                 if (num_del) {
2694                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2695                                              num_del, &retval);
2696                 }
2697
2698                 kfree(del_list);
2699                 del_list = NULL;
2700         }
2701
2702         if (!hlist_empty(&tmp_add_list)) {
2703                 /* Do all the adds now. */
2704                 filter_list_len = hw->aq.asq_buf_size /
2705                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2706                 list_size = filter_list_len *
2707                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2708                 add_list = kzalloc(list_size, GFP_ATOMIC);
2709                 if (!add_list)
2710                         goto err_no_memory;
2711
2712                 num_add = 0;
2713                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2714                         /* handle broadcast filters by updating the broadcast
2715                          * promiscuous flag instead of adding a MAC filter.
2716                          */
2717                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2718                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2719                                                               new->f))
2720                                         new->state = I40E_FILTER_FAILED;
2721                                 else
2722                                         new->state = I40E_FILTER_ACTIVE;
2723                                 continue;
2724                         }
2725
2726                         /* add to add array */
2727                         if (num_add == 0)
2728                                 add_head = new;
2729                         cmd_flags = 0;
2730                         ether_addr_copy(add_list[num_add].mac_addr,
2731                                         new->f->macaddr);
2732                         if (new->f->vlan == I40E_VLAN_ANY) {
2733                                 add_list[num_add].vlan_tag = 0;
2734                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2735                         } else {
2736                                 add_list[num_add].vlan_tag =
2737                                         cpu_to_le16((u16)(new->f->vlan));
2738                         }
2739                         add_list[num_add].queue_number = 0;
2740                         /* set invalid match method for later detection */
2741                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2742                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2743                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2744                         num_add++;
2745
2746                         /* flush a full buffer */
2747                         if (num_add == filter_list_len) {
2748                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2749                                                      add_head, num_add);
2750                                 memset(add_list, 0, list_size);
2751                                 num_add = 0;
2752                         }
2753                 }
2754                 if (num_add) {
2755                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2756                                              num_add);
2757                 }
2758                 /* Now move all of the filters from the temp add list back to
2759                  * the VSI's list.
2760                  */
2761                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2762                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2763                         /* Only update the state if we're still NEW */
2764                         if (new->f->state == I40E_FILTER_NEW)
2765                                 new->f->state = new->state;
2766                         hlist_del(&new->hlist);
2767                         netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2768                         kfree(new);
2769                 }
2770                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2771                 kfree(add_list);
2772                 add_list = NULL;
2773         }
2774
2775         /* Determine the number of active and failed filters. */
2776         spin_lock_bh(&vsi->mac_filter_hash_lock);
2777         vsi->active_filters = 0;
2778         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2779                 if (f->state == I40E_FILTER_ACTIVE)
2780                         vsi->active_filters++;
2781                 else if (f->state == I40E_FILTER_FAILED)
2782                         failed_filters++;
2783         }
2784         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2785
2786         /* Check if we are able to exit overflow promiscuous mode. We can
2787          * safely exit if we didn't just enter, we no longer have any failed
2788          * filters, and we have reduced filters below the threshold value.
2789          */
2790         if (old_overflow && !failed_filters &&
2791             vsi->active_filters < vsi->promisc_threshold) {
2792                 dev_info(&pf->pdev->dev,
2793                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2794                          vsi_name);
2795                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2796                 vsi->promisc_threshold = 0;
2797         }
2798
2799         /* if the VF is not trusted do not do promisc */
2800         if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2801             !pf->vf[vsi->vf_id].trusted) {
2802                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2803                 goto out;
2804         }
2805
2806         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2807
2808         /* If we are entering overflow promiscuous, we need to calculate a new
2809          * threshold for when we are safe to exit
2810          */
2811         if (!old_overflow && new_overflow)
2812                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2813
2814         /* check for changes in promiscuous modes */
2815         if (changed_flags & IFF_ALLMULTI) {
2816                 bool cur_multipromisc;
2817
2818                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2819                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2820                                                                vsi->seid,
2821                                                                cur_multipromisc,
2822                                                                NULL);
2823                 if (aq_ret) {
2824                         retval = i40e_aq_rc_to_posix(aq_ret,
2825                                                      hw->aq.asq_last_status);
2826                         dev_info(&pf->pdev->dev,
2827                                  "set multi promisc failed on %s, err %pe aq_err %s\n",
2828                                  vsi_name,
2829                                  ERR_PTR(aq_ret),
2830                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2831                 } else {
2832                         dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2833                                  cur_multipromisc ? "entering" : "leaving");
2834                 }
2835         }
2836
2837         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2838                 bool cur_promisc;
2839
2840                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2841                                new_overflow);
2842                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2843                 if (aq_ret) {
2844                         retval = i40e_aq_rc_to_posix(aq_ret,
2845                                                      hw->aq.asq_last_status);
2846                         dev_info(&pf->pdev->dev,
2847                                  "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2848                                  cur_promisc ? "on" : "off",
2849                                  vsi_name,
2850                                  ERR_PTR(aq_ret),
2851                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2852                 }
2853         }
2854 out:
2855         /* if something went wrong then set the changed flag so we try again */
2856         if (retval)
2857                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2858
2859         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2860         return retval;
2861
2862 err_no_memory:
2863         /* Restore elements on the temporary add and delete lists */
2864         spin_lock_bh(&vsi->mac_filter_hash_lock);
2865 err_no_memory_locked:
2866         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2867         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2868         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2869
2870         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2871         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2872         return -ENOMEM;
2873 }
2874
2875 /**
2876  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2877  * @pf: board private structure
2878  **/
2879 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2880 {
2881         struct i40e_vsi *vsi;
2882         int v;
2883
2884         if (!pf)
2885                 return;
2886         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2887                 return;
2888         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2889                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2890                 return;
2891         }
2892
2893         i40e_pf_for_each_vsi(pf, v, vsi) {
2894                 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2895                     !test_bit(__I40E_VSI_RELEASING, vsi->state)) {
2896                         int ret = i40e_sync_vsi_filters(vsi);
2897
2898                         if (ret) {
2899                                 /* come back and try again later */
2900                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2901                                         pf->state);
2902                                 break;
2903                         }
2904                 }
2905         }
2906 }
2907
2908 /**
2909  * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2910  *
2911  * @vsi: VSI to calculate rx_buf_len from
2912  */
2913 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2914 {
2915         if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
2916                 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2917
2918         return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2919 }
2920
2921 /**
2922  * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2923  * @vsi: the vsi
2924  * @xdp_prog: XDP program
2925  **/
2926 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2927                                    struct bpf_prog *xdp_prog)
2928 {
2929         u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2930         u16 chain_len;
2931
2932         if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2933                 chain_len = 1;
2934         else
2935                 chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2936
2937         return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2938 }
2939
2940 /**
2941  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2942  * @netdev: network interface device structure
2943  * @new_mtu: new value for maximum frame size
2944  *
2945  * Returns 0 on success, negative on failure
2946  **/
2947 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2948 {
2949         struct i40e_netdev_priv *np = netdev_priv(netdev);
2950         struct i40e_vsi *vsi = np->vsi;
2951         struct i40e_pf *pf = vsi->back;
2952         int frame_size;
2953
2954         frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
2955         if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2956                 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
2957                            new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2958                 return -EINVAL;
2959         }
2960
2961         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2962                    netdev->mtu, new_mtu);
2963         netdev->mtu = new_mtu;
2964         if (netif_running(netdev))
2965                 i40e_vsi_reinit_locked(vsi);
2966         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2967         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2968         return 0;
2969 }
2970
2971 /**
2972  * i40e_ioctl - Access the hwtstamp interface
2973  * @netdev: network interface device structure
2974  * @ifr: interface request data
2975  * @cmd: ioctl command
2976  **/
2977 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2978 {
2979         struct i40e_netdev_priv *np = netdev_priv(netdev);
2980         struct i40e_pf *pf = np->vsi->back;
2981
2982         switch (cmd) {
2983         case SIOCGHWTSTAMP:
2984                 return i40e_ptp_get_ts_config(pf, ifr);
2985         case SIOCSHWTSTAMP:
2986                 return i40e_ptp_set_ts_config(pf, ifr);
2987         default:
2988                 return -EOPNOTSUPP;
2989         }
2990 }
2991
2992 /**
2993  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2994  * @vsi: the vsi being adjusted
2995  **/
2996 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2997 {
2998         struct i40e_vsi_context ctxt;
2999         int ret;
3000
3001         /* Don't modify stripping options if a port VLAN is active */
3002         if (vsi->info.pvid)
3003                 return;
3004
3005         if ((vsi->info.valid_sections &
3006              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3007             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
3008                 return;  /* already enabled */
3009
3010         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3011         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3012                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3013
3014         ctxt.seid = vsi->seid;
3015         ctxt.info = vsi->info;
3016         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3017         if (ret) {
3018                 dev_info(&vsi->back->pdev->dev,
3019                          "update vlan stripping failed, err %pe aq_err %s\n",
3020                          ERR_PTR(ret),
3021                          i40e_aq_str(&vsi->back->hw,
3022                                      vsi->back->hw.aq.asq_last_status));
3023         }
3024 }
3025
3026 /**
3027  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3028  * @vsi: the vsi being adjusted
3029  **/
3030 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3031 {
3032         struct i40e_vsi_context ctxt;
3033         int ret;
3034
3035         /* Don't modify stripping options if a port VLAN is active */
3036         if (vsi->info.pvid)
3037                 return;
3038
3039         if ((vsi->info.valid_sections &
3040              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3041             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3042              I40E_AQ_VSI_PVLAN_EMOD_MASK))
3043                 return;  /* already disabled */
3044
3045         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3046         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3047                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3048
3049         ctxt.seid = vsi->seid;
3050         ctxt.info = vsi->info;
3051         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3052         if (ret) {
3053                 dev_info(&vsi->back->pdev->dev,
3054                          "update vlan stripping failed, err %pe aq_err %s\n",
3055                          ERR_PTR(ret),
3056                          i40e_aq_str(&vsi->back->hw,
3057                                      vsi->back->hw.aq.asq_last_status));
3058         }
3059 }
3060
3061 /**
3062  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3063  * @vsi: the vsi being configured
3064  * @vid: vlan id to be added (0 = untagged only , -1 = any)
3065  *
3066  * This is a helper function for adding a new MAC/VLAN filter with the
3067  * specified VLAN for each existing MAC address already in the hash table.
3068  * This function does *not* perform any accounting to update filters based on
3069  * VLAN mode.
3070  *
3071  * NOTE: this function expects to be called while under the
3072  * mac_filter_hash_lock
3073  **/
3074 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3075 {
3076         struct i40e_mac_filter *f, *add_f;
3077         struct hlist_node *h;
3078         int bkt;
3079
3080         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3081                 /* If we're asked to add a filter that has been marked for
3082                  * removal, it is safe to simply restore it to active state.
3083                  * __i40e_del_filter will have simply deleted any filters which
3084                  * were previously marked NEW or FAILED, so if it is currently
3085                  * marked REMOVE it must have previously been ACTIVE. Since we
3086                  * haven't yet run the sync filters task, just restore this
3087                  * filter to the ACTIVE state so that the sync task leaves it
3088                  * in place.
3089                  */
3090                 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3091                         f->state = I40E_FILTER_ACTIVE;
3092                         continue;
3093                 } else if (f->state == I40E_FILTER_REMOVE) {
3094                         continue;
3095                 }
3096                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
3097                 if (!add_f) {
3098                         dev_info(&vsi->back->pdev->dev,
3099                                  "Could not add vlan filter %d for %pM\n",
3100                                  vid, f->macaddr);
3101                         return -ENOMEM;
3102                 }
3103         }
3104
3105         return 0;
3106 }
3107
3108 /**
3109  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3110  * @vsi: the VSI being configured
3111  * @vid: VLAN id to be added
3112  **/
3113 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3114 {
3115         int err;
3116
3117         if (vsi->info.pvid)
3118                 return -EINVAL;
3119
3120         /* The network stack will attempt to add VID=0, with the intention to
3121          * receive priority tagged packets with a VLAN of 0. Our HW receives
3122          * these packets by default when configured to receive untagged
3123          * packets, so we don't need to add a filter for this case.
3124          * Additionally, HW interprets adding a VID=0 filter as meaning to
3125          * receive *only* tagged traffic and stops receiving untagged traffic.
3126          * Thus, we do not want to actually add a filter for VID=0
3127          */
3128         if (!vid)
3129                 return 0;
3130
3131         /* Locked once because all functions invoked below iterates list*/
3132         spin_lock_bh(&vsi->mac_filter_hash_lock);
3133         err = i40e_add_vlan_all_mac(vsi, vid);
3134         spin_unlock_bh(&vsi->mac_filter_hash_lock);
3135         if (err)
3136                 return err;
3137
3138         /* schedule our worker thread which will take care of
3139          * applying the new filter changes
3140          */
3141         i40e_service_event_schedule(vsi->back);
3142         return 0;
3143 }
3144
3145 /**
3146  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3147  * @vsi: the vsi being configured
3148  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3149  *
3150  * This function should be used to remove all VLAN filters which match the
3151  * given VID. It does not schedule the service event and does not take the
3152  * mac_filter_hash_lock so it may be combined with other operations under
3153  * a single invocation of the mac_filter_hash_lock.
3154  *
3155  * NOTE: this function expects to be called while under the
3156  * mac_filter_hash_lock
3157  */
3158 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3159 {
3160         struct i40e_mac_filter *f;
3161         struct hlist_node *h;
3162         int bkt;
3163
3164         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3165                 if (f->vlan == vid)
3166                         __i40e_del_filter(vsi, f);
3167         }
3168 }
3169
3170 /**
3171  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3172  * @vsi: the VSI being configured
3173  * @vid: VLAN id to be removed
3174  **/
3175 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3176 {
3177         if (!vid || vsi->info.pvid)
3178                 return;
3179
3180         spin_lock_bh(&vsi->mac_filter_hash_lock);
3181         i40e_rm_vlan_all_mac(vsi, vid);
3182         spin_unlock_bh(&vsi->mac_filter_hash_lock);
3183
3184         /* schedule our worker thread which will take care of
3185          * applying the new filter changes
3186          */
3187         i40e_service_event_schedule(vsi->back);
3188 }
3189
3190 /**
3191  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3192  * @netdev: network interface to be adjusted
3193  * @proto: unused protocol value
3194  * @vid: vlan id to be added
3195  *
3196  * net_device_ops implementation for adding vlan ids
3197  **/
3198 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3199                                 __always_unused __be16 proto, u16 vid)
3200 {
3201         struct i40e_netdev_priv *np = netdev_priv(netdev);
3202         struct i40e_vsi *vsi = np->vsi;
3203         int ret = 0;
3204
3205         if (vid >= VLAN_N_VID)
3206                 return -EINVAL;
3207
3208         ret = i40e_vsi_add_vlan(vsi, vid);
3209         if (!ret)
3210                 set_bit(vid, vsi->active_vlans);
3211
3212         return ret;
3213 }
3214
3215 /**
3216  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3217  * @netdev: network interface to be adjusted
3218  * @proto: unused protocol value
3219  * @vid: vlan id to be added
3220  **/
3221 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3222                                     __always_unused __be16 proto, u16 vid)
3223 {
3224         struct i40e_netdev_priv *np = netdev_priv(netdev);
3225         struct i40e_vsi *vsi = np->vsi;
3226
3227         if (vid >= VLAN_N_VID)
3228                 return;
3229         set_bit(vid, vsi->active_vlans);
3230 }
3231
3232 /**
3233  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3234  * @netdev: network interface to be adjusted
3235  * @proto: unused protocol value
3236  * @vid: vlan id to be removed
3237  *
3238  * net_device_ops implementation for removing vlan ids
3239  **/
3240 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3241                                  __always_unused __be16 proto, u16 vid)
3242 {
3243         struct i40e_netdev_priv *np = netdev_priv(netdev);
3244         struct i40e_vsi *vsi = np->vsi;
3245
3246         /* return code is ignored as there is nothing a user
3247          * can do about failure to remove and a log message was
3248          * already printed from the other function
3249          */
3250         i40e_vsi_kill_vlan(vsi, vid);
3251
3252         clear_bit(vid, vsi->active_vlans);
3253
3254         return 0;
3255 }
3256
3257 /**
3258  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3259  * @vsi: the vsi being brought back up
3260  **/
3261 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3262 {
3263         u16 vid;
3264
3265         if (!vsi->netdev)
3266                 return;
3267
3268         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3269                 i40e_vlan_stripping_enable(vsi);
3270         else
3271                 i40e_vlan_stripping_disable(vsi);
3272
3273         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3274                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3275                                         vid);
3276 }
3277
3278 /**
3279  * i40e_vsi_add_pvid - Add pvid for the VSI
3280  * @vsi: the vsi being adjusted
3281  * @vid: the vlan id to set as a PVID
3282  **/
3283 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3284 {
3285         struct i40e_vsi_context ctxt;
3286         int ret;
3287
3288         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3289         vsi->info.pvid = cpu_to_le16(vid);
3290         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3291                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
3292                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
3293
3294         ctxt.seid = vsi->seid;
3295         ctxt.info = vsi->info;
3296         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3297         if (ret) {
3298                 dev_info(&vsi->back->pdev->dev,
3299                          "add pvid failed, err %pe aq_err %s\n",
3300                          ERR_PTR(ret),
3301                          i40e_aq_str(&vsi->back->hw,
3302                                      vsi->back->hw.aq.asq_last_status));
3303                 return -ENOENT;
3304         }
3305
3306         return 0;
3307 }
3308
3309 /**
3310  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3311  * @vsi: the vsi being adjusted
3312  *
3313  * Just use the vlan_rx_register() service to put it back to normal
3314  **/
3315 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3316 {
3317         vsi->info.pvid = 0;
3318
3319         i40e_vlan_stripping_disable(vsi);
3320 }
3321
3322 /**
3323  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3324  * @vsi: ptr to the VSI
3325  *
3326  * If this function returns with an error, then it's possible one or
3327  * more of the rings is populated (while the rest are not).  It is the
3328  * callers duty to clean those orphaned rings.
3329  *
3330  * Return 0 on success, negative on failure
3331  **/
3332 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3333 {
3334         int i, err = 0;
3335
3336         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3337                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3338
3339         if (!i40e_enabled_xdp_vsi(vsi))
3340                 return err;
3341
3342         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3343                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3344
3345         return err;
3346 }
3347
3348 /**
3349  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3350  * @vsi: ptr to the VSI
3351  *
3352  * Free VSI's transmit software resources
3353  **/
3354 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3355 {
3356         int i;
3357
3358         if (vsi->tx_rings) {
3359                 for (i = 0; i < vsi->num_queue_pairs; i++)
3360                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3361                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3362         }
3363
3364         if (vsi->xdp_rings) {
3365                 for (i = 0; i < vsi->num_queue_pairs; i++)
3366                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3367                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3368         }
3369 }
3370
3371 /**
3372  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3373  * @vsi: ptr to the VSI
3374  *
3375  * If this function returns with an error, then it's possible one or
3376  * more of the rings is populated (while the rest are not).  It is the
3377  * callers duty to clean those orphaned rings.
3378  *
3379  * Return 0 on success, negative on failure
3380  **/
3381 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3382 {
3383         int i, err = 0;
3384
3385         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3386                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3387         return err;
3388 }
3389
3390 /**
3391  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3392  * @vsi: ptr to the VSI
3393  *
3394  * Free all receive software resources
3395  **/
3396 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3397 {
3398         int i;
3399
3400         if (!vsi->rx_rings)
3401                 return;
3402
3403         for (i = 0; i < vsi->num_queue_pairs; i++)
3404                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3405                         i40e_free_rx_resources(vsi->rx_rings[i]);
3406 }
3407
3408 /**
3409  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3410  * @ring: The Tx ring to configure
3411  *
3412  * This enables/disables XPS for a given Tx descriptor ring
3413  * based on the TCs enabled for the VSI that ring belongs to.
3414  **/
3415 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3416 {
3417         int cpu;
3418
3419         if (!ring->q_vector || !ring->netdev || ring->ch)
3420                 return;
3421
3422         /* We only initialize XPS once, so as not to overwrite user settings */
3423         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3424                 return;
3425
3426         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3427         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3428                             ring->queue_index);
3429 }
3430
3431 /**
3432  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3433  * @ring: The Tx or Rx ring
3434  *
3435  * Returns the AF_XDP buffer pool or NULL.
3436  **/
3437 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3438 {
3439         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3440         int qid = ring->queue_index;
3441
3442         if (ring_is_xdp(ring))
3443                 qid -= ring->vsi->alloc_queue_pairs;
3444
3445         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3446                 return NULL;
3447
3448         return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3449 }
3450
3451 /**
3452  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3453  * @ring: The Tx ring to configure
3454  *
3455  * Configure the Tx descriptor ring in the HMC context.
3456  **/
3457 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3458 {
3459         struct i40e_vsi *vsi = ring->vsi;
3460         u16 pf_q = vsi->base_queue + ring->queue_index;
3461         struct i40e_hw *hw = &vsi->back->hw;
3462         struct i40e_hmc_obj_txq tx_ctx;
3463         u32 qtx_ctl = 0;
3464         int err = 0;
3465
3466         if (ring_is_xdp(ring))
3467                 ring->xsk_pool = i40e_xsk_pool(ring);
3468
3469         /* some ATR related tx ring init */
3470         if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
3471                 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
3472                 ring->atr_count = 0;
3473         } else {
3474                 ring->atr_sample_rate = 0;
3475         }
3476
3477         /* configure XPS */
3478         i40e_config_xps_tx_ring(ring);
3479
3480         /* clear the context structure first */
3481         memset(&tx_ctx, 0, sizeof(tx_ctx));
3482
3483         tx_ctx.new_context = 1;
3484         tx_ctx.base = (ring->dma / 128);
3485         tx_ctx.qlen = ring->count;
3486         if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
3487             test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
3488                 tx_ctx.fd_ena = 1;
3489         if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
3490                 tx_ctx.timesync_ena = 1;
3491         /* FDIR VSI tx ring can still use RS bit and writebacks */
3492         if (vsi->type != I40E_VSI_FDIR)
3493                 tx_ctx.head_wb_ena = 1;
3494         tx_ctx.head_wb_addr = ring->dma +
3495                               (ring->count * sizeof(struct i40e_tx_desc));
3496
3497         /* As part of VSI creation/update, FW allocates certain
3498          * Tx arbitration queue sets for each TC enabled for
3499          * the VSI. The FW returns the handles to these queue
3500          * sets as part of the response buffer to Add VSI,
3501          * Update VSI, etc. AQ commands. It is expected that
3502          * these queue set handles be associated with the Tx
3503          * queues by the driver as part of the TX queue context
3504          * initialization. This has to be done regardless of
3505          * DCB as by default everything is mapped to TC0.
3506          */
3507
3508         if (ring->ch)
3509                 tx_ctx.rdylist =
3510                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3511
3512         else
3513                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3514
3515         tx_ctx.rdylist_act = 0;
3516
3517         /* clear the context in the HMC */
3518         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3519         if (err) {
3520                 dev_info(&vsi->back->pdev->dev,
3521                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3522                          ring->queue_index, pf_q, err);
3523                 return -ENOMEM;
3524         }
3525
3526         /* set the context in the HMC */
3527         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3528         if (err) {
3529                 dev_info(&vsi->back->pdev->dev,
3530                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3531                          ring->queue_index, pf_q, err);
3532                 return -ENOMEM;
3533         }
3534
3535         /* Now associate this queue with this PCI function */
3536         if (ring->ch) {
3537                 if (ring->ch->type == I40E_VSI_VMDQ2)
3538                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3539                 else
3540                         return -EINVAL;
3541
3542                 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3543                                       ring->ch->vsi_number);
3544         } else {
3545                 if (vsi->type == I40E_VSI_VMDQ2) {
3546                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3547                         qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3548                                               vsi->id);
3549                 } else {
3550                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3551                 }
3552         }
3553
3554         qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);
3555         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3556         i40e_flush(hw);
3557
3558         /* cache tail off for easier writes later */
3559         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3560
3561         return 0;
3562 }
3563
3564 /**
3565  * i40e_rx_offset - Return expected offset into page to access data
3566  * @rx_ring: Ring we are requesting offset of
3567  *
3568  * Returns the offset value for ring into the data buffer.
3569  */
3570 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3571 {
3572         return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3573 }
3574
3575 /**
3576  * i40e_configure_rx_ring - Configure a receive ring context
3577  * @ring: The Rx ring to configure
3578  *
3579  * Configure the Rx descriptor ring in the HMC context.
3580  **/
3581 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3582 {
3583         struct i40e_vsi *vsi = ring->vsi;
3584         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3585         u16 pf_q = vsi->base_queue + ring->queue_index;
3586         struct i40e_hw *hw = &vsi->back->hw;
3587         struct i40e_hmc_obj_rxq rx_ctx;
3588         int err = 0;
3589         bool ok;
3590
3591         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3592
3593         /* clear the context structure first */
3594         memset(&rx_ctx, 0, sizeof(rx_ctx));
3595
3596         ring->rx_buf_len = vsi->rx_buf_len;
3597
3598         /* XDP RX-queue info only needed for RX rings exposed to XDP */
3599         if (ring->vsi->type != I40E_VSI_MAIN)
3600                 goto skip;
3601
3602         if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
3603                 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3604                                          ring->queue_index,
3605                                          ring->q_vector->napi.napi_id,
3606                                          ring->rx_buf_len);
3607                 if (err)
3608                         return err;
3609         }
3610
3611         ring->xsk_pool = i40e_xsk_pool(ring);
3612         if (ring->xsk_pool) {
3613                 xdp_rxq_info_unreg(&ring->xdp_rxq);
3614                 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);
3615                 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3616                                          ring->queue_index,
3617                                          ring->q_vector->napi.napi_id,
3618                                          ring->rx_buf_len);
3619                 if (err)
3620                         return err;
3621                 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3622                                                  MEM_TYPE_XSK_BUFF_POOL,
3623                                                  NULL);
3624                 if (err)
3625                         return err;
3626                 dev_info(&vsi->back->pdev->dev,
3627                          "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3628                          ring->queue_index);
3629
3630         } else {
3631                 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3632                                                  MEM_TYPE_PAGE_SHARED,
3633                                                  NULL);
3634                 if (err)
3635                         return err;
3636         }
3637
3638 skip:
3639         xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
3640
3641         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3642                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3643
3644         rx_ctx.base = (ring->dma / 128);
3645         rx_ctx.qlen = ring->count;
3646
3647         /* use 16 byte descriptors */
3648         rx_ctx.dsize = 0;
3649
3650         /* descriptor type is always zero
3651          * rx_ctx.dtype = 0;
3652          */
3653         rx_ctx.hsplit_0 = 0;
3654
3655         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3656         if (hw->revision_id == 0)
3657                 rx_ctx.lrxqthresh = 0;
3658         else
3659                 rx_ctx.lrxqthresh = 1;
3660         rx_ctx.crcstrip = 1;
3661         rx_ctx.l2tsel = 1;
3662         /* this controls whether VLAN is stripped from inner headers */
3663         rx_ctx.showiv = 0;
3664         /* set the prefena field to 1 because the manual says to */
3665         rx_ctx.prefena = 1;
3666
3667         /* clear the context in the HMC */
3668         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3669         if (err) {
3670                 dev_info(&vsi->back->pdev->dev,
3671                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3672                          ring->queue_index, pf_q, err);
3673                 return -ENOMEM;
3674         }
3675
3676         /* set the context in the HMC */
3677         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3678         if (err) {
3679                 dev_info(&vsi->back->pdev->dev,
3680                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3681                          ring->queue_index, pf_q, err);
3682                 return -ENOMEM;
3683         }
3684
3685         /* configure Rx buffer alignment */
3686         if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
3687                 if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3688                         dev_info(&vsi->back->pdev->dev,
3689                                  "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3690                         return -EOPNOTSUPP;
3691                 }
3692                 clear_ring_build_skb_enabled(ring);
3693         } else {
3694                 set_ring_build_skb_enabled(ring);
3695         }
3696
3697         ring->rx_offset = i40e_rx_offset(ring);
3698
3699         /* cache tail for quicker writes, and clear the reg before use */
3700         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3701         writel(0, ring->tail);
3702
3703         if (ring->xsk_pool) {
3704                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3705                 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3706         } else {
3707                 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3708         }
3709         if (!ok) {
3710                 /* Log this in case the user has forgotten to give the kernel
3711                  * any buffers, even later in the application.
3712                  */
3713                 dev_info(&vsi->back->pdev->dev,
3714                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3715                          ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3716                          ring->queue_index, pf_q);
3717         }
3718
3719         return 0;
3720 }
3721
3722 /**
3723  * i40e_vsi_configure_tx - Configure the VSI for Tx
3724  * @vsi: VSI structure describing this set of rings and resources
3725  *
3726  * Configure the Tx VSI for operation.
3727  **/
3728 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3729 {
3730         int err = 0;
3731         u16 i;
3732
3733         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3734                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3735
3736         if (err || !i40e_enabled_xdp_vsi(vsi))
3737                 return err;
3738
3739         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3740                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3741
3742         return err;
3743 }
3744
3745 /**
3746  * i40e_vsi_configure_rx - Configure the VSI for Rx
3747  * @vsi: the VSI being configured
3748  *
3749  * Configure the Rx VSI for operation.
3750  **/
3751 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3752 {
3753         int err = 0;
3754         u16 i;
3755
3756         vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
3757         vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3758
3759 #if (PAGE_SIZE < 8192)
3760         if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3761             vsi->netdev->mtu <= ETH_DATA_LEN) {
3762                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3763                 vsi->max_frame = vsi->rx_buf_len;
3764         }
3765 #endif
3766
3767         /* set up individual rings */
3768         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3769                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3770
3771         return err;
3772 }
3773
3774 /**
3775  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3776  * @vsi: ptr to the VSI
3777  **/
3778 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3779 {
3780         struct i40e_ring *tx_ring, *rx_ring;
3781         u16 qoffset, qcount;
3782         int i, n;
3783
3784         if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
3785                 /* Reset the TC information */
3786                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3787                         rx_ring = vsi->rx_rings[i];
3788                         tx_ring = vsi->tx_rings[i];
3789                         rx_ring->dcb_tc = 0;
3790                         tx_ring->dcb_tc = 0;
3791                 }
3792                 return;
3793         }
3794
3795         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3796                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3797                         continue;
3798
3799                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3800                 qcount = vsi->tc_config.tc_info[n].qcount;
3801                 for (i = qoffset; i < (qoffset + qcount); i++) {
3802                         rx_ring = vsi->rx_rings[i];
3803                         tx_ring = vsi->tx_rings[i];
3804                         rx_ring->dcb_tc = n;
3805                         tx_ring->dcb_tc = n;
3806                 }
3807         }
3808 }
3809
3810 /**
3811  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3812  * @vsi: ptr to the VSI
3813  **/
3814 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3815 {
3816         if (vsi->netdev)
3817                 i40e_set_rx_mode(vsi->netdev);
3818 }
3819
3820 /**
3821  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3822  * @pf: Pointer to the targeted PF
3823  *
3824  * Set all flow director counters to 0.
3825  */
3826 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3827 {
3828         pf->fd_tcp4_filter_cnt = 0;
3829         pf->fd_udp4_filter_cnt = 0;
3830         pf->fd_sctp4_filter_cnt = 0;
3831         pf->fd_ip4_filter_cnt = 0;
3832         pf->fd_tcp6_filter_cnt = 0;
3833         pf->fd_udp6_filter_cnt = 0;
3834         pf->fd_sctp6_filter_cnt = 0;
3835         pf->fd_ip6_filter_cnt = 0;
3836 }
3837
3838 /**
3839  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3840  * @vsi: Pointer to the targeted VSI
3841  *
3842  * This function replays the hlist on the hw where all the SB Flow Director
3843  * filters were saved.
3844  **/
3845 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3846 {
3847         struct i40e_fdir_filter *filter;
3848         struct i40e_pf *pf = vsi->back;
3849         struct hlist_node *node;
3850
3851         if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
3852                 return;
3853
3854         /* Reset FDir counters as we're replaying all existing filters */
3855         i40e_reset_fdir_filter_cnt(pf);
3856
3857         hlist_for_each_entry_safe(filter, node,
3858                                   &pf->fdir_filter_list, fdir_node) {
3859                 i40e_add_del_fdir(vsi, filter, true);
3860         }
3861 }
3862
3863 /**
3864  * i40e_vsi_configure - Set up the VSI for action
3865  * @vsi: the VSI being configured
3866  **/
3867 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3868 {
3869         int err;
3870
3871         i40e_set_vsi_rx_mode(vsi);
3872         i40e_restore_vlan(vsi);
3873         i40e_vsi_config_dcb_rings(vsi);
3874         err = i40e_vsi_configure_tx(vsi);
3875         if (!err)
3876                 err = i40e_vsi_configure_rx(vsi);
3877
3878         return err;
3879 }
3880
3881 /**
3882  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3883  * @vsi: the VSI being configured
3884  **/
3885 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3886 {
3887         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3888         struct i40e_pf *pf = vsi->back;
3889         struct i40e_hw *hw = &pf->hw;
3890         u16 vector;
3891         int i, q;
3892         u32 qp;
3893
3894         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3895          * and PFINT_LNKLSTn registers, e.g.:
3896          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3897          */
3898         qp = vsi->base_queue;
3899         vector = vsi->base_vector;
3900         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3901                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3902
3903                 q_vector->rx.next_update = jiffies + 1;
3904                 q_vector->rx.target_itr =
3905                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3906                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3907                      q_vector->rx.target_itr >> 1);
3908                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3909
3910                 q_vector->tx.next_update = jiffies + 1;
3911                 q_vector->tx.target_itr =
3912                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3913                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3914                      q_vector->tx.target_itr >> 1);
3915                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3916
3917                 /* Set ITR for software interrupts triggered after exiting
3918                  * busy-loop polling.
3919                  */
3920                 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),
3921                      I40E_ITR_20K);
3922
3923                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3924                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3925
3926                 /* begin of linked list for RX queue assigned to this vector */
3927                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3928                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3929                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3930                         u32 val;
3931
3932                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3933                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3934                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3935                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3936                               (I40E_QUEUE_TYPE_TX <<
3937                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3938
3939                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3940
3941                         if (has_xdp) {
3942                                 /* TX queue with next queue set to TX */
3943                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3944                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3945                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3946                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3947                                       (I40E_QUEUE_TYPE_TX <<
3948                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3949
3950                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3951                         }
3952                         /* TX queue with next RX or end of linked list */
3953                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3954                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3955                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3956                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3957                               (I40E_QUEUE_TYPE_RX <<
3958                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3959
3960                         /* Terminate the linked list */
3961                         if (q == (q_vector->num_ringpairs - 1))
3962                                 val |= (I40E_QUEUE_END_OF_LIST <<
3963                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3964
3965                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3966                         qp++;
3967                 }
3968         }
3969
3970         i40e_flush(hw);
3971 }
3972
3973 /**
3974  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3975  * @pf: pointer to private device data structure
3976  **/
3977 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3978 {
3979         struct i40e_hw *hw = &pf->hw;
3980         u32 val;
3981
3982         /* clear things first */
3983         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3984         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3985
3986         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3987               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3988               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3989               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3990               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3991               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3992               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3993               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3994
3995         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
3996                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3997
3998         if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
3999                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4000
4001         wr32(hw, I40E_PFINT_ICR0_ENA, val);
4002
4003         /* SW_ITR_IDX = 0, but don't change INTENA */
4004         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
4005                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
4006
4007         /* OTHER_ITR_IDX = 0 */
4008         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
4009 }
4010
4011 /**
4012  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
4013  * @vsi: the VSI being configured
4014  **/
4015 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
4016 {
4017         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
4018         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4019         struct i40e_pf *pf = vsi->back;
4020         struct i40e_hw *hw = &pf->hw;
4021
4022         /* set the ITR configuration */
4023         q_vector->rx.next_update = jiffies + 1;
4024         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
4025         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
4026         q_vector->rx.current_itr = q_vector->rx.target_itr;
4027         q_vector->tx.next_update = jiffies + 1;
4028         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
4029         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
4030         q_vector->tx.current_itr = q_vector->tx.target_itr;
4031
4032         i40e_enable_misc_int_causes(pf);
4033
4034         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
4035         wr32(hw, I40E_PFINT_LNKLST0, 0);
4036
4037         /* Associate the queue pair to the vector and enable the queue
4038          * interrupt RX queue in linked list with next queue set to TX
4039          */
4040         wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4041
4042         if (i40e_enabled_xdp_vsi(vsi)) {
4043                 /* TX queue in linked list with next queue set to TX */
4044                 wr32(hw, I40E_QINT_TQCTL(nextqp),
4045                      I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4046         }
4047
4048         /* last TX queue so the next RX queue doesn't matter */
4049         wr32(hw, I40E_QINT_TQCTL(0),
4050              I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4051         i40e_flush(hw);
4052 }
4053
4054 /**
4055  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4056  * @pf: board private structure
4057  **/
4058 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4059 {
4060         struct i40e_hw *hw = &pf->hw;
4061
4062         wr32(hw, I40E_PFINT_DYN_CTL0,
4063              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4064         i40e_flush(hw);
4065 }
4066
4067 /**
4068  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4069  * @pf: board private structure
4070  **/
4071 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4072 {
4073         struct i40e_hw *hw = &pf->hw;
4074         u32 val;
4075
4076         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
4077               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4078               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4079
4080         wr32(hw, I40E_PFINT_DYN_CTL0, val);
4081         i40e_flush(hw);
4082 }
4083
4084 /**
4085  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4086  * @irq: interrupt number
4087  * @data: pointer to a q_vector
4088  **/
4089 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4090 {
4091         struct i40e_q_vector *q_vector = data;
4092
4093         if (!q_vector->tx.ring && !q_vector->rx.ring)
4094                 return IRQ_HANDLED;
4095
4096         napi_schedule_irqoff(&q_vector->napi);
4097
4098         return IRQ_HANDLED;
4099 }
4100
4101 /**
4102  * i40e_irq_affinity_notify - Callback for affinity changes
4103  * @notify: context as to what irq was changed
4104  * @mask: the new affinity mask
4105  *
4106  * This is a callback function used by the irq_set_affinity_notifier function
4107  * so that we may register to receive changes to the irq affinity masks.
4108  **/
4109 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4110                                      const cpumask_t *mask)
4111 {
4112         struct i40e_q_vector *q_vector =
4113                 container_of(notify, struct i40e_q_vector, affinity_notify);
4114
4115         cpumask_copy(&q_vector->affinity_mask, mask);
4116 }
4117
4118 /**
4119  * i40e_irq_affinity_release - Callback for affinity notifier release
4120  * @ref: internal core kernel usage
4121  *
4122  * This is a callback function used by the irq_set_affinity_notifier function
4123  * to inform the current notification subscriber that they will no longer
4124  * receive notifications.
4125  **/
4126 static void i40e_irq_affinity_release(struct kref *ref) {}
4127
4128 /**
4129  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4130  * @vsi: the VSI being configured
4131  * @basename: name for the vector
4132  *
4133  * Allocates MSI-X vectors and requests interrupts from the kernel.
4134  **/
4135 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4136 {
4137         int q_vectors = vsi->num_q_vectors;
4138         struct i40e_pf *pf = vsi->back;
4139         int base = vsi->base_vector;
4140         int rx_int_idx = 0;
4141         int tx_int_idx = 0;
4142         int vector, err;
4143         int irq_num;
4144         int cpu;
4145
4146         for (vector = 0; vector < q_vectors; vector++) {
4147                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4148
4149                 irq_num = pf->msix_entries[base + vector].vector;
4150
4151                 if (q_vector->tx.ring && q_vector->rx.ring) {
4152                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4153                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4154                         tx_int_idx++;
4155                 } else if (q_vector->rx.ring) {
4156                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4157                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
4158                 } else if (q_vector->tx.ring) {
4159                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4160                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
4161                 } else {
4162                         /* skip this unused q_vector */
4163                         continue;
4164                 }
4165                 err = request_irq(irq_num,
4166                                   vsi->irq_handler,
4167                                   0,
4168                                   q_vector->name,
4169                                   q_vector);
4170                 if (err) {
4171                         dev_info(&pf->pdev->dev,
4172                                  "MSIX request_irq failed, error: %d\n", err);
4173                         goto free_queue_irqs;
4174                 }
4175
4176                 /* register for affinity change notifications */
4177                 q_vector->irq_num = irq_num;
4178                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4179                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
4180                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4181                 /* Spread affinity hints out across online CPUs.
4182                  *
4183                  * get_cpu_mask returns a static constant mask with
4184                  * a permanent lifetime so it's ok to pass to
4185                  * irq_update_affinity_hint without making a copy.
4186                  */
4187                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
4188                 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
4189         }
4190
4191         vsi->irqs_ready = true;
4192         return 0;
4193
4194 free_queue_irqs:
4195         while (vector) {
4196                 vector--;
4197                 irq_num = pf->msix_entries[base + vector].vector;
4198                 irq_set_affinity_notifier(irq_num, NULL);
4199                 irq_update_affinity_hint(irq_num, NULL);
4200                 free_irq(irq_num, &vsi->q_vectors[vector]);
4201         }
4202         return err;
4203 }
4204
4205 /**
4206  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4207  * @vsi: the VSI being un-configured
4208  **/
4209 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4210 {
4211         struct i40e_pf *pf = vsi->back;
4212         struct i40e_hw *hw = &pf->hw;
4213         int base = vsi->base_vector;
4214         int i;
4215
4216         /* disable interrupt causation from each queue */
4217         for (i = 0; i < vsi->num_queue_pairs; i++) {
4218                 u32 val;
4219
4220                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4221                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4222                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4223
4224                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4225                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4226                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4227
4228                 if (!i40e_enabled_xdp_vsi(vsi))
4229                         continue;
4230                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4231         }
4232
4233         /* disable each interrupt */
4234         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4235                 for (i = vsi->base_vector;
4236                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
4237                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4238
4239                 i40e_flush(hw);
4240                 for (i = 0; i < vsi->num_q_vectors; i++)
4241                         synchronize_irq(pf->msix_entries[i + base].vector);
4242         } else {
4243                 /* Legacy and MSI mode - this stops all interrupt handling */
4244                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4245                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4246                 i40e_flush(hw);
4247                 synchronize_irq(pf->pdev->irq);
4248         }
4249 }
4250
4251 /**
4252  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4253  * @vsi: the VSI being configured
4254  **/
4255 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4256 {
4257         struct i40e_pf *pf = vsi->back;
4258         int i;
4259
4260         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4261                 for (i = 0; i < vsi->num_q_vectors; i++)
4262                         i40e_irq_dynamic_enable(vsi, i);
4263         } else {
4264                 i40e_irq_dynamic_enable_icr0(pf);
4265         }
4266
4267         i40e_flush(&pf->hw);
4268         return 0;
4269 }
4270
4271 /**
4272  * i40e_free_misc_vector - Free the vector that handles non-queue events
4273  * @pf: board private structure
4274  **/
4275 static void i40e_free_misc_vector(struct i40e_pf *pf)
4276 {
4277         /* Disable ICR 0 */
4278         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4279         i40e_flush(&pf->hw);
4280
4281         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
4282                 free_irq(pf->msix_entries[0].vector, pf);
4283                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4284         }
4285 }
4286
4287 /**
4288  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4289  * @irq: interrupt number
4290  * @data: pointer to a q_vector
4291  *
4292  * This is the handler used for all MSI/Legacy interrupts, and deals
4293  * with both queue and non-queue interrupts.  This is also used in
4294  * MSIX mode to handle the non-queue interrupts.
4295  **/
4296 static irqreturn_t i40e_intr(int irq, void *data)
4297 {
4298         struct i40e_pf *pf = (struct i40e_pf *)data;
4299         struct i40e_hw *hw = &pf->hw;
4300         irqreturn_t ret = IRQ_NONE;
4301         u32 icr0, icr0_remaining;
4302         u32 val, ena_mask;
4303
4304         icr0 = rd32(hw, I40E_PFINT_ICR0);
4305         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4306
4307         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4308         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4309                 goto enable_intr;
4310
4311         /* if interrupt but no bits showing, must be SWINT */
4312         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4313             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4314                 pf->sw_int_count++;
4315
4316         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
4317             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4318                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4319                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4320                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4321         }
4322
4323         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4324         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4325                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4326                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4327
4328                 /* We do not have a way to disarm Queue causes while leaving
4329                  * interrupt enabled for all other causes, ideally
4330                  * interrupt should be disabled while we are in NAPI but
4331                  * this is not a performance path and napi_schedule()
4332                  * can deal with rescheduling.
4333                  */
4334                 if (!test_bit(__I40E_DOWN, pf->state))
4335                         napi_schedule_irqoff(&q_vector->napi);
4336         }
4337
4338         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4339                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4340                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4341                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4342         }
4343
4344         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4345                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4346                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4347         }
4348
4349         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4350                 /* disable any further VFLR event notifications */
4351                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4352                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4353
4354                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4355                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4356                 } else {
4357                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4358                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4359                 }
4360         }
4361
4362         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4363                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4364                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4365                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4366                 val = rd32(hw, I40E_GLGEN_RSTAT);
4367                 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);
4368                 if (val == I40E_RESET_CORER) {
4369                         pf->corer_count++;
4370                 } else if (val == I40E_RESET_GLOBR) {
4371                         pf->globr_count++;
4372                 } else if (val == I40E_RESET_EMPR) {
4373                         pf->empr_count++;
4374                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4375                 }
4376         }
4377
4378         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4379                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4380                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4381                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4382                          rd32(hw, I40E_PFHMC_ERRORINFO),
4383                          rd32(hw, I40E_PFHMC_ERRORDATA));
4384         }
4385
4386         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4387                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4388
4389                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4390                         schedule_work(&pf->ptp_extts0_work);
4391
4392                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4393                         i40e_ptp_tx_hwtstamp(pf);
4394
4395                 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4396         }
4397
4398         /* If a critical error is pending we have no choice but to reset the
4399          * device.
4400          * Report and mask out any remaining unexpected interrupts.
4401          */
4402         icr0_remaining = icr0 & ena_mask;
4403         if (icr0_remaining) {
4404                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4405                          icr0_remaining);
4406                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4407                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4408                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4409                         dev_info(&pf->pdev->dev, "device will be reset\n");
4410                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4411                         i40e_service_event_schedule(pf);
4412                 }
4413                 ena_mask &= ~icr0_remaining;
4414         }
4415         ret = IRQ_HANDLED;
4416
4417 enable_intr:
4418         /* re-enable interrupt causes */
4419         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4420         if (!test_bit(__I40E_DOWN, pf->state) ||
4421             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4422                 i40e_service_event_schedule(pf);
4423                 i40e_irq_dynamic_enable_icr0(pf);
4424         }
4425
4426         return ret;
4427 }
4428
4429 /**
4430  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4431  * @tx_ring:  tx ring to clean
4432  * @budget:   how many cleans we're allowed
4433  *
4434  * Returns true if there's any budget left (e.g. the clean is finished)
4435  **/
4436 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4437 {
4438         struct i40e_vsi *vsi = tx_ring->vsi;
4439         u16 i = tx_ring->next_to_clean;
4440         struct i40e_tx_buffer *tx_buf;
4441         struct i40e_tx_desc *tx_desc;
4442
4443         tx_buf = &tx_ring->tx_bi[i];
4444         tx_desc = I40E_TX_DESC(tx_ring, i);
4445         i -= tx_ring->count;
4446
4447         do {
4448                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4449
4450                 /* if next_to_watch is not set then there is no work pending */
4451                 if (!eop_desc)
4452                         break;
4453
4454                 /* prevent any other reads prior to eop_desc */
4455                 smp_rmb();
4456
4457                 /* if the descriptor isn't done, no work yet to do */
4458                 if (!(eop_desc->cmd_type_offset_bsz &
4459                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4460                         break;
4461
4462                 /* clear next_to_watch to prevent false hangs */
4463                 tx_buf->next_to_watch = NULL;
4464
4465                 tx_desc->buffer_addr = 0;
4466                 tx_desc->cmd_type_offset_bsz = 0;
4467                 /* move past filter desc */
4468                 tx_buf++;
4469                 tx_desc++;
4470                 i++;
4471                 if (unlikely(!i)) {
4472                         i -= tx_ring->count;
4473                         tx_buf = tx_ring->tx_bi;
4474                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4475                 }
4476                 /* unmap skb header data */
4477                 dma_unmap_single(tx_ring->dev,
4478                                  dma_unmap_addr(tx_buf, dma),
4479                                  dma_unmap_len(tx_buf, len),
4480                                  DMA_TO_DEVICE);
4481                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4482                         kfree(tx_buf->raw_buf);
4483
4484                 tx_buf->raw_buf = NULL;
4485                 tx_buf->tx_flags = 0;
4486                 tx_buf->next_to_watch = NULL;
4487                 dma_unmap_len_set(tx_buf, len, 0);
4488                 tx_desc->buffer_addr = 0;
4489                 tx_desc->cmd_type_offset_bsz = 0;
4490
4491                 /* move us past the eop_desc for start of next FD desc */
4492                 tx_buf++;
4493                 tx_desc++;
4494                 i++;
4495                 if (unlikely(!i)) {
4496                         i -= tx_ring->count;
4497                         tx_buf = tx_ring->tx_bi;
4498                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4499                 }
4500
4501                 /* update budget accounting */
4502                 budget--;
4503         } while (likely(budget));
4504
4505         i += tx_ring->count;
4506         tx_ring->next_to_clean = i;
4507
4508         if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
4509                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4510
4511         return budget > 0;
4512 }
4513
4514 /**
4515  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4516  * @irq: interrupt number
4517  * @data: pointer to a q_vector
4518  **/
4519 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4520 {
4521         struct i40e_q_vector *q_vector = data;
4522         struct i40e_vsi *vsi;
4523
4524         if (!q_vector->tx.ring)
4525                 return IRQ_HANDLED;
4526
4527         vsi = q_vector->tx.ring->vsi;
4528         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4529
4530         return IRQ_HANDLED;
4531 }
4532
4533 /**
4534  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4535  * @vsi: the VSI being configured
4536  * @v_idx: vector index
4537  * @qp_idx: queue pair index
4538  **/
4539 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4540 {
4541         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4542         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4543         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4544
4545         tx_ring->q_vector = q_vector;
4546         tx_ring->next = q_vector->tx.ring;
4547         q_vector->tx.ring = tx_ring;
4548         q_vector->tx.count++;
4549
4550         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4551         if (i40e_enabled_xdp_vsi(vsi)) {
4552                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4553
4554                 xdp_ring->q_vector = q_vector;
4555                 xdp_ring->next = q_vector->tx.ring;
4556                 q_vector->tx.ring = xdp_ring;
4557                 q_vector->tx.count++;
4558         }
4559
4560         rx_ring->q_vector = q_vector;
4561         rx_ring->next = q_vector->rx.ring;
4562         q_vector->rx.ring = rx_ring;
4563         q_vector->rx.count++;
4564 }
4565
4566 /**
4567  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4568  * @vsi: the VSI being configured
4569  *
4570  * This function maps descriptor rings to the queue-specific vectors
4571  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4572  * one vector per queue pair, but on a constrained vector budget, we
4573  * group the queue pairs as "efficiently" as possible.
4574  **/
4575 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4576 {
4577         int qp_remaining = vsi->num_queue_pairs;
4578         int q_vectors = vsi->num_q_vectors;
4579         int num_ringpairs;
4580         int v_start = 0;
4581         int qp_idx = 0;
4582
4583         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4584          * group them so there are multiple queues per vector.
4585          * It is also important to go through all the vectors available to be
4586          * sure that if we don't use all the vectors, that the remaining vectors
4587          * are cleared. This is especially important when decreasing the
4588          * number of queues in use.
4589          */
4590         for (; v_start < q_vectors; v_start++) {
4591                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4592
4593                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4594
4595                 q_vector->num_ringpairs = num_ringpairs;
4596                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4597
4598                 q_vector->rx.count = 0;
4599                 q_vector->tx.count = 0;
4600                 q_vector->rx.ring = NULL;
4601                 q_vector->tx.ring = NULL;
4602
4603                 while (num_ringpairs--) {
4604                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4605                         qp_idx++;
4606                         qp_remaining--;
4607                 }
4608         }
4609 }
4610
4611 /**
4612  * i40e_vsi_request_irq - Request IRQ from the OS
4613  * @vsi: the VSI being configured
4614  * @basename: name for the vector
4615  **/
4616 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4617 {
4618         struct i40e_pf *pf = vsi->back;
4619         int err;
4620
4621         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
4622                 err = i40e_vsi_request_irq_msix(vsi, basename);
4623         else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
4624                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4625                                   pf->int_name, pf);
4626         else
4627                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4628                                   pf->int_name, pf);
4629
4630         if (err)
4631                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4632
4633         return err;
4634 }
4635
4636 #ifdef CONFIG_NET_POLL_CONTROLLER
4637 /**
4638  * i40e_netpoll - A Polling 'interrupt' handler
4639  * @netdev: network interface device structure
4640  *
4641  * This is used by netconsole to send skbs without having to re-enable
4642  * interrupts.  It's not called while the normal interrupt routine is executing.
4643  **/
4644 static void i40e_netpoll(struct net_device *netdev)
4645 {
4646         struct i40e_netdev_priv *np = netdev_priv(netdev);
4647         struct i40e_vsi *vsi = np->vsi;
4648         struct i40e_pf *pf = vsi->back;
4649         int i;
4650
4651         /* if interface is down do nothing */
4652         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4653                 return;
4654
4655         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4656                 for (i = 0; i < vsi->num_q_vectors; i++)
4657                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4658         } else {
4659                 i40e_intr(pf->pdev->irq, netdev);
4660         }
4661 }
4662 #endif
4663
4664 #define I40E_QTX_ENA_WAIT_COUNT 50
4665
4666 /**
4667  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4668  * @pf: the PF being configured
4669  * @pf_q: the PF queue
4670  * @enable: enable or disable state of the queue
4671  *
4672  * This routine will wait for the given Tx queue of the PF to reach the
4673  * enabled or disabled state.
4674  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4675  * multiple retries; else will return 0 in case of success.
4676  **/
4677 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4678 {
4679         int i;
4680         u32 tx_reg;
4681
4682         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4683                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4684                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4685                         break;
4686
4687                 usleep_range(10, 20);
4688         }
4689         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4690                 return -ETIMEDOUT;
4691
4692         return 0;
4693 }
4694
4695 /**
4696  * i40e_control_tx_q - Start or stop a particular Tx queue
4697  * @pf: the PF structure
4698  * @pf_q: the PF queue to configure
4699  * @enable: start or stop the queue
4700  *
4701  * This function enables or disables a single queue. Note that any delay
4702  * required after the operation is expected to be handled by the caller of
4703  * this function.
4704  **/
4705 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4706 {
4707         struct i40e_hw *hw = &pf->hw;
4708         u32 tx_reg;
4709         int i;
4710
4711         /* warn the TX unit of coming changes */
4712         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4713         if (!enable)
4714                 usleep_range(10, 20);
4715
4716         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4717                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4718                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4719                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4720                         break;
4721                 usleep_range(1000, 2000);
4722         }
4723
4724         /* Skip if the queue is already in the requested state */
4725         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4726                 return;
4727
4728         /* turn on/off the queue */
4729         if (enable) {
4730                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4731                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4732         } else {
4733                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4734         }
4735
4736         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4737 }
4738
4739 /**
4740  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4741  * @seid: VSI SEID
4742  * @pf: the PF structure
4743  * @pf_q: the PF queue to configure
4744  * @is_xdp: true if the queue is used for XDP
4745  * @enable: start or stop the queue
4746  **/
4747 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4748                            bool is_xdp, bool enable)
4749 {
4750         int ret;
4751
4752         i40e_control_tx_q(pf, pf_q, enable);
4753
4754         /* wait for the change to finish */
4755         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4756         if (ret) {
4757                 dev_info(&pf->pdev->dev,
4758                          "VSI seid %d %sTx ring %d %sable timeout\n",
4759                          seid, (is_xdp ? "XDP " : ""), pf_q,
4760                          (enable ? "en" : "dis"));
4761         }
4762
4763         return ret;
4764 }
4765
4766 /**
4767  * i40e_vsi_enable_tx - Start a VSI's rings
4768  * @vsi: the VSI being configured
4769  **/
4770 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4771 {
4772         struct i40e_pf *pf = vsi->back;
4773         int i, pf_q, ret = 0;
4774
4775         pf_q = vsi->base_queue;
4776         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4777                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4778                                              pf_q,
4779                                              false /*is xdp*/, true);
4780                 if (ret)
4781                         break;
4782
4783                 if (!i40e_enabled_xdp_vsi(vsi))
4784                         continue;
4785
4786                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4787                                              pf_q + vsi->alloc_queue_pairs,
4788                                              true /*is xdp*/, true);
4789                 if (ret)
4790                         break;
4791         }
4792         return ret;
4793 }
4794
4795 /**
4796  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4797  * @pf: the PF being configured
4798  * @pf_q: the PF queue
4799  * @enable: enable or disable state of the queue
4800  *
4801  * This routine will wait for the given Rx queue of the PF to reach the
4802  * enabled or disabled state.
4803  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4804  * multiple retries; else will return 0 in case of success.
4805  **/
4806 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4807 {
4808         int i;
4809         u32 rx_reg;
4810
4811         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4812                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4813                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4814                         break;
4815
4816                 usleep_range(10, 20);
4817         }
4818         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4819                 return -ETIMEDOUT;
4820
4821         return 0;
4822 }
4823
4824 /**
4825  * i40e_control_rx_q - Start or stop a particular Rx queue
4826  * @pf: the PF structure
4827  * @pf_q: the PF queue to configure
4828  * @enable: start or stop the queue
4829  *
4830  * This function enables or disables a single queue. Note that
4831  * any delay required after the operation is expected to be
4832  * handled by the caller of this function.
4833  **/
4834 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4835 {
4836         struct i40e_hw *hw = &pf->hw;
4837         u32 rx_reg;
4838         int i;
4839
4840         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4841                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4842                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4843                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4844                         break;
4845                 usleep_range(1000, 2000);
4846         }
4847
4848         /* Skip if the queue is already in the requested state */
4849         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4850                 return;
4851
4852         /* turn on/off the queue */
4853         if (enable)
4854                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4855         else
4856                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4857
4858         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4859 }
4860
4861 /**
4862  * i40e_control_wait_rx_q
4863  * @pf: the PF structure
4864  * @pf_q: queue being configured
4865  * @enable: start or stop the rings
4866  *
4867  * This function enables or disables a single queue along with waiting
4868  * for the change to finish. The caller of this function should handle
4869  * the delays needed in the case of disabling queues.
4870  **/
4871 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4872 {
4873         int ret = 0;
4874
4875         i40e_control_rx_q(pf, pf_q, enable);
4876
4877         /* wait for the change to finish */
4878         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4879         if (ret)
4880                 return ret;
4881
4882         return ret;
4883 }
4884
4885 /**
4886  * i40e_vsi_enable_rx - Start a VSI's rings
4887  * @vsi: the VSI being configured
4888  **/
4889 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4890 {
4891         struct i40e_pf *pf = vsi->back;
4892         int i, pf_q, ret = 0;
4893
4894         pf_q = vsi->base_queue;
4895         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4896                 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4897                 if (ret) {
4898                         dev_info(&pf->pdev->dev,
4899                                  "VSI seid %d Rx ring %d enable timeout\n",
4900                                  vsi->seid, pf_q);
4901                         break;
4902                 }
4903         }
4904
4905         return ret;
4906 }
4907
4908 /**
4909  * i40e_vsi_start_rings - Start a VSI's rings
4910  * @vsi: the VSI being configured
4911  **/
4912 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4913 {
4914         int ret = 0;
4915
4916         /* do rx first for enable and last for disable */
4917         ret = i40e_vsi_enable_rx(vsi);
4918         if (ret)
4919                 return ret;
4920         ret = i40e_vsi_enable_tx(vsi);
4921
4922         return ret;
4923 }
4924
4925 #define I40E_DISABLE_TX_GAP_MSEC        50
4926
4927 /**
4928  * i40e_vsi_stop_rings - Stop a VSI's rings
4929  * @vsi: the VSI being configured
4930  **/
4931 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4932 {
4933         struct i40e_pf *pf = vsi->back;
4934         u32 pf_q, tx_q_end, rx_q_end;
4935
4936         /* When port TX is suspended, don't wait */
4937         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4938                 return i40e_vsi_stop_rings_no_wait(vsi);
4939
4940         tx_q_end = vsi->base_queue +
4941                 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
4942         for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4943                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);
4944
4945         rx_q_end = vsi->base_queue + vsi->num_queue_pairs;
4946         for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)
4947                 i40e_control_rx_q(pf, pf_q, false);
4948
4949         msleep(I40E_DISABLE_TX_GAP_MSEC);
4950         for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4951                 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4952
4953         i40e_vsi_wait_queues_disabled(vsi);
4954 }
4955
4956 /**
4957  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4958  * @vsi: the VSI being shutdown
4959  *
4960  * This function stops all the rings for a VSI but does not delay to verify
4961  * that rings have been disabled. It is expected that the caller is shutting
4962  * down multiple VSIs at once and will delay together for all the VSIs after
4963  * initiating the shutdown. This is particularly useful for shutting down lots
4964  * of VFs together. Otherwise, a large delay can be incurred while configuring
4965  * each VSI in serial.
4966  **/
4967 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4968 {
4969         struct i40e_pf *pf = vsi->back;
4970         int i, pf_q;
4971
4972         pf_q = vsi->base_queue;
4973         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4974                 i40e_control_tx_q(pf, pf_q, false);
4975                 i40e_control_rx_q(pf, pf_q, false);
4976         }
4977 }
4978
4979 /**
4980  * i40e_vsi_free_irq - Free the irq association with the OS
4981  * @vsi: the VSI being configured
4982  **/
4983 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4984 {
4985         struct i40e_pf *pf = vsi->back;
4986         struct i40e_hw *hw = &pf->hw;
4987         int base = vsi->base_vector;
4988         u32 val, qp;
4989         int i;
4990
4991         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4992                 if (!vsi->q_vectors)
4993                         return;
4994
4995                 if (!vsi->irqs_ready)
4996                         return;
4997
4998                 vsi->irqs_ready = false;
4999                 for (i = 0; i < vsi->num_q_vectors; i++) {
5000                         int irq_num;
5001                         u16 vector;
5002
5003                         vector = i + base;
5004                         irq_num = pf->msix_entries[vector].vector;
5005
5006                         /* free only the irqs that were actually requested */
5007                         if (!vsi->q_vectors[i] ||
5008                             !vsi->q_vectors[i]->num_ringpairs)
5009                                 continue;
5010
5011                         /* clear the affinity notifier in the IRQ descriptor */
5012                         irq_set_affinity_notifier(irq_num, NULL);
5013                         /* remove our suggested affinity mask for this IRQ */
5014                         irq_update_affinity_hint(irq_num, NULL);
5015                         free_irq(irq_num, vsi->q_vectors[i]);
5016
5017                         /* Tear down the interrupt queue link list
5018                          *
5019                          * We know that they come in pairs and always
5020                          * the Rx first, then the Tx.  To clear the
5021                          * link list, stick the EOL value into the
5022                          * next_q field of the registers.
5023                          */
5024                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
5025                         qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,
5026                                        val);
5027                         val |= I40E_QUEUE_END_OF_LIST
5028                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5029                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
5030
5031                         while (qp != I40E_QUEUE_END_OF_LIST) {
5032                                 u32 next;
5033
5034                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
5035
5036                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5037                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5038                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5039                                          I40E_QINT_RQCTL_INTEVENT_MASK);
5040
5041                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5042                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5043
5044                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
5045
5046                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
5047
5048                                 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,
5049                                                  val);
5050
5051                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5052                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5053                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5054                                          I40E_QINT_TQCTL_INTEVENT_MASK);
5055
5056                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5057                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5058
5059                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
5060                                 qp = next;
5061                         }
5062                 }
5063         } else {
5064                 free_irq(pf->pdev->irq, pf);
5065
5066                 val = rd32(hw, I40E_PFINT_LNKLST0);
5067                 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);
5068                 val |= I40E_QUEUE_END_OF_LIST
5069                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5070                 wr32(hw, I40E_PFINT_LNKLST0, val);
5071
5072                 val = rd32(hw, I40E_QINT_RQCTL(qp));
5073                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5074                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5075                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5076                          I40E_QINT_RQCTL_INTEVENT_MASK);
5077
5078                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5079                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5080
5081                 wr32(hw, I40E_QINT_RQCTL(qp), val);
5082
5083                 val = rd32(hw, I40E_QINT_TQCTL(qp));
5084
5085                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5086                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5087                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5088                          I40E_QINT_TQCTL_INTEVENT_MASK);
5089
5090                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5091                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5092
5093                 wr32(hw, I40E_QINT_TQCTL(qp), val);
5094         }
5095 }
5096
5097 /**
5098  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5099  * @vsi: the VSI being configured
5100  * @v_idx: Index of vector to be freed
5101  *
5102  * This function frees the memory allocated to the q_vector.  In addition if
5103  * NAPI is enabled it will delete any references to the NAPI struct prior
5104  * to freeing the q_vector.
5105  **/
5106 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5107 {
5108         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5109         struct i40e_ring *ring;
5110
5111         if (!q_vector)
5112                 return;
5113
5114         /* disassociate q_vector from rings */
5115         i40e_for_each_ring(ring, q_vector->tx)
5116                 ring->q_vector = NULL;
5117
5118         i40e_for_each_ring(ring, q_vector->rx)
5119                 ring->q_vector = NULL;
5120
5121         /* only VSI w/ an associated netdev is set up w/ NAPI */
5122         if (vsi->netdev)
5123                 netif_napi_del(&q_vector->napi);
5124
5125         vsi->q_vectors[v_idx] = NULL;
5126
5127         kfree_rcu(q_vector, rcu);
5128 }
5129
5130 /**
5131  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5132  * @vsi: the VSI being un-configured
5133  *
5134  * This frees the memory allocated to the q_vectors and
5135  * deletes references to the NAPI struct.
5136  **/
5137 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5138 {
5139         int v_idx;
5140
5141         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5142                 i40e_free_q_vector(vsi, v_idx);
5143 }
5144
5145 /**
5146  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5147  * @pf: board private structure
5148  **/
5149 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5150 {
5151         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5152         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
5153                 pci_disable_msix(pf->pdev);
5154                 kfree(pf->msix_entries);
5155                 pf->msix_entries = NULL;
5156                 kfree(pf->irq_pile);
5157                 pf->irq_pile = NULL;
5158         } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
5159                 pci_disable_msi(pf->pdev);
5160         }
5161         clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
5162         clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
5163 }
5164
5165 /**
5166  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5167  * @pf: board private structure
5168  *
5169  * We go through and clear interrupt specific resources and reset the structure
5170  * to pre-load conditions
5171  **/
5172 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5173 {
5174         struct i40e_vsi *vsi;
5175         int i;
5176
5177         if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5178                 i40e_free_misc_vector(pf);
5179
5180         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5181                       I40E_IWARP_IRQ_PILE_ID);
5182
5183         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5184
5185         i40e_pf_for_each_vsi(pf, i, vsi)
5186                 i40e_vsi_free_q_vectors(vsi);
5187
5188         i40e_reset_interrupt_capability(pf);
5189 }
5190
5191 /**
5192  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5193  * @vsi: the VSI being configured
5194  **/
5195 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5196 {
5197         int q_idx;
5198
5199         if (!vsi->netdev)
5200                 return;
5201
5202         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5203                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5204
5205                 if (q_vector->rx.ring || q_vector->tx.ring)
5206                         napi_enable(&q_vector->napi);
5207         }
5208 }
5209
5210 /**
5211  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5212  * @vsi: the VSI being configured
5213  **/
5214 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5215 {
5216         int q_idx;
5217
5218         if (!vsi->netdev)
5219                 return;
5220
5221         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5222                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5223
5224                 if (q_vector->rx.ring || q_vector->tx.ring)
5225                         napi_disable(&q_vector->napi);
5226         }
5227 }
5228
5229 /**
5230  * i40e_vsi_close - Shut down a VSI
5231  * @vsi: the vsi to be quelled
5232  **/
5233 static void i40e_vsi_close(struct i40e_vsi *vsi)
5234 {
5235         struct i40e_pf *pf = vsi->back;
5236         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5237                 i40e_down(vsi);
5238         i40e_vsi_free_irq(vsi);
5239         i40e_vsi_free_tx_resources(vsi);
5240         i40e_vsi_free_rx_resources(vsi);
5241         vsi->current_netdev_flags = 0;
5242         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5243         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5244                 set_bit(__I40E_CLIENT_RESET, pf->state);
5245 }
5246
5247 /**
5248  * i40e_quiesce_vsi - Pause a given VSI
5249  * @vsi: the VSI being paused
5250  **/
5251 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5252 {
5253         if (test_bit(__I40E_VSI_DOWN, vsi->state))
5254                 return;
5255
5256         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5257         if (vsi->netdev && netif_running(vsi->netdev))
5258                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5259         else
5260                 i40e_vsi_close(vsi);
5261 }
5262
5263 /**
5264  * i40e_unquiesce_vsi - Resume a given VSI
5265  * @vsi: the VSI being resumed
5266  **/
5267 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5268 {
5269         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5270                 return;
5271
5272         if (vsi->netdev && netif_running(vsi->netdev))
5273                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5274         else
5275                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
5276 }
5277
5278 /**
5279  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5280  * @pf: the PF
5281  **/
5282 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5283 {
5284         struct i40e_vsi *vsi;
5285         int v;
5286
5287         i40e_pf_for_each_vsi(pf, v, vsi)
5288                 i40e_quiesce_vsi(vsi);
5289 }
5290
5291 /**
5292  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5293  * @pf: the PF
5294  **/
5295 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5296 {
5297         struct i40e_vsi *vsi;
5298         int v;
5299
5300         i40e_pf_for_each_vsi(pf, v, vsi)
5301                 i40e_unquiesce_vsi(vsi);
5302 }
5303
5304 /**
5305  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5306  * @vsi: the VSI being configured
5307  *
5308  * Wait until all queues on a given VSI have been disabled.
5309  **/
5310 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5311 {
5312         struct i40e_pf *pf = vsi->back;
5313         int i, pf_q, ret;
5314
5315         pf_q = vsi->base_queue;
5316         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5317                 /* Check and wait for the Tx queue */
5318                 ret = i40e_pf_txq_wait(pf, pf_q, false);
5319                 if (ret) {
5320                         dev_info(&pf->pdev->dev,
5321                                  "VSI seid %d Tx ring %d disable timeout\n",
5322                                  vsi->seid, pf_q);
5323                         return ret;
5324                 }
5325
5326                 if (!i40e_enabled_xdp_vsi(vsi))
5327                         goto wait_rx;
5328
5329                 /* Check and wait for the XDP Tx queue */
5330                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5331                                        false);
5332                 if (ret) {
5333                         dev_info(&pf->pdev->dev,
5334                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
5335                                  vsi->seid, pf_q);
5336                         return ret;
5337                 }
5338 wait_rx:
5339                 /* Check and wait for the Rx queue */
5340                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5341                 if (ret) {
5342                         dev_info(&pf->pdev->dev,
5343                                  "VSI seid %d Rx ring %d disable timeout\n",
5344                                  vsi->seid, pf_q);
5345                         return ret;
5346                 }
5347         }
5348
5349         return 0;
5350 }
5351
5352 #ifdef CONFIG_I40E_DCB
5353 /**
5354  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5355  * @pf: the PF
5356  *
5357  * This function waits for the queues to be in disabled state for all the
5358  * VSIs that are managed by this PF.
5359  **/
5360 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5361 {
5362         struct i40e_vsi *vsi;
5363         int v, ret = 0;
5364
5365         i40e_pf_for_each_vsi(pf, v, vsi) {
5366                 ret = i40e_vsi_wait_queues_disabled(vsi);
5367                 if (ret)
5368                         break;
5369         }
5370
5371         return ret;
5372 }
5373
5374 #endif
5375
5376 /**
5377  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5378  * @pf: pointer to PF
5379  *
5380  * Get TC map for ISCSI PF type that will include iSCSI TC
5381  * and LAN TC.
5382  **/
5383 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5384 {
5385         struct i40e_dcb_app_priority_table app;
5386         struct i40e_hw *hw = &pf->hw;
5387         u8 enabled_tc = 1; /* TC0 is always enabled */
5388         u8 tc, i;
5389         /* Get the iSCSI APP TLV */
5390         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5391
5392         for (i = 0; i < dcbcfg->numapps; i++) {
5393                 app = dcbcfg->app[i];
5394                 if (app.selector == I40E_APP_SEL_TCPIP &&
5395                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5396                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5397                         enabled_tc |= BIT(tc);
5398                         break;
5399                 }
5400         }
5401
5402         return enabled_tc;
5403 }
5404
5405 /**
5406  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5407  * @dcbcfg: the corresponding DCBx configuration structure
5408  *
5409  * Return the number of TCs from given DCBx configuration
5410  **/
5411 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5412 {
5413         int i, tc_unused = 0;
5414         u8 num_tc = 0;
5415         u8 ret = 0;
5416
5417         /* Scan the ETS Config Priority Table to find
5418          * traffic class enabled for a given priority
5419          * and create a bitmask of enabled TCs
5420          */
5421         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5422                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5423
5424         /* Now scan the bitmask to check for
5425          * contiguous TCs starting with TC0
5426          */
5427         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5428                 if (num_tc & BIT(i)) {
5429                         if (!tc_unused) {
5430                                 ret++;
5431                         } else {
5432                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5433                                 return 1;
5434                         }
5435                 } else {
5436                         tc_unused = 1;
5437                 }
5438         }
5439
5440         /* There is always at least TC0 */
5441         if (!ret)
5442                 ret = 1;
5443
5444         return ret;
5445 }
5446
5447 /**
5448  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5449  * @dcbcfg: the corresponding DCBx configuration structure
5450  *
5451  * Query the current DCB configuration and return the number of
5452  * traffic classes enabled from the given DCBX config
5453  **/
5454 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5455 {
5456         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5457         u8 enabled_tc = 1;
5458         u8 i;
5459
5460         for (i = 0; i < num_tc; i++)
5461                 enabled_tc |= BIT(i);
5462
5463         return enabled_tc;
5464 }
5465
5466 /**
5467  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5468  * @pf: PF being queried
5469  *
5470  * Query the current MQPRIO configuration and return the number of
5471  * traffic classes enabled.
5472  **/
5473 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5474 {
5475         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5476         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5477         u8 enabled_tc = 1, i;
5478
5479         for (i = 1; i < num_tc; i++)
5480                 enabled_tc |= BIT(i);
5481         return enabled_tc;
5482 }
5483
5484 /**
5485  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5486  * @pf: PF being queried
5487  *
5488  * Return number of traffic classes enabled for the given PF
5489  **/
5490 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5491 {
5492         struct i40e_hw *hw = &pf->hw;
5493         u8 i, enabled_tc = 1;
5494         u8 num_tc = 0;
5495         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5496
5497         if (i40e_is_tc_mqprio_enabled(pf))
5498                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5499
5500         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5501         if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5502                 return 1;
5503
5504         /* SFP mode will be enabled for all TCs on port */
5505         if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5506                 return i40e_dcb_get_num_tc(dcbcfg);
5507
5508         /* MFP mode return count of enabled TCs for this PF */
5509         if (pf->hw.func_caps.iscsi)
5510                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5511         else
5512                 return 1; /* Only TC0 */
5513
5514         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5515                 if (enabled_tc & BIT(i))
5516                         num_tc++;
5517         }
5518         return num_tc;
5519 }
5520
5521 /**
5522  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5523  * @pf: PF being queried
5524  *
5525  * Return a bitmap for enabled traffic classes for this PF.
5526  **/
5527 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5528 {
5529         if (i40e_is_tc_mqprio_enabled(pf))
5530                 return i40e_mqprio_get_enabled_tc(pf);
5531
5532         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5533          * default TC
5534          */
5535         if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5536                 return I40E_DEFAULT_TRAFFIC_CLASS;
5537
5538         /* SFP mode we want PF to be enabled for all TCs */
5539         if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5540                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5541
5542         /* MFP enabled and iSCSI PF type */
5543         if (pf->hw.func_caps.iscsi)
5544                 return i40e_get_iscsi_tc_map(pf);
5545         else
5546                 return I40E_DEFAULT_TRAFFIC_CLASS;
5547 }
5548
5549 /**
5550  * i40e_vsi_get_bw_info - Query VSI BW Information
5551  * @vsi: the VSI being queried
5552  *
5553  * Returns 0 on success, negative value on failure
5554  **/
5555 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5556 {
5557         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5558         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5559         struct i40e_pf *pf = vsi->back;
5560         struct i40e_hw *hw = &pf->hw;
5561         u32 tc_bw_max;
5562         int ret;
5563         int i;
5564
5565         /* Get the VSI level BW configuration */
5566         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5567         if (ret) {
5568                 dev_info(&pf->pdev->dev,
5569                          "couldn't get PF vsi bw config, err %pe aq_err %s\n",
5570                          ERR_PTR(ret),
5571                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5572                 return -EINVAL;
5573         }
5574
5575         /* Get the VSI level BW configuration per TC */
5576         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5577                                                NULL);
5578         if (ret) {
5579                 dev_info(&pf->pdev->dev,
5580                          "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5581                          ERR_PTR(ret),
5582                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5583                 return -EINVAL;
5584         }
5585
5586         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5587                 dev_info(&pf->pdev->dev,
5588                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5589                          bw_config.tc_valid_bits,
5590                          bw_ets_config.tc_valid_bits);
5591                 /* Still continuing */
5592         }
5593
5594         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5595         vsi->bw_max_quanta = bw_config.max_bw;
5596         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5597                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5598         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5599                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5600                 vsi->bw_ets_limit_credits[i] =
5601                                         le16_to_cpu(bw_ets_config.credits[i]);
5602                 /* 3 bits out of 4 for each TC */
5603                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5604         }
5605
5606         return 0;
5607 }
5608
5609 /**
5610  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5611  * @vsi: the VSI being configured
5612  * @enabled_tc: TC bitmap
5613  * @bw_share: BW shared credits per TC
5614  *
5615  * Returns 0 on success, negative value on failure
5616  **/
5617 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5618                                        u8 *bw_share)
5619 {
5620         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5621         struct i40e_pf *pf = vsi->back;
5622         int ret;
5623         int i;
5624
5625         /* There is no need to reset BW when mqprio mode is on.  */
5626         if (i40e_is_tc_mqprio_enabled(pf))
5627                 return 0;
5628         if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
5629                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5630                 if (ret)
5631                         dev_info(&pf->pdev->dev,
5632                                  "Failed to reset tx rate for vsi->seid %u\n",
5633                                  vsi->seid);
5634                 return ret;
5635         }
5636         memset(&bw_data, 0, sizeof(bw_data));
5637         bw_data.tc_valid_bits = enabled_tc;
5638         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5639                 bw_data.tc_bw_credits[i] = bw_share[i];
5640
5641         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5642         if (ret) {
5643                 dev_info(&pf->pdev->dev,
5644                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5645                          pf->hw.aq.asq_last_status);
5646                 return -EINVAL;
5647         }
5648
5649         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5650                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5651
5652         return 0;
5653 }
5654
5655 /**
5656  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5657  * @vsi: the VSI being configured
5658  * @enabled_tc: TC map to be enabled
5659  *
5660  **/
5661 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5662 {
5663         struct net_device *netdev = vsi->netdev;
5664         struct i40e_pf *pf = vsi->back;
5665         struct i40e_hw *hw = &pf->hw;
5666         u8 netdev_tc = 0;
5667         int i;
5668         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5669
5670         if (!netdev)
5671                 return;
5672
5673         if (!enabled_tc) {
5674                 netdev_reset_tc(netdev);
5675                 return;
5676         }
5677
5678         /* Set up actual enabled TCs on the VSI */
5679         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5680                 return;
5681
5682         /* set per TC queues for the VSI */
5683         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5684                 /* Only set TC queues for enabled tcs
5685                  *
5686                  * e.g. For a VSI that has TC0 and TC3 enabled the
5687                  * enabled_tc bitmap would be 0x00001001; the driver
5688                  * will set the numtc for netdev as 2 that will be
5689                  * referenced by the netdev layer as TC 0 and 1.
5690                  */
5691                 if (vsi->tc_config.enabled_tc & BIT(i))
5692                         netdev_set_tc_queue(netdev,
5693                                         vsi->tc_config.tc_info[i].netdev_tc,
5694                                         vsi->tc_config.tc_info[i].qcount,
5695                                         vsi->tc_config.tc_info[i].qoffset);
5696         }
5697
5698         if (i40e_is_tc_mqprio_enabled(pf))
5699                 return;
5700
5701         /* Assign UP2TC map for the VSI */
5702         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5703                 /* Get the actual TC# for the UP */
5704                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5705                 /* Get the mapped netdev TC# for the UP */
5706                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5707                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5708         }
5709 }
5710
5711 /**
5712  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5713  * @vsi: the VSI being configured
5714  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5715  **/
5716 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5717                                       struct i40e_vsi_context *ctxt)
5718 {
5719         /* copy just the sections touched not the entire info
5720          * since not all sections are valid as returned by
5721          * update vsi params
5722          */
5723         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5724         memcpy(&vsi->info.queue_mapping,
5725                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5726         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5727                sizeof(vsi->info.tc_mapping));
5728 }
5729
5730 /**
5731  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5732  * @vsi: the VSI being reconfigured
5733  * @vsi_offset: offset from main VF VSI
5734  */
5735 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5736 {
5737         struct i40e_vsi_context ctxt = {};
5738         struct i40e_pf *pf;
5739         struct i40e_hw *hw;
5740         int ret;
5741
5742         if (!vsi)
5743                 return -EINVAL;
5744         pf = vsi->back;
5745         hw = &pf->hw;
5746
5747         ctxt.seid = vsi->seid;
5748         ctxt.pf_num = hw->pf_id;
5749         ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5750         ctxt.uplink_seid = vsi->uplink_seid;
5751         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5752         ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5753         ctxt.info = vsi->info;
5754
5755         i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5756                                  false);
5757         if (vsi->reconfig_rss) {
5758                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5759                                       vsi->num_queue_pairs);
5760                 ret = i40e_vsi_config_rss(vsi);
5761                 if (ret) {
5762                         dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5763                         return ret;
5764                 }
5765                 vsi->reconfig_rss = false;
5766         }
5767
5768         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5769         if (ret) {
5770                 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5771                          ERR_PTR(ret),
5772                          i40e_aq_str(hw, hw->aq.asq_last_status));
5773                 return ret;
5774         }
5775         /* update the local VSI info with updated queue map */
5776         i40e_vsi_update_queue_map(vsi, &ctxt);
5777         vsi->info.valid_sections = 0;
5778
5779         return ret;
5780 }
5781
5782 /**
5783  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5784  * @vsi: VSI to be configured
5785  * @enabled_tc: TC bitmap
5786  *
5787  * This configures a particular VSI for TCs that are mapped to the
5788  * given TC bitmap. It uses default bandwidth share for TCs across
5789  * VSIs to configure TC for a particular VSI.
5790  *
5791  * NOTE:
5792  * It is expected that the VSI queues have been quisced before calling
5793  * this function.
5794  **/
5795 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5796 {
5797         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5798         struct i40e_pf *pf = vsi->back;
5799         struct i40e_hw *hw = &pf->hw;
5800         struct i40e_vsi_context ctxt;
5801         int ret = 0;
5802         int i;
5803
5804         /* Check if enabled_tc is same as existing or new TCs */
5805         if (vsi->tc_config.enabled_tc == enabled_tc &&
5806             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5807                 return ret;
5808
5809         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5810         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5811                 if (enabled_tc & BIT(i))
5812                         bw_share[i] = 1;
5813         }
5814
5815         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5816         if (ret) {
5817                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5818
5819                 dev_info(&pf->pdev->dev,
5820                          "Failed configuring TC map %d for VSI %d\n",
5821                          enabled_tc, vsi->seid);
5822                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5823                                                   &bw_config, NULL);
5824                 if (ret) {
5825                         dev_info(&pf->pdev->dev,
5826                                  "Failed querying vsi bw info, err %pe aq_err %s\n",
5827                                  ERR_PTR(ret),
5828                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5829                         goto out;
5830                 }
5831                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5832                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5833
5834                         if (!valid_tc)
5835                                 valid_tc = bw_config.tc_valid_bits;
5836                         /* Always enable TC0, no matter what */
5837                         valid_tc |= 1;
5838                         dev_info(&pf->pdev->dev,
5839                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5840                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5841                         enabled_tc = valid_tc;
5842                 }
5843
5844                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5845                 if (ret) {
5846                         dev_err(&pf->pdev->dev,
5847                                 "Unable to  configure TC map %d for VSI %d\n",
5848                                 enabled_tc, vsi->seid);
5849                         goto out;
5850                 }
5851         }
5852
5853         /* Update Queue Pairs Mapping for currently enabled UPs */
5854         ctxt.seid = vsi->seid;
5855         ctxt.pf_num = vsi->back->hw.pf_id;
5856         ctxt.vf_num = 0;
5857         ctxt.uplink_seid = vsi->uplink_seid;
5858         ctxt.info = vsi->info;
5859         if (i40e_is_tc_mqprio_enabled(pf)) {
5860                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5861                 if (ret)
5862                         goto out;
5863         } else {
5864                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5865         }
5866
5867         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5868          * queues changed.
5869          */
5870         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5871                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5872                                       vsi->num_queue_pairs);
5873                 ret = i40e_vsi_config_rss(vsi);
5874                 if (ret) {
5875                         dev_info(&vsi->back->pdev->dev,
5876                                  "Failed to reconfig rss for num_queues\n");
5877                         return ret;
5878                 }
5879                 vsi->reconfig_rss = false;
5880         }
5881         if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
5882                 ctxt.info.valid_sections |=
5883                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5884                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5885         }
5886
5887         /* Update the VSI after updating the VSI queue-mapping
5888          * information
5889          */
5890         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5891         if (ret) {
5892                 dev_info(&pf->pdev->dev,
5893                          "Update vsi tc config failed, err %pe aq_err %s\n",
5894                          ERR_PTR(ret),
5895                          i40e_aq_str(hw, hw->aq.asq_last_status));
5896                 goto out;
5897         }
5898         /* update the local VSI info with updated queue map */
5899         i40e_vsi_update_queue_map(vsi, &ctxt);
5900         vsi->info.valid_sections = 0;
5901
5902         /* Update current VSI BW information */
5903         ret = i40e_vsi_get_bw_info(vsi);
5904         if (ret) {
5905                 dev_info(&pf->pdev->dev,
5906                          "Failed updating vsi bw info, err %pe aq_err %s\n",
5907                          ERR_PTR(ret),
5908                          i40e_aq_str(hw, hw->aq.asq_last_status));
5909                 goto out;
5910         }
5911
5912         /* Update the netdev TC setup */
5913         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5914 out:
5915         return ret;
5916 }
5917
5918 /**
5919  * i40e_get_link_speed - Returns link speed for the interface
5920  * @vsi: VSI to be configured
5921  *
5922  **/
5923 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5924 {
5925         struct i40e_pf *pf = vsi->back;
5926
5927         switch (pf->hw.phy.link_info.link_speed) {
5928         case I40E_LINK_SPEED_40GB:
5929                 return 40000;
5930         case I40E_LINK_SPEED_25GB:
5931                 return 25000;
5932         case I40E_LINK_SPEED_20GB:
5933                 return 20000;
5934         case I40E_LINK_SPEED_10GB:
5935                 return 10000;
5936         case I40E_LINK_SPEED_1GB:
5937                 return 1000;
5938         default:
5939                 return -EINVAL;
5940         }
5941 }
5942
5943 /**
5944  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5945  * @vsi: Pointer to vsi structure
5946  * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5947  *
5948  * Helper function to convert units before send to set BW limit
5949  **/
5950 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5951 {
5952         if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5953                 dev_warn(&vsi->back->pdev->dev,
5954                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5955                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5956         } else {
5957                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5958         }
5959
5960         return max_tx_rate;
5961 }
5962
5963 /**
5964  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5965  * @vsi: VSI to be configured
5966  * @seid: seid of the channel/VSI
5967  * @max_tx_rate: max TX rate to be configured as BW limit
5968  *
5969  * Helper function to set BW limit for a given VSI
5970  **/
5971 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5972 {
5973         struct i40e_pf *pf = vsi->back;
5974         u64 credits = 0;
5975         int speed = 0;
5976         int ret = 0;
5977
5978         speed = i40e_get_link_speed(vsi);
5979         if (max_tx_rate > speed) {
5980                 dev_err(&pf->pdev->dev,
5981                         "Invalid max tx rate %llu specified for VSI seid %d.",
5982                         max_tx_rate, seid);
5983                 return -EINVAL;
5984         }
5985         if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5986                 dev_warn(&pf->pdev->dev,
5987                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5988                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5989         }
5990
5991         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5992         credits = max_tx_rate;
5993         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5994         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5995                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5996         if (ret)
5997                 dev_err(&pf->pdev->dev,
5998                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
5999                         max_tx_rate, seid, ERR_PTR(ret),
6000                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6001         return ret;
6002 }
6003
6004 /**
6005  * i40e_remove_queue_channels - Remove queue channels for the TCs
6006  * @vsi: VSI to be configured
6007  *
6008  * Remove queue channels for the TCs
6009  **/
6010 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
6011 {
6012         enum i40e_admin_queue_err last_aq_status;
6013         struct i40e_cloud_filter *cfilter;
6014         struct i40e_channel *ch, *ch_tmp;
6015         struct i40e_pf *pf = vsi->back;
6016         struct hlist_node *node;
6017         int ret, i;
6018
6019         /* Reset rss size that was stored when reconfiguring rss for
6020          * channel VSIs with non-power-of-2 queue count.
6021          */
6022         vsi->current_rss_size = 0;
6023
6024         /* perform cleanup for channels if they exist */
6025         if (list_empty(&vsi->ch_list))
6026                 return;
6027
6028         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6029                 struct i40e_vsi *p_vsi;
6030
6031                 list_del(&ch->list);
6032                 p_vsi = ch->parent_vsi;
6033                 if (!p_vsi || !ch->initialized) {
6034                         kfree(ch);
6035                         continue;
6036                 }
6037                 /* Reset queue contexts */
6038                 for (i = 0; i < ch->num_queue_pairs; i++) {
6039                         struct i40e_ring *tx_ring, *rx_ring;
6040                         u16 pf_q;
6041
6042                         pf_q = ch->base_queue + i;
6043                         tx_ring = vsi->tx_rings[pf_q];
6044                         tx_ring->ch = NULL;
6045
6046                         rx_ring = vsi->rx_rings[pf_q];
6047                         rx_ring->ch = NULL;
6048                 }
6049
6050                 /* Reset BW configured for this VSI via mqprio */
6051                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6052                 if (ret)
6053                         dev_info(&vsi->back->pdev->dev,
6054                                  "Failed to reset tx rate for ch->seid %u\n",
6055                                  ch->seid);
6056
6057                 /* delete cloud filters associated with this channel */
6058                 hlist_for_each_entry_safe(cfilter, node,
6059                                           &pf->cloud_filter_list, cloud_node) {
6060                         if (cfilter->seid != ch->seid)
6061                                 continue;
6062
6063                         hash_del(&cfilter->cloud_node);
6064                         if (cfilter->dst_port)
6065                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
6066                                                                         cfilter,
6067                                                                         false);
6068                         else
6069                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
6070                                                                 false);
6071                         last_aq_status = pf->hw.aq.asq_last_status;
6072                         if (ret)
6073                                 dev_info(&pf->pdev->dev,
6074                                          "Failed to delete cloud filter, err %pe aq_err %s\n",
6075                                          ERR_PTR(ret),
6076                                          i40e_aq_str(&pf->hw, last_aq_status));
6077                         kfree(cfilter);
6078                 }
6079
6080                 /* delete VSI from FW */
6081                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6082                                              NULL);
6083                 if (ret)
6084                         dev_err(&vsi->back->pdev->dev,
6085                                 "unable to remove channel (%d) for parent VSI(%d)\n",
6086                                 ch->seid, p_vsi->seid);
6087                 kfree(ch);
6088         }
6089         INIT_LIST_HEAD(&vsi->ch_list);
6090 }
6091
6092 /**
6093  * i40e_get_max_queues_for_channel
6094  * @vsi: ptr to VSI to which channels are associated with
6095  *
6096  * Helper function which returns max value among the queue counts set on the
6097  * channels/TCs created.
6098  **/
6099 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6100 {
6101         struct i40e_channel *ch, *ch_tmp;
6102         int max = 0;
6103
6104         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6105                 if (!ch->initialized)
6106                         continue;
6107                 if (ch->num_queue_pairs > max)
6108                         max = ch->num_queue_pairs;
6109         }
6110
6111         return max;
6112 }
6113
6114 /**
6115  * i40e_validate_num_queues - validate num_queues w.r.t channel
6116  * @pf: ptr to PF device
6117  * @num_queues: number of queues
6118  * @vsi: the parent VSI
6119  * @reconfig_rss: indicates should the RSS be reconfigured or not
6120  *
6121  * This function validates number of queues in the context of new channel
6122  * which is being established and determines if RSS should be reconfigured
6123  * or not for parent VSI.
6124  **/
6125 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6126                                     struct i40e_vsi *vsi, bool *reconfig_rss)
6127 {
6128         int max_ch_queues;
6129
6130         if (!reconfig_rss)
6131                 return -EINVAL;
6132
6133         *reconfig_rss = false;
6134         if (vsi->current_rss_size) {
6135                 if (num_queues > vsi->current_rss_size) {
6136                         dev_dbg(&pf->pdev->dev,
6137                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
6138                                 num_queues, vsi->current_rss_size);
6139                         return -EINVAL;
6140                 } else if ((num_queues < vsi->current_rss_size) &&
6141                            (!is_power_of_2(num_queues))) {
6142                         dev_dbg(&pf->pdev->dev,
6143                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6144                                 num_queues, vsi->current_rss_size);
6145                         return -EINVAL;
6146                 }
6147         }
6148
6149         if (!is_power_of_2(num_queues)) {
6150                 /* Find the max num_queues configured for channel if channel
6151                  * exist.
6152                  * if channel exist, then enforce 'num_queues' to be more than
6153                  * max ever queues configured for channel.
6154                  */
6155                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6156                 if (num_queues < max_ch_queues) {
6157                         dev_dbg(&pf->pdev->dev,
6158                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
6159                                 num_queues, max_ch_queues);
6160                         return -EINVAL;
6161                 }
6162                 *reconfig_rss = true;
6163         }
6164
6165         return 0;
6166 }
6167
6168 /**
6169  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6170  * @vsi: the VSI being setup
6171  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6172  *
6173  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6174  **/
6175 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6176 {
6177         struct i40e_pf *pf = vsi->back;
6178         u8 seed[I40E_HKEY_ARRAY_SIZE];
6179         struct i40e_hw *hw = &pf->hw;
6180         int local_rss_size;
6181         u8 *lut;
6182         int ret;
6183
6184         if (!vsi->rss_size)
6185                 return -EINVAL;
6186
6187         if (rss_size > vsi->rss_size)
6188                 return -EINVAL;
6189
6190         local_rss_size = min_t(int, vsi->rss_size, rss_size);
6191         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6192         if (!lut)
6193                 return -ENOMEM;
6194
6195         /* Ignoring user configured lut if there is one */
6196         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6197
6198         /* Use user configured hash key if there is one, otherwise
6199          * use default.
6200          */
6201         if (vsi->rss_hkey_user)
6202                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6203         else
6204                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6205
6206         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6207         if (ret) {
6208                 dev_info(&pf->pdev->dev,
6209                          "Cannot set RSS lut, err %pe aq_err %s\n",
6210                          ERR_PTR(ret),
6211                          i40e_aq_str(hw, hw->aq.asq_last_status));
6212                 kfree(lut);
6213                 return ret;
6214         }
6215         kfree(lut);
6216
6217         /* Do the update w.r.t. storing rss_size */
6218         if (!vsi->orig_rss_size)
6219                 vsi->orig_rss_size = vsi->rss_size;
6220         vsi->current_rss_size = local_rss_size;
6221
6222         return ret;
6223 }
6224
6225 /**
6226  * i40e_channel_setup_queue_map - Setup a channel queue map
6227  * @pf: ptr to PF device
6228  * @ctxt: VSI context structure
6229  * @ch: ptr to channel structure
6230  *
6231  * Setup queue map for a specific channel
6232  **/
6233 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6234                                          struct i40e_vsi_context *ctxt,
6235                                          struct i40e_channel *ch)
6236 {
6237         u16 qcount, qmap, sections = 0;
6238         u8 offset = 0;
6239         int pow;
6240
6241         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6242         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6243
6244         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6245         ch->num_queue_pairs = qcount;
6246
6247         /* find the next higher power-of-2 of num queue pairs */
6248         pow = ilog2(qcount);
6249         if (!is_power_of_2(qcount))
6250                 pow++;
6251
6252         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6253                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6254
6255         /* Setup queue TC[0].qmap for given VSI context */
6256         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6257
6258         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6259         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6260         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6261         ctxt->info.valid_sections |= cpu_to_le16(sections);
6262 }
6263
6264 /**
6265  * i40e_add_channel - add a channel by adding VSI
6266  * @pf: ptr to PF device
6267  * @uplink_seid: underlying HW switching element (VEB) ID
6268  * @ch: ptr to channel structure
6269  *
6270  * Add a channel (VSI) using add_vsi and queue_map
6271  **/
6272 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6273                             struct i40e_channel *ch)
6274 {
6275         struct i40e_hw *hw = &pf->hw;
6276         struct i40e_vsi_context ctxt;
6277         u8 enabled_tc = 0x1; /* TC0 enabled */
6278         int ret;
6279
6280         if (ch->type != I40E_VSI_VMDQ2) {
6281                 dev_info(&pf->pdev->dev,
6282                          "add new vsi failed, ch->type %d\n", ch->type);
6283                 return -EINVAL;
6284         }
6285
6286         memset(&ctxt, 0, sizeof(ctxt));
6287         ctxt.pf_num = hw->pf_id;
6288         ctxt.vf_num = 0;
6289         ctxt.uplink_seid = uplink_seid;
6290         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6291         if (ch->type == I40E_VSI_VMDQ2)
6292                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6293
6294         if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6295                 ctxt.info.valid_sections |=
6296                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6297                 ctxt.info.switch_id =
6298                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6299         }
6300
6301         /* Set queue map for a given VSI context */
6302         i40e_channel_setup_queue_map(pf, &ctxt, ch);
6303
6304         /* Now time to create VSI */
6305         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6306         if (ret) {
6307                 dev_info(&pf->pdev->dev,
6308                          "add new vsi failed, err %pe aq_err %s\n",
6309                          ERR_PTR(ret),
6310                          i40e_aq_str(&pf->hw,
6311                                      pf->hw.aq.asq_last_status));
6312                 return -ENOENT;
6313         }
6314
6315         /* Success, update channel, set enabled_tc only if the channel
6316          * is not a macvlan
6317          */
6318         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6319         ch->seid = ctxt.seid;
6320         ch->vsi_number = ctxt.vsi_number;
6321         ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6322
6323         /* copy just the sections touched not the entire info
6324          * since not all sections are valid as returned by
6325          * update vsi params
6326          */
6327         ch->info.mapping_flags = ctxt.info.mapping_flags;
6328         memcpy(&ch->info.queue_mapping,
6329                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6330         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6331                sizeof(ctxt.info.tc_mapping));
6332
6333         return 0;
6334 }
6335
6336 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6337                                   u8 *bw_share)
6338 {
6339         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6340         int ret;
6341         int i;
6342
6343         memset(&bw_data, 0, sizeof(bw_data));
6344         bw_data.tc_valid_bits = ch->enabled_tc;
6345         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6346                 bw_data.tc_bw_credits[i] = bw_share[i];
6347
6348         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6349                                        &bw_data, NULL);
6350         if (ret) {
6351                 dev_info(&vsi->back->pdev->dev,
6352                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6353                          vsi->back->hw.aq.asq_last_status, ch->seid);
6354                 return -EINVAL;
6355         }
6356
6357         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6358                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6359
6360         return 0;
6361 }
6362
6363 /**
6364  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6365  * @pf: ptr to PF device
6366  * @vsi: the VSI being setup
6367  * @ch: ptr to channel structure
6368  *
6369  * Configure TX rings associated with channel (VSI) since queues are being
6370  * from parent VSI.
6371  **/
6372 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6373                                        struct i40e_vsi *vsi,
6374                                        struct i40e_channel *ch)
6375 {
6376         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6377         int ret;
6378         int i;
6379
6380         /* Enable ETS TCs with equal BW Share for now across all VSIs */
6381         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6382                 if (ch->enabled_tc & BIT(i))
6383                         bw_share[i] = 1;
6384         }
6385
6386         /* configure BW for new VSI */
6387         ret = i40e_channel_config_bw(vsi, ch, bw_share);
6388         if (ret) {
6389                 dev_info(&vsi->back->pdev->dev,
6390                          "Failed configuring TC map %d for channel (seid %u)\n",
6391                          ch->enabled_tc, ch->seid);
6392                 return ret;
6393         }
6394
6395         for (i = 0; i < ch->num_queue_pairs; i++) {
6396                 struct i40e_ring *tx_ring, *rx_ring;
6397                 u16 pf_q;
6398
6399                 pf_q = ch->base_queue + i;
6400
6401                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6402                  * context
6403                  */
6404                 tx_ring = vsi->tx_rings[pf_q];
6405                 tx_ring->ch = ch;
6406
6407                 /* Get the RX ring ptr */
6408                 rx_ring = vsi->rx_rings[pf_q];
6409                 rx_ring->ch = ch;
6410         }
6411
6412         return 0;
6413 }
6414
6415 /**
6416  * i40e_setup_hw_channel - setup new channel
6417  * @pf: ptr to PF device
6418  * @vsi: the VSI being setup
6419  * @ch: ptr to channel structure
6420  * @uplink_seid: underlying HW switching element (VEB) ID
6421  * @type: type of channel to be created (VMDq2/VF)
6422  *
6423  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6424  * and configures TX rings accordingly
6425  **/
6426 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6427                                         struct i40e_vsi *vsi,
6428                                         struct i40e_channel *ch,
6429                                         u16 uplink_seid, u8 type)
6430 {
6431         int ret;
6432
6433         ch->initialized = false;
6434         ch->base_queue = vsi->next_base_queue;
6435         ch->type = type;
6436
6437         /* Proceed with creation of channel (VMDq2) VSI */
6438         ret = i40e_add_channel(pf, uplink_seid, ch);
6439         if (ret) {
6440                 dev_info(&pf->pdev->dev,
6441                          "failed to add_channel using uplink_seid %u\n",
6442                          uplink_seid);
6443                 return ret;
6444         }
6445
6446         /* Mark the successful creation of channel */
6447         ch->initialized = true;
6448
6449         /* Reconfigure TX queues using QTX_CTL register */
6450         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6451         if (ret) {
6452                 dev_info(&pf->pdev->dev,
6453                          "failed to configure TX rings for channel %u\n",
6454                          ch->seid);
6455                 return ret;
6456         }
6457
6458         /* update 'next_base_queue' */
6459         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6460         dev_dbg(&pf->pdev->dev,
6461                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6462                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6463                 ch->num_queue_pairs,
6464                 vsi->next_base_queue);
6465         return ret;
6466 }
6467
6468 /**
6469  * i40e_setup_channel - setup new channel using uplink element
6470  * @pf: ptr to PF device
6471  * @vsi: pointer to the VSI to set up the channel within
6472  * @ch: ptr to channel structure
6473  *
6474  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6475  * and uplink switching element (uplink_seid)
6476  **/
6477 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6478                                struct i40e_channel *ch)
6479 {
6480         u8 vsi_type;
6481         u16 seid;
6482         int ret;
6483
6484         if (vsi->type == I40E_VSI_MAIN) {
6485                 vsi_type = I40E_VSI_VMDQ2;
6486         } else {
6487                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6488                         vsi->type);
6489                 return false;
6490         }
6491
6492         /* underlying switching element */
6493         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6494
6495         /* create channel (VSI), configure TX rings */
6496         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6497         if (ret) {
6498                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6499                 return false;
6500         }
6501
6502         return ch->initialized ? true : false;
6503 }
6504
6505 /**
6506  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6507  * @vsi: ptr to VSI which has PF backing
6508  *
6509  * Sets up switch mode correctly if it needs to be changed and perform
6510  * what are allowed modes.
6511  **/
6512 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6513 {
6514         u8 mode;
6515         struct i40e_pf *pf = vsi->back;
6516         struct i40e_hw *hw = &pf->hw;
6517         int ret;
6518
6519         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6520         if (ret)
6521                 return -EINVAL;
6522
6523         if (hw->dev_caps.switch_mode) {
6524                 /* if switch mode is set, support mode2 (non-tunneled for
6525                  * cloud filter) for now
6526                  */
6527                 u32 switch_mode = hw->dev_caps.switch_mode &
6528                                   I40E_SWITCH_MODE_MASK;
6529                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6530                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6531                                 return 0;
6532                         dev_err(&pf->pdev->dev,
6533                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6534                                 hw->dev_caps.switch_mode);
6535                         return -EINVAL;
6536                 }
6537         }
6538
6539         /* Set Bit 7 to be valid */
6540         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6541
6542         /* Set L4type for TCP support */
6543         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6544
6545         /* Set cloud filter mode */
6546         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6547
6548         /* Prep mode field for set_switch_config */
6549         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6550                                         pf->last_sw_conf_valid_flags,
6551                                         mode, NULL);
6552         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6553                 dev_err(&pf->pdev->dev,
6554                         "couldn't set switch config bits, err %pe aq_err %s\n",
6555                         ERR_PTR(ret),
6556                         i40e_aq_str(hw,
6557                                     hw->aq.asq_last_status));
6558
6559         return ret;
6560 }
6561
6562 /**
6563  * i40e_create_queue_channel - function to create channel
6564  * @vsi: VSI to be configured
6565  * @ch: ptr to channel (it contains channel specific params)
6566  *
6567  * This function creates channel (VSI) using num_queues specified by user,
6568  * reconfigs RSS if needed.
6569  **/
6570 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6571                               struct i40e_channel *ch)
6572 {
6573         struct i40e_pf *pf = vsi->back;
6574         bool reconfig_rss;
6575         int err;
6576
6577         if (!ch)
6578                 return -EINVAL;
6579
6580         if (!ch->num_queue_pairs) {
6581                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6582                         ch->num_queue_pairs);
6583                 return -EINVAL;
6584         }
6585
6586         /* validate user requested num_queues for channel */
6587         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6588                                        &reconfig_rss);
6589         if (err) {
6590                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6591                          ch->num_queue_pairs);
6592                 return -EINVAL;
6593         }
6594
6595         /* By default we are in VEPA mode, if this is the first VF/VMDq
6596          * VSI to be added switch to VEB mode.
6597          */
6598
6599         if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6600                 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
6601
6602                 if (vsi->type == I40E_VSI_MAIN) {
6603                         if (i40e_is_tc_mqprio_enabled(pf))
6604                                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6605                         else
6606                                 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6607                 }
6608                 /* now onwards for main VSI, number of queues will be value
6609                  * of TC0's queue count
6610                  */
6611         }
6612
6613         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6614          * it should be more than num_queues
6615          */
6616         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6617                 dev_dbg(&pf->pdev->dev,
6618                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6619                         vsi->cnt_q_avail, ch->num_queue_pairs);
6620                 return -EINVAL;
6621         }
6622
6623         /* reconfig_rss only if vsi type is MAIN_VSI */
6624         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6625                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6626                 if (err) {
6627                         dev_info(&pf->pdev->dev,
6628                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6629                                  ch->num_queue_pairs);
6630                         return -EINVAL;
6631                 }
6632         }
6633
6634         if (!i40e_setup_channel(pf, vsi, ch)) {
6635                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6636                 return -EINVAL;
6637         }
6638
6639         dev_info(&pf->pdev->dev,
6640                  "Setup channel (id:%u) utilizing num_queues %d\n",
6641                  ch->seid, ch->num_queue_pairs);
6642
6643         /* configure VSI for BW limit */
6644         if (ch->max_tx_rate) {
6645                 u64 credits = ch->max_tx_rate;
6646
6647                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6648                         return -EINVAL;
6649
6650                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6651                 dev_dbg(&pf->pdev->dev,
6652                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6653                         ch->max_tx_rate,
6654                         credits,
6655                         ch->seid);
6656         }
6657
6658         /* in case of VF, this will be main SRIOV VSI */
6659         ch->parent_vsi = vsi;
6660
6661         /* and update main_vsi's count for queue_available to use */
6662         vsi->cnt_q_avail -= ch->num_queue_pairs;
6663
6664         return 0;
6665 }
6666
6667 /**
6668  * i40e_configure_queue_channels - Add queue channel for the given TCs
6669  * @vsi: VSI to be configured
6670  *
6671  * Configures queue channel mapping to the given TCs
6672  **/
6673 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6674 {
6675         struct i40e_channel *ch;
6676         u64 max_rate = 0;
6677         int ret = 0, i;
6678
6679         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6680         vsi->tc_seid_map[0] = vsi->seid;
6681         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6682                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6683                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6684                         if (!ch) {
6685                                 ret = -ENOMEM;
6686                                 goto err_free;
6687                         }
6688
6689                         INIT_LIST_HEAD(&ch->list);
6690                         ch->num_queue_pairs =
6691                                 vsi->tc_config.tc_info[i].qcount;
6692                         ch->base_queue =
6693                                 vsi->tc_config.tc_info[i].qoffset;
6694
6695                         /* Bandwidth limit through tc interface is in bytes/s,
6696                          * change to Mbit/s
6697                          */
6698                         max_rate = vsi->mqprio_qopt.max_rate[i];
6699                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6700                         ch->max_tx_rate = max_rate;
6701
6702                         list_add_tail(&ch->list, &vsi->ch_list);
6703
6704                         ret = i40e_create_queue_channel(vsi, ch);
6705                         if (ret) {
6706                                 dev_err(&vsi->back->pdev->dev,
6707                                         "Failed creating queue channel with TC%d: queues %d\n",
6708                                         i, ch->num_queue_pairs);
6709                                 goto err_free;
6710                         }
6711                         vsi->tc_seid_map[i] = ch->seid;
6712                 }
6713         }
6714
6715         /* reset to reconfigure TX queue contexts */
6716         i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6717         return ret;
6718
6719 err_free:
6720         i40e_remove_queue_channels(vsi);
6721         return ret;
6722 }
6723
6724 /**
6725  * i40e_veb_config_tc - Configure TCs for given VEB
6726  * @veb: given VEB
6727  * @enabled_tc: TC bitmap
6728  *
6729  * Configures given TC bitmap for VEB (switching) element
6730  **/
6731 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6732 {
6733         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6734         struct i40e_pf *pf = veb->pf;
6735         int ret = 0;
6736         int i;
6737
6738         /* No TCs or already enabled TCs just return */
6739         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6740                 return ret;
6741
6742         bw_data.tc_valid_bits = enabled_tc;
6743         /* bw_data.absolute_credits is not set (relative) */
6744
6745         /* Enable ETS TCs with equal BW Share for now */
6746         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6747                 if (enabled_tc & BIT(i))
6748                         bw_data.tc_bw_share_credits[i] = 1;
6749         }
6750
6751         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6752                                                    &bw_data, NULL);
6753         if (ret) {
6754                 dev_info(&pf->pdev->dev,
6755                          "VEB bw config failed, err %pe aq_err %s\n",
6756                          ERR_PTR(ret),
6757                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6758                 goto out;
6759         }
6760
6761         /* Update the BW information */
6762         ret = i40e_veb_get_bw_info(veb);
6763         if (ret) {
6764                 dev_info(&pf->pdev->dev,
6765                          "Failed getting veb bw config, err %pe aq_err %s\n",
6766                          ERR_PTR(ret),
6767                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6768         }
6769
6770 out:
6771         return ret;
6772 }
6773
6774 #ifdef CONFIG_I40E_DCB
6775 /**
6776  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6777  * @pf: PF struct
6778  *
6779  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6780  * the caller would've quiesce all the VSIs before calling
6781  * this function
6782  **/
6783 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6784 {
6785         struct i40e_vsi *vsi;
6786         struct i40e_veb *veb;
6787         u8 tc_map = 0;
6788         int ret;
6789         int v;
6790
6791         /* Enable the TCs available on PF to all VEBs */
6792         tc_map = i40e_pf_get_tc_map(pf);
6793         if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6794                 return;
6795
6796         i40e_pf_for_each_veb(pf, v, veb) {
6797                 ret = i40e_veb_config_tc(veb, tc_map);
6798                 if (ret) {
6799                         dev_info(&pf->pdev->dev,
6800                                  "Failed configuring TC for VEB seid=%d\n",
6801                                  veb->seid);
6802                         /* Will try to configure as many components */
6803                 }
6804         }
6805
6806         /* Update each VSI */
6807         i40e_pf_for_each_vsi(pf, v, vsi) {
6808                 /* - Enable all TCs for the LAN VSI
6809                  * - For all others keep them at TC0 for now
6810                  */
6811                 if (v == pf->lan_vsi)
6812                         tc_map = i40e_pf_get_tc_map(pf);
6813                 else
6814                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6815
6816                 ret = i40e_vsi_config_tc(vsi, tc_map);
6817                 if (ret) {
6818                         dev_info(&pf->pdev->dev,
6819                                  "Failed configuring TC for VSI seid=%d\n",
6820                                  vsi->seid);
6821                         /* Will try to configure as many components */
6822                 } else {
6823                         /* Re-configure VSI vectors based on updated TC map */
6824                         i40e_vsi_map_rings_to_vectors(vsi);
6825                         if (vsi->netdev)
6826                                 i40e_dcbnl_set_all(vsi);
6827                 }
6828         }
6829 }
6830
6831 /**
6832  * i40e_resume_port_tx - Resume port Tx
6833  * @pf: PF struct
6834  *
6835  * Resume a port's Tx and issue a PF reset in case of failure to
6836  * resume.
6837  **/
6838 static int i40e_resume_port_tx(struct i40e_pf *pf)
6839 {
6840         struct i40e_hw *hw = &pf->hw;
6841         int ret;
6842
6843         ret = i40e_aq_resume_port_tx(hw, NULL);
6844         if (ret) {
6845                 dev_info(&pf->pdev->dev,
6846                          "Resume Port Tx failed, err %pe aq_err %s\n",
6847                           ERR_PTR(ret),
6848                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6849                 /* Schedule PF reset to recover */
6850                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6851                 i40e_service_event_schedule(pf);
6852         }
6853
6854         return ret;
6855 }
6856
6857 /**
6858  * i40e_suspend_port_tx - Suspend port Tx
6859  * @pf: PF struct
6860  *
6861  * Suspend a port's Tx and issue a PF reset in case of failure.
6862  **/
6863 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6864 {
6865         struct i40e_hw *hw = &pf->hw;
6866         int ret;
6867
6868         ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6869         if (ret) {
6870                 dev_info(&pf->pdev->dev,
6871                          "Suspend Port Tx failed, err %pe aq_err %s\n",
6872                          ERR_PTR(ret),
6873                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6874                 /* Schedule PF reset to recover */
6875                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6876                 i40e_service_event_schedule(pf);
6877         }
6878
6879         return ret;
6880 }
6881
6882 /**
6883  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6884  * @pf: PF being configured
6885  * @new_cfg: New DCBX configuration
6886  *
6887  * Program DCB settings into HW and reconfigure VEB/VSIs on
6888  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6889  **/
6890 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6891                                   struct i40e_dcbx_config *new_cfg)
6892 {
6893         struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6894         int ret;
6895
6896         /* Check if need reconfiguration */
6897         if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6898                 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6899                 return 0;
6900         }
6901
6902         /* Config change disable all VSIs */
6903         i40e_pf_quiesce_all_vsi(pf);
6904
6905         /* Copy the new config to the current config */
6906         *old_cfg = *new_cfg;
6907         old_cfg->etsrec = old_cfg->etscfg;
6908         ret = i40e_set_dcb_config(&pf->hw);
6909         if (ret) {
6910                 dev_info(&pf->pdev->dev,
6911                          "Set DCB Config failed, err %pe aq_err %s\n",
6912                          ERR_PTR(ret),
6913                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6914                 goto out;
6915         }
6916
6917         /* Changes in configuration update VEB/VSI */
6918         i40e_dcb_reconfigure(pf);
6919 out:
6920         /* In case of reset do not try to resume anything */
6921         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6922                 /* Re-start the VSIs if disabled */
6923                 ret = i40e_resume_port_tx(pf);
6924                 /* In case of error no point in resuming VSIs */
6925                 if (ret)
6926                         goto err;
6927                 i40e_pf_unquiesce_all_vsi(pf);
6928         }
6929 err:
6930         return ret;
6931 }
6932
6933 /**
6934  * i40e_hw_dcb_config - Program new DCBX settings into HW
6935  * @pf: PF being configured
6936  * @new_cfg: New DCBX configuration
6937  *
6938  * Program DCB settings into HW and reconfigure VEB/VSIs on
6939  * given PF
6940  **/
6941 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6942 {
6943         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6944         u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6945         u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6946         struct i40e_dcbx_config *old_cfg;
6947         u8 mode[I40E_MAX_TRAFFIC_CLASS];
6948         struct i40e_rx_pb_config pb_cfg;
6949         struct i40e_hw *hw = &pf->hw;
6950         u8 num_ports = hw->num_ports;
6951         bool need_reconfig;
6952         int ret = -EINVAL;
6953         u8 lltc_map = 0;
6954         u8 tc_map = 0;
6955         u8 new_numtc;
6956         u8 i;
6957
6958         dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6959         /* Un-pack information to Program ETS HW via shared API
6960          * numtc, tcmap
6961          * LLTC map
6962          * ETS/NON-ETS arbiter mode
6963          * max exponent (credit refills)
6964          * Total number of ports
6965          * PFC priority bit-map
6966          * Priority Table
6967          * BW % per TC
6968          * Arbiter mode between UPs sharing same TC
6969          * TSA table (ETS or non-ETS)
6970          * EEE enabled or not
6971          * MFS TC table
6972          */
6973
6974         new_numtc = i40e_dcb_get_num_tc(new_cfg);
6975
6976         memset(&ets_data, 0, sizeof(ets_data));
6977         for (i = 0; i < new_numtc; i++) {
6978                 tc_map |= BIT(i);
6979                 switch (new_cfg->etscfg.tsatable[i]) {
6980                 case I40E_IEEE_TSA_ETS:
6981                         prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6982                         ets_data.tc_bw_share_credits[i] =
6983                                         new_cfg->etscfg.tcbwtable[i];
6984                         break;
6985                 case I40E_IEEE_TSA_STRICT:
6986                         prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6987                         lltc_map |= BIT(i);
6988                         ets_data.tc_bw_share_credits[i] =
6989                                         I40E_DCB_STRICT_PRIO_CREDITS;
6990                         break;
6991                 default:
6992                         /* Invalid TSA type */
6993                         need_reconfig = false;
6994                         goto out;
6995                 }
6996         }
6997
6998         old_cfg = &hw->local_dcbx_config;
6999         /* Check if need reconfiguration */
7000         need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
7001
7002         /* If needed, enable/disable frame tagging, disable all VSIs
7003          * and suspend port tx
7004          */
7005         if (need_reconfig) {
7006                 /* Enable DCB tagging only when more than one TC */
7007                 if (new_numtc > 1)
7008                         set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7009                 else
7010                         clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7011
7012                 set_bit(__I40E_PORT_SUSPENDED, pf->state);
7013                 /* Reconfiguration needed quiesce all VSIs */
7014                 i40e_pf_quiesce_all_vsi(pf);
7015                 ret = i40e_suspend_port_tx(pf);
7016                 if (ret)
7017                         goto err;
7018         }
7019
7020         /* Configure Port ETS Tx Scheduler */
7021         ets_data.tc_valid_bits = tc_map;
7022         ets_data.tc_strict_priority_flags = lltc_map;
7023         ret = i40e_aq_config_switch_comp_ets
7024                 (hw, pf->mac_seid, &ets_data,
7025                  i40e_aqc_opc_modify_switching_comp_ets, NULL);
7026         if (ret) {
7027                 dev_info(&pf->pdev->dev,
7028                          "Modify Port ETS failed, err %pe aq_err %s\n",
7029                          ERR_PTR(ret),
7030                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7031                 goto out;
7032         }
7033
7034         /* Configure Rx ETS HW */
7035         memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7036         i40e_dcb_hw_set_num_tc(hw, new_numtc);
7037         i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7038                                    I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7039                                    I40E_DCB_DEFAULT_MAX_EXPONENT,
7040                                    lltc_map);
7041         i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7042         i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7043                                      prio_type);
7044         i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7045                                new_cfg->etscfg.prioritytable);
7046         i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7047
7048         /* Configure Rx Packet Buffers in HW */
7049         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7050                 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
7051                 mfs_tc[i] += I40E_PACKET_HDR_PAD;
7052         }
7053
7054         i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7055                                          false, new_cfg->pfc.pfcenable,
7056                                          mfs_tc, &pb_cfg);
7057         i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7058
7059         /* Update the local Rx Packet buffer config */
7060         pf->pb_cfg = pb_cfg;
7061
7062         /* Inform the FW about changes to DCB configuration */
7063         ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7064         if (ret) {
7065                 dev_info(&pf->pdev->dev,
7066                          "DCB Updated failed, err %pe aq_err %s\n",
7067                          ERR_PTR(ret),
7068                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7069                 goto out;
7070         }
7071
7072         /* Update the port DCBx configuration */
7073         *old_cfg = *new_cfg;
7074
7075         /* Changes in configuration update VEB/VSI */
7076         i40e_dcb_reconfigure(pf);
7077 out:
7078         /* Re-start the VSIs if disabled */
7079         if (need_reconfig) {
7080                 ret = i40e_resume_port_tx(pf);
7081
7082                 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7083                 /* In case of error no point in resuming VSIs */
7084                 if (ret)
7085                         goto err;
7086
7087                 /* Wait for the PF's queues to be disabled */
7088                 ret = i40e_pf_wait_queues_disabled(pf);
7089                 if (ret) {
7090                         /* Schedule PF reset to recover */
7091                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7092                         i40e_service_event_schedule(pf);
7093                         goto err;
7094                 } else {
7095                         i40e_pf_unquiesce_all_vsi(pf);
7096                         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7097                         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7098                 }
7099                 /* registers are set, lets apply */
7100                 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
7101                         ret = i40e_hw_set_dcb_config(pf, new_cfg);
7102         }
7103
7104 err:
7105         return ret;
7106 }
7107
7108 /**
7109  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7110  * @pf: PF being queried
7111  *
7112  * Set default DCB configuration in case DCB is to be done in SW.
7113  **/
7114 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7115 {
7116         struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7117         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7118         struct i40e_hw *hw = &pf->hw;
7119         int err;
7120
7121         if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
7122                 /* Update the local cached instance with TC0 ETS */
7123                 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7124                 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7125                 pf->tmp_cfg.etscfg.maxtcs = 0;
7126                 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7127                 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7128                 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7129                 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7130                 /* FW needs one App to configure HW */
7131                 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7132                 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7133                 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7134                 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7135
7136                 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7137         }
7138
7139         memset(&ets_data, 0, sizeof(ets_data));
7140         ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7141         ets_data.tc_strict_priority_flags = 0; /* ETS */
7142         ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7143
7144         /* Enable ETS on the Physical port */
7145         err = i40e_aq_config_switch_comp_ets
7146                 (hw, pf->mac_seid, &ets_data,
7147                  i40e_aqc_opc_enable_switching_comp_ets, NULL);
7148         if (err) {
7149                 dev_info(&pf->pdev->dev,
7150                          "Enable Port ETS failed, err %pe aq_err %s\n",
7151                          ERR_PTR(err),
7152                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7153                 err = -ENOENT;
7154                 goto out;
7155         }
7156
7157         /* Update the local cached instance with TC0 ETS */
7158         dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7159         dcb_cfg->etscfg.cbs = 0;
7160         dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7161         dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7162
7163 out:
7164         return err;
7165 }
7166
7167 /**
7168  * i40e_init_pf_dcb - Initialize DCB configuration
7169  * @pf: PF being configured
7170  *
7171  * Query the current DCB configuration and cache it
7172  * in the hardware structure
7173  **/
7174 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7175 {
7176         struct i40e_hw *hw = &pf->hw;
7177         int err;
7178
7179         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7180          * Also do not enable DCBx if FW LLDP agent is disabled
7181          */
7182         if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
7183                 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7184                 err = -EOPNOTSUPP;
7185                 goto out;
7186         }
7187         if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
7188                 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7189                 err = i40e_dcb_sw_default_config(pf);
7190                 if (err) {
7191                         dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7192                         goto out;
7193                 }
7194                 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7195                 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7196                                DCB_CAP_DCBX_VER_IEEE;
7197                 /* at init capable but disabled */
7198                 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7199                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7200                 goto out;
7201         }
7202         err = i40e_init_dcb(hw, true);
7203         if (!err) {
7204                 /* Device/Function is not DCBX capable */
7205                 if ((!hw->func_caps.dcb) ||
7206                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7207                         dev_info(&pf->pdev->dev,
7208                                  "DCBX offload is not supported or is disabled for this PF.\n");
7209                 } else {
7210                         /* When status is not DISABLED then DCBX in FW */
7211                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7212                                        DCB_CAP_DCBX_VER_IEEE;
7213
7214                         set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7215                         /* Enable DCB tagging only when more than one TC
7216                          * or explicitly disable if only one TC
7217                          */
7218                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7219                                 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7220                         else
7221                                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7222                         dev_dbg(&pf->pdev->dev,
7223                                 "DCBX offload is supported for this PF.\n");
7224                 }
7225         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7226                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7227                 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
7228         } else {
7229                 dev_info(&pf->pdev->dev,
7230                          "Query for DCB configuration failed, err %pe aq_err %s\n",
7231                          ERR_PTR(err),
7232                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7233         }
7234
7235 out:
7236         return err;
7237 }
7238 #endif /* CONFIG_I40E_DCB */
7239
7240 /**
7241  * i40e_print_link_message - print link up or down
7242  * @vsi: the VSI for which link needs a message
7243  * @isup: true of link is up, false otherwise
7244  */
7245 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7246 {
7247         enum i40e_aq_link_speed new_speed;
7248         struct i40e_pf *pf = vsi->back;
7249         char *speed = "Unknown";
7250         char *fc = "Unknown";
7251         char *fec = "";
7252         char *req_fec = "";
7253         char *an = "";
7254
7255         if (isup)
7256                 new_speed = pf->hw.phy.link_info.link_speed;
7257         else
7258                 new_speed = I40E_LINK_SPEED_UNKNOWN;
7259
7260         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7261                 return;
7262         vsi->current_isup = isup;
7263         vsi->current_speed = new_speed;
7264         if (!isup) {
7265                 netdev_info(vsi->netdev, "NIC Link is Down\n");
7266                 return;
7267         }
7268
7269         /* Warn user if link speed on NPAR enabled partition is not at
7270          * least 10GB
7271          */
7272         if (pf->hw.func_caps.npar_enable &&
7273             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7274              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7275                 netdev_warn(vsi->netdev,
7276                             "The partition detected link speed that is less than 10Gbps\n");
7277
7278         switch (pf->hw.phy.link_info.link_speed) {
7279         case I40E_LINK_SPEED_40GB:
7280                 speed = "40 G";
7281                 break;
7282         case I40E_LINK_SPEED_20GB:
7283                 speed = "20 G";
7284                 break;
7285         case I40E_LINK_SPEED_25GB:
7286                 speed = "25 G";
7287                 break;
7288         case I40E_LINK_SPEED_10GB:
7289                 speed = "10 G";
7290                 break;
7291         case I40E_LINK_SPEED_5GB:
7292                 speed = "5 G";
7293                 break;
7294         case I40E_LINK_SPEED_2_5GB:
7295                 speed = "2.5 G";
7296                 break;
7297         case I40E_LINK_SPEED_1GB:
7298                 speed = "1000 M";
7299                 break;
7300         case I40E_LINK_SPEED_100MB:
7301                 speed = "100 M";
7302                 break;
7303         default:
7304                 break;
7305         }
7306
7307         switch (pf->hw.fc.current_mode) {
7308         case I40E_FC_FULL:
7309                 fc = "RX/TX";
7310                 break;
7311         case I40E_FC_TX_PAUSE:
7312                 fc = "TX";
7313                 break;
7314         case I40E_FC_RX_PAUSE:
7315                 fc = "RX";
7316                 break;
7317         default:
7318                 fc = "None";
7319                 break;
7320         }
7321
7322         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7323                 req_fec = "None";
7324                 fec = "None";
7325                 an = "False";
7326
7327                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7328                         an = "True";
7329
7330                 if (pf->hw.phy.link_info.fec_info &
7331                     I40E_AQ_CONFIG_FEC_KR_ENA)
7332                         fec = "CL74 FC-FEC/BASE-R";
7333                 else if (pf->hw.phy.link_info.fec_info &
7334                          I40E_AQ_CONFIG_FEC_RS_ENA)
7335                         fec = "CL108 RS-FEC";
7336
7337                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7338                  * both RS and FC are requested
7339                  */
7340                 if (vsi->back->hw.phy.link_info.req_fec_info &
7341                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7342                         if (vsi->back->hw.phy.link_info.req_fec_info &
7343                             I40E_AQ_REQUEST_FEC_RS)
7344                                 req_fec = "CL108 RS-FEC";
7345                         else
7346                                 req_fec = "CL74 FC-FEC/BASE-R";
7347                 }
7348                 netdev_info(vsi->netdev,
7349                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7350                             speed, req_fec, fec, an, fc);
7351         } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7352                 req_fec = "None";
7353                 fec = "None";
7354                 an = "False";
7355
7356                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7357                         an = "True";
7358
7359                 if (pf->hw.phy.link_info.fec_info &
7360                     I40E_AQ_CONFIG_FEC_KR_ENA)
7361                         fec = "CL74 FC-FEC/BASE-R";
7362
7363                 if (pf->hw.phy.link_info.req_fec_info &
7364                     I40E_AQ_REQUEST_FEC_KR)
7365                         req_fec = "CL74 FC-FEC/BASE-R";
7366
7367                 netdev_info(vsi->netdev,
7368                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7369                             speed, req_fec, fec, an, fc);
7370         } else {
7371                 netdev_info(vsi->netdev,
7372                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7373                             speed, fc);
7374         }
7375
7376 }
7377
7378 /**
7379  * i40e_up_complete - Finish the last steps of bringing up a connection
7380  * @vsi: the VSI being configured
7381  **/
7382 static int i40e_up_complete(struct i40e_vsi *vsi)
7383 {
7384         struct i40e_pf *pf = vsi->back;
7385         int err;
7386
7387         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
7388                 i40e_vsi_configure_msix(vsi);
7389         else
7390                 i40e_configure_msi_and_legacy(vsi);
7391
7392         /* start rings */
7393         err = i40e_vsi_start_rings(vsi);
7394         if (err)
7395                 return err;
7396
7397         clear_bit(__I40E_VSI_DOWN, vsi->state);
7398         i40e_napi_enable_all(vsi);
7399         i40e_vsi_enable_irq(vsi);
7400
7401         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7402             (vsi->netdev)) {
7403                 i40e_print_link_message(vsi, true);
7404                 netif_tx_start_all_queues(vsi->netdev);
7405                 netif_carrier_on(vsi->netdev);
7406         }
7407
7408         /* replay FDIR SB filters */
7409         if (vsi->type == I40E_VSI_FDIR) {
7410                 /* reset fd counters */
7411                 pf->fd_add_err = 0;
7412                 pf->fd_atr_cnt = 0;
7413                 i40e_fdir_filter_restore(vsi);
7414         }
7415
7416         /* On the next run of the service_task, notify any clients of the new
7417          * opened netdev
7418          */
7419         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7420         i40e_service_event_schedule(pf);
7421
7422         return 0;
7423 }
7424
7425 /**
7426  * i40e_vsi_reinit_locked - Reset the VSI
7427  * @vsi: the VSI being configured
7428  *
7429  * Rebuild the ring structs after some configuration
7430  * has changed, e.g. MTU size.
7431  **/
7432 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7433 {
7434         struct i40e_pf *pf = vsi->back;
7435
7436         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7437                 usleep_range(1000, 2000);
7438         i40e_down(vsi);
7439
7440         i40e_up(vsi);
7441         clear_bit(__I40E_CONFIG_BUSY, pf->state);
7442 }
7443
7444 /**
7445  * i40e_force_link_state - Force the link status
7446  * @pf: board private structure
7447  * @is_up: whether the link state should be forced up or down
7448  **/
7449 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7450 {
7451         struct i40e_aq_get_phy_abilities_resp abilities;
7452         struct i40e_aq_set_phy_config config = {0};
7453         bool non_zero_phy_type = is_up;
7454         struct i40e_hw *hw = &pf->hw;
7455         u64 mask;
7456         u8 speed;
7457         int err;
7458
7459         /* Card might've been put in an unstable state by other drivers
7460          * and applications, which causes incorrect speed values being
7461          * set on startup. In order to clear speed registers, we call
7462          * get_phy_capabilities twice, once to get initial state of
7463          * available speeds, and once to get current PHY config.
7464          */
7465         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7466                                            NULL);
7467         if (err) {
7468                 dev_err(&pf->pdev->dev,
7469                         "failed to get phy cap., ret =  %pe last_status =  %s\n",
7470                         ERR_PTR(err),
7471                         i40e_aq_str(hw, hw->aq.asq_last_status));
7472                 return err;
7473         }
7474         speed = abilities.link_speed;
7475
7476         /* Get the current phy config */
7477         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7478                                            NULL);
7479         if (err) {
7480                 dev_err(&pf->pdev->dev,
7481                         "failed to get phy cap., ret =  %pe last_status =  %s\n",
7482                         ERR_PTR(err),
7483                         i40e_aq_str(hw, hw->aq.asq_last_status));
7484                 return err;
7485         }
7486
7487         /* If link needs to go up, but was not forced to go down,
7488          * and its speed values are OK, no need for a flap
7489          * if non_zero_phy_type was set, still need to force up
7490          */
7491         if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
7492                 non_zero_phy_type = true;
7493         else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7494                 return 0;
7495
7496         /* To force link we need to set bits for all supported PHY types,
7497          * but there are now more than 32, so we need to split the bitmap
7498          * across two fields.
7499          */
7500         mask = I40E_PHY_TYPES_BITMASK;
7501         config.phy_type =
7502                 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7503         config.phy_type_ext =
7504                 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7505         /* Copy the old settings, except of phy_type */
7506         config.abilities = abilities.abilities;
7507         if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
7508                 if (is_up)
7509                         config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7510                 else
7511                         config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7512         }
7513         if (abilities.link_speed != 0)
7514                 config.link_speed = abilities.link_speed;
7515         else
7516                 config.link_speed = speed;
7517         config.eee_capability = abilities.eee_capability;
7518         config.eeer = abilities.eeer_val;
7519         config.low_power_ctrl = abilities.d3_lpan;
7520         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7521                             I40E_AQ_PHY_FEC_CONFIG_MASK;
7522         err = i40e_aq_set_phy_config(hw, &config, NULL);
7523
7524         if (err) {
7525                 dev_err(&pf->pdev->dev,
7526                         "set phy config ret =  %pe last_status =  %s\n",
7527                         ERR_PTR(err),
7528                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7529                 return err;
7530         }
7531
7532         /* Update the link info */
7533         err = i40e_update_link_info(hw);
7534         if (err) {
7535                 /* Wait a little bit (on 40G cards it sometimes takes a really
7536                  * long time for link to come back from the atomic reset)
7537                  * and try once more
7538                  */
7539                 msleep(1000);
7540                 i40e_update_link_info(hw);
7541         }
7542
7543         i40e_aq_set_link_restart_an(hw, is_up, NULL);
7544
7545         return 0;
7546 }
7547
7548 /**
7549  * i40e_up - Bring the connection back up after being down
7550  * @vsi: the VSI being configured
7551  **/
7552 int i40e_up(struct i40e_vsi *vsi)
7553 {
7554         int err;
7555
7556         if (vsi->type == I40E_VSI_MAIN &&
7557             (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7558              test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7559                 i40e_force_link_state(vsi->back, true);
7560
7561         err = i40e_vsi_configure(vsi);
7562         if (!err)
7563                 err = i40e_up_complete(vsi);
7564
7565         return err;
7566 }
7567
7568 /**
7569  * i40e_down - Shutdown the connection processing
7570  * @vsi: the VSI being stopped
7571  **/
7572 void i40e_down(struct i40e_vsi *vsi)
7573 {
7574         int i;
7575
7576         /* It is assumed that the caller of this function
7577          * sets the vsi->state __I40E_VSI_DOWN bit.
7578          */
7579         if (vsi->netdev) {
7580                 netif_carrier_off(vsi->netdev);
7581                 netif_tx_disable(vsi->netdev);
7582         }
7583         i40e_vsi_disable_irq(vsi);
7584         i40e_vsi_stop_rings(vsi);
7585         if (vsi->type == I40E_VSI_MAIN &&
7586            (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7587             test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7588                 i40e_force_link_state(vsi->back, false);
7589         i40e_napi_disable_all(vsi);
7590
7591         for (i = 0; i < vsi->num_queue_pairs; i++) {
7592                 i40e_clean_tx_ring(vsi->tx_rings[i]);
7593                 if (i40e_enabled_xdp_vsi(vsi)) {
7594                         /* Make sure that in-progress ndo_xdp_xmit and
7595                          * ndo_xsk_wakeup calls are completed.
7596                          */
7597                         synchronize_rcu();
7598                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
7599                 }
7600                 i40e_clean_rx_ring(vsi->rx_rings[i]);
7601         }
7602
7603 }
7604
7605 /**
7606  * i40e_validate_mqprio_qopt- validate queue mapping info
7607  * @vsi: the VSI being configured
7608  * @mqprio_qopt: queue parametrs
7609  **/
7610 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7611                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
7612 {
7613         u64 sum_max_rate = 0;
7614         u64 max_rate = 0;
7615         int i;
7616
7617         if (mqprio_qopt->qopt.offset[0] != 0 ||
7618             mqprio_qopt->qopt.num_tc < 1 ||
7619             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7620                 return -EINVAL;
7621         for (i = 0; ; i++) {
7622                 if (!mqprio_qopt->qopt.count[i])
7623                         return -EINVAL;
7624                 if (mqprio_qopt->min_rate[i]) {
7625                         dev_err(&vsi->back->pdev->dev,
7626                                 "Invalid min tx rate (greater than 0) specified\n");
7627                         return -EINVAL;
7628                 }
7629                 max_rate = mqprio_qopt->max_rate[i];
7630                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7631                 sum_max_rate += max_rate;
7632
7633                 if (i >= mqprio_qopt->qopt.num_tc - 1)
7634                         break;
7635                 if (mqprio_qopt->qopt.offset[i + 1] !=
7636                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7637                         return -EINVAL;
7638         }
7639         if (vsi->num_queue_pairs <
7640             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7641                 dev_err(&vsi->back->pdev->dev,
7642                         "Failed to create traffic channel, insufficient number of queues.\n");
7643                 return -EINVAL;
7644         }
7645         if (sum_max_rate > i40e_get_link_speed(vsi)) {
7646                 dev_err(&vsi->back->pdev->dev,
7647                         "Invalid max tx rate specified\n");
7648                 return -EINVAL;
7649         }
7650         return 0;
7651 }
7652
7653 /**
7654  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7655  * @vsi: the VSI being configured
7656  **/
7657 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7658 {
7659         u16 qcount;
7660         int i;
7661
7662         /* Only TC0 is enabled */
7663         vsi->tc_config.numtc = 1;
7664         vsi->tc_config.enabled_tc = 1;
7665         qcount = min_t(int, vsi->alloc_queue_pairs,
7666                        i40e_pf_get_max_q_per_tc(vsi->back));
7667         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7668                 /* For the TC that is not enabled set the offset to default
7669                  * queue and allocate one queue for the given TC.
7670                  */
7671                 vsi->tc_config.tc_info[i].qoffset = 0;
7672                 if (i == 0)
7673                         vsi->tc_config.tc_info[i].qcount = qcount;
7674                 else
7675                         vsi->tc_config.tc_info[i].qcount = 1;
7676                 vsi->tc_config.tc_info[i].netdev_tc = 0;
7677         }
7678 }
7679
7680 /**
7681  * i40e_del_macvlan_filter
7682  * @hw: pointer to the HW structure
7683  * @seid: seid of the channel VSI
7684  * @macaddr: the mac address to apply as a filter
7685  * @aq_err: store the admin Q error
7686  *
7687  * This function deletes a mac filter on the channel VSI which serves as the
7688  * macvlan. Returns 0 on success.
7689  **/
7690 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7691                                    const u8 *macaddr, int *aq_err)
7692 {
7693         struct i40e_aqc_remove_macvlan_element_data element;
7694         int status;
7695
7696         memset(&element, 0, sizeof(element));
7697         ether_addr_copy(element.mac_addr, macaddr);
7698         element.vlan_tag = 0;
7699         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7700         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7701         *aq_err = hw->aq.asq_last_status;
7702
7703         return status;
7704 }
7705
7706 /**
7707  * i40e_add_macvlan_filter
7708  * @hw: pointer to the HW structure
7709  * @seid: seid of the channel VSI
7710  * @macaddr: the mac address to apply as a filter
7711  * @aq_err: store the admin Q error
7712  *
7713  * This function adds a mac filter on the channel VSI which serves as the
7714  * macvlan. Returns 0 on success.
7715  **/
7716 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7717                                    const u8 *macaddr, int *aq_err)
7718 {
7719         struct i40e_aqc_add_macvlan_element_data element;
7720         u16 cmd_flags = 0;
7721         int status;
7722
7723         ether_addr_copy(element.mac_addr, macaddr);
7724         element.vlan_tag = 0;
7725         element.queue_number = 0;
7726         element.match_method = I40E_AQC_MM_ERR_NO_RES;
7727         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7728         element.flags = cpu_to_le16(cmd_flags);
7729         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7730         *aq_err = hw->aq.asq_last_status;
7731
7732         return status;
7733 }
7734
7735 /**
7736  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7737  * @vsi: the VSI we want to access
7738  * @ch: the channel we want to access
7739  */
7740 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7741 {
7742         struct i40e_ring *tx_ring, *rx_ring;
7743         u16 pf_q;
7744         int i;
7745
7746         for (i = 0; i < ch->num_queue_pairs; i++) {
7747                 pf_q = ch->base_queue + i;
7748                 tx_ring = vsi->tx_rings[pf_q];
7749                 tx_ring->ch = NULL;
7750                 rx_ring = vsi->rx_rings[pf_q];
7751                 rx_ring->ch = NULL;
7752         }
7753 }
7754
7755 /**
7756  * i40e_free_macvlan_channels
7757  * @vsi: the VSI we want to access
7758  *
7759  * This function frees the Qs of the channel VSI from
7760  * the stack and also deletes the channel VSIs which
7761  * serve as macvlans.
7762  */
7763 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7764 {
7765         struct i40e_channel *ch, *ch_tmp;
7766         int ret;
7767
7768         if (list_empty(&vsi->macvlan_list))
7769                 return;
7770
7771         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7772                 struct i40e_vsi *parent_vsi;
7773
7774                 if (i40e_is_channel_macvlan(ch)) {
7775                         i40e_reset_ch_rings(vsi, ch);
7776                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7777                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7778                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7779                         kfree(ch->fwd);
7780                         ch->fwd = NULL;
7781                 }
7782
7783                 list_del(&ch->list);
7784                 parent_vsi = ch->parent_vsi;
7785                 if (!parent_vsi || !ch->initialized) {
7786                         kfree(ch);
7787                         continue;
7788                 }
7789
7790                 /* remove the VSI */
7791                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7792                                              NULL);
7793                 if (ret)
7794                         dev_err(&vsi->back->pdev->dev,
7795                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7796                                 ch->seid, parent_vsi->seid);
7797                 kfree(ch);
7798         }
7799         vsi->macvlan_cnt = 0;
7800 }
7801
7802 /**
7803  * i40e_fwd_ring_up - bring the macvlan device up
7804  * @vsi: the VSI we want to access
7805  * @vdev: macvlan netdevice
7806  * @fwd: the private fwd structure
7807  */
7808 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7809                             struct i40e_fwd_adapter *fwd)
7810 {
7811         struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7812         int ret = 0, num_tc = 1,  i, aq_err;
7813         struct i40e_pf *pf = vsi->back;
7814         struct i40e_hw *hw = &pf->hw;
7815
7816         /* Go through the list and find an available channel */
7817         list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7818                 if (!i40e_is_channel_macvlan(iter)) {
7819                         iter->fwd = fwd;
7820                         /* record configuration for macvlan interface in vdev */
7821                         for (i = 0; i < num_tc; i++)
7822                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7823                                                              i,
7824                                                              iter->num_queue_pairs,
7825                                                              iter->base_queue);
7826                         for (i = 0; i < iter->num_queue_pairs; i++) {
7827                                 struct i40e_ring *tx_ring, *rx_ring;
7828                                 u16 pf_q;
7829
7830                                 pf_q = iter->base_queue + i;
7831
7832                                 /* Get to TX ring ptr */
7833                                 tx_ring = vsi->tx_rings[pf_q];
7834                                 tx_ring->ch = iter;
7835
7836                                 /* Get the RX ring ptr */
7837                                 rx_ring = vsi->rx_rings[pf_q];
7838                                 rx_ring->ch = iter;
7839                         }
7840                         ch = iter;
7841                         break;
7842                 }
7843         }
7844
7845         if (!ch)
7846                 return -EINVAL;
7847
7848         /* Guarantee all rings are updated before we update the
7849          * MAC address filter.
7850          */
7851         wmb();
7852
7853         /* Add a mac filter */
7854         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7855         if (ret) {
7856                 /* if we cannot add the MAC rule then disable the offload */
7857                 macvlan_release_l2fw_offload(vdev);
7858                 for (i = 0; i < ch->num_queue_pairs; i++) {
7859                         struct i40e_ring *rx_ring;
7860                         u16 pf_q;
7861
7862                         pf_q = ch->base_queue + i;
7863                         rx_ring = vsi->rx_rings[pf_q];
7864                         rx_ring->netdev = NULL;
7865                 }
7866                 dev_info(&pf->pdev->dev,
7867                          "Error adding mac filter on macvlan err %pe, aq_err %s\n",
7868                           ERR_PTR(ret),
7869                           i40e_aq_str(hw, aq_err));
7870                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7871         }
7872
7873         return ret;
7874 }
7875
7876 /**
7877  * i40e_setup_macvlans - create the channels which will be macvlans
7878  * @vsi: the VSI we want to access
7879  * @macvlan_cnt: no. of macvlans to be setup
7880  * @qcnt: no. of Qs per macvlan
7881  * @vdev: macvlan netdevice
7882  */
7883 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7884                                struct net_device *vdev)
7885 {
7886         struct i40e_pf *pf = vsi->back;
7887         struct i40e_hw *hw = &pf->hw;
7888         struct i40e_vsi_context ctxt;
7889         u16 sections, qmap, num_qps;
7890         struct i40e_channel *ch;
7891         int i, pow, ret = 0;
7892         u8 offset = 0;
7893
7894         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7895                 return -EINVAL;
7896
7897         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7898
7899         /* find the next higher power-of-2 of num queue pairs */
7900         pow = fls(roundup_pow_of_two(num_qps) - 1);
7901
7902         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7903                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7904
7905         /* Setup context bits for the main VSI */
7906         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7907         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7908         memset(&ctxt, 0, sizeof(ctxt));
7909         ctxt.seid = vsi->seid;
7910         ctxt.pf_num = vsi->back->hw.pf_id;
7911         ctxt.vf_num = 0;
7912         ctxt.uplink_seid = vsi->uplink_seid;
7913         ctxt.info = vsi->info;
7914         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7915         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7916         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7917         ctxt.info.valid_sections |= cpu_to_le16(sections);
7918
7919         /* Reconfigure RSS for main VSI with new max queue count */
7920         vsi->rss_size = max_t(u16, num_qps, qcnt);
7921         ret = i40e_vsi_config_rss(vsi);
7922         if (ret) {
7923                 dev_info(&pf->pdev->dev,
7924                          "Failed to reconfig RSS for num_queues (%u)\n",
7925                          vsi->rss_size);
7926                 return ret;
7927         }
7928         vsi->reconfig_rss = true;
7929         dev_dbg(&vsi->back->pdev->dev,
7930                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7931         vsi->next_base_queue = num_qps;
7932         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7933
7934         /* Update the VSI after updating the VSI queue-mapping
7935          * information
7936          */
7937         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7938         if (ret) {
7939                 dev_info(&pf->pdev->dev,
7940                          "Update vsi tc config failed, err %pe aq_err %s\n",
7941                          ERR_PTR(ret),
7942                          i40e_aq_str(hw, hw->aq.asq_last_status));
7943                 return ret;
7944         }
7945         /* update the local VSI info with updated queue map */
7946         i40e_vsi_update_queue_map(vsi, &ctxt);
7947         vsi->info.valid_sections = 0;
7948
7949         /* Create channels for macvlans */
7950         INIT_LIST_HEAD(&vsi->macvlan_list);
7951         for (i = 0; i < macvlan_cnt; i++) {
7952                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7953                 if (!ch) {
7954                         ret = -ENOMEM;
7955                         goto err_free;
7956                 }
7957                 INIT_LIST_HEAD(&ch->list);
7958                 ch->num_queue_pairs = qcnt;
7959                 if (!i40e_setup_channel(pf, vsi, ch)) {
7960                         ret = -EINVAL;
7961                         kfree(ch);
7962                         goto err_free;
7963                 }
7964                 ch->parent_vsi = vsi;
7965                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7966                 vsi->macvlan_cnt++;
7967                 list_add_tail(&ch->list, &vsi->macvlan_list);
7968         }
7969
7970         return ret;
7971
7972 err_free:
7973         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7974         i40e_free_macvlan_channels(vsi);
7975
7976         return ret;
7977 }
7978
7979 /**
7980  * i40e_fwd_add - configure macvlans
7981  * @netdev: net device to configure
7982  * @vdev: macvlan netdevice
7983  **/
7984 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7985 {
7986         struct i40e_netdev_priv *np = netdev_priv(netdev);
7987         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7988         struct i40e_vsi *vsi = np->vsi;
7989         struct i40e_pf *pf = vsi->back;
7990         struct i40e_fwd_adapter *fwd;
7991         int avail_macvlan, ret;
7992
7993         if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
7994                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7995                 return ERR_PTR(-EINVAL);
7996         }
7997         if (i40e_is_tc_mqprio_enabled(pf)) {
7998                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7999                 return ERR_PTR(-EINVAL);
8000         }
8001         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
8002                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
8003                 return ERR_PTR(-EINVAL);
8004         }
8005
8006         /* The macvlan device has to be a single Q device so that the
8007          * tc_to_txq field can be reused to pick the tx queue.
8008          */
8009         if (netif_is_multiqueue(vdev))
8010                 return ERR_PTR(-ERANGE);
8011
8012         if (!vsi->macvlan_cnt) {
8013                 /* reserve bit 0 for the pf device */
8014                 set_bit(0, vsi->fwd_bitmask);
8015
8016                 /* Try to reserve as many queues as possible for macvlans. First
8017                  * reserve 3/4th of max vectors, then half, then quarter and
8018                  * calculate Qs per macvlan as you go
8019                  */
8020                 vectors = pf->num_lan_msix;
8021                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
8022                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
8023                         q_per_macvlan = 4;
8024                         macvlan_cnt = (vectors - 32) / 4;
8025                 } else if (vectors <= 64 && vectors > 32) {
8026                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
8027                         q_per_macvlan = 2;
8028                         macvlan_cnt = (vectors - 16) / 2;
8029                 } else if (vectors <= 32 && vectors > 16) {
8030                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
8031                         q_per_macvlan = 1;
8032                         macvlan_cnt = vectors - 16;
8033                 } else if (vectors <= 16 && vectors > 8) {
8034                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
8035                         q_per_macvlan = 1;
8036                         macvlan_cnt = vectors - 8;
8037                 } else {
8038                         /* allocate 1 Q per macvlan and 1 Q to the PF */
8039                         q_per_macvlan = 1;
8040                         macvlan_cnt = vectors - 1;
8041                 }
8042
8043                 if (macvlan_cnt == 0)
8044                         return ERR_PTR(-EBUSY);
8045
8046                 /* Quiesce VSI queues */
8047                 i40e_quiesce_vsi(vsi);
8048
8049                 /* sets up the macvlans but does not "enable" them */
8050                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8051                                           vdev);
8052                 if (ret)
8053                         return ERR_PTR(ret);
8054
8055                 /* Unquiesce VSI */
8056                 i40e_unquiesce_vsi(vsi);
8057         }
8058         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8059                                             vsi->macvlan_cnt);
8060         if (avail_macvlan >= I40E_MAX_MACVLANS)
8061                 return ERR_PTR(-EBUSY);
8062
8063         /* create the fwd struct */
8064         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8065         if (!fwd)
8066                 return ERR_PTR(-ENOMEM);
8067
8068         set_bit(avail_macvlan, vsi->fwd_bitmask);
8069         fwd->bit_no = avail_macvlan;
8070         netdev_set_sb_channel(vdev, avail_macvlan);
8071         fwd->netdev = vdev;
8072
8073         if (!netif_running(netdev))
8074                 return fwd;
8075
8076         /* Set fwd ring up */
8077         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8078         if (ret) {
8079                 /* unbind the queues and drop the subordinate channel config */
8080                 netdev_unbind_sb_channel(netdev, vdev);
8081                 netdev_set_sb_channel(vdev, 0);
8082
8083                 kfree(fwd);
8084                 return ERR_PTR(-EINVAL);
8085         }
8086
8087         return fwd;
8088 }
8089
8090 /**
8091  * i40e_del_all_macvlans - Delete all the mac filters on the channels
8092  * @vsi: the VSI we want to access
8093  */
8094 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8095 {
8096         struct i40e_channel *ch, *ch_tmp;
8097         struct i40e_pf *pf = vsi->back;
8098         struct i40e_hw *hw = &pf->hw;
8099         int aq_err, ret = 0;
8100
8101         if (list_empty(&vsi->macvlan_list))
8102                 return;
8103
8104         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8105                 if (i40e_is_channel_macvlan(ch)) {
8106                         ret = i40e_del_macvlan_filter(hw, ch->seid,
8107                                                       i40e_channel_mac(ch),
8108                                                       &aq_err);
8109                         if (!ret) {
8110                                 /* Reset queue contexts */
8111                                 i40e_reset_ch_rings(vsi, ch);
8112                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8113                                 netdev_unbind_sb_channel(vsi->netdev,
8114                                                          ch->fwd->netdev);
8115                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
8116                                 kfree(ch->fwd);
8117                                 ch->fwd = NULL;
8118                         }
8119                 }
8120         }
8121 }
8122
8123 /**
8124  * i40e_fwd_del - delete macvlan interfaces
8125  * @netdev: net device to configure
8126  * @vdev: macvlan netdevice
8127  */
8128 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8129 {
8130         struct i40e_netdev_priv *np = netdev_priv(netdev);
8131         struct i40e_fwd_adapter *fwd = vdev;
8132         struct i40e_channel *ch, *ch_tmp;
8133         struct i40e_vsi *vsi = np->vsi;
8134         struct i40e_pf *pf = vsi->back;
8135         struct i40e_hw *hw = &pf->hw;
8136         int aq_err, ret = 0;
8137
8138         /* Find the channel associated with the macvlan and del mac filter */
8139         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8140                 if (i40e_is_channel_macvlan(ch) &&
8141                     ether_addr_equal(i40e_channel_mac(ch),
8142                                      fwd->netdev->dev_addr)) {
8143                         ret = i40e_del_macvlan_filter(hw, ch->seid,
8144                                                       i40e_channel_mac(ch),
8145                                                       &aq_err);
8146                         if (!ret) {
8147                                 /* Reset queue contexts */
8148                                 i40e_reset_ch_rings(vsi, ch);
8149                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8150                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
8151                                 netdev_set_sb_channel(fwd->netdev, 0);
8152                                 kfree(ch->fwd);
8153                                 ch->fwd = NULL;
8154                         } else {
8155                                 dev_info(&pf->pdev->dev,
8156                                          "Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8157                                           ERR_PTR(ret),
8158                                           i40e_aq_str(hw, aq_err));
8159                         }
8160                         break;
8161                 }
8162         }
8163 }
8164
8165 /**
8166  * i40e_setup_tc - configure multiple traffic classes
8167  * @netdev: net device to configure
8168  * @type_data: tc offload data
8169  **/
8170 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8171 {
8172         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8173         struct i40e_netdev_priv *np = netdev_priv(netdev);
8174         struct i40e_vsi *vsi = np->vsi;
8175         struct i40e_pf *pf = vsi->back;
8176         u8 enabled_tc = 0, num_tc, hw;
8177         bool need_reset = false;
8178         int old_queue_pairs;
8179         int ret = -EINVAL;
8180         u16 mode;
8181         int i;
8182
8183         old_queue_pairs = vsi->num_queue_pairs;
8184         num_tc = mqprio_qopt->qopt.num_tc;
8185         hw = mqprio_qopt->qopt.hw;
8186         mode = mqprio_qopt->mode;
8187         if (!hw) {
8188                 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8189                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8190                 goto config_tc;
8191         }
8192
8193         /* Check if MFP enabled */
8194         if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
8195                 netdev_info(netdev,
8196                             "Configuring TC not supported in MFP mode\n");
8197                 return ret;
8198         }
8199         switch (mode) {
8200         case TC_MQPRIO_MODE_DCB:
8201                 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8202
8203                 /* Check if DCB enabled to continue */
8204                 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8205                         netdev_info(netdev,
8206                                     "DCB is not enabled for adapter\n");
8207                         return ret;
8208                 }
8209
8210                 /* Check whether tc count is within enabled limit */
8211                 if (num_tc > i40e_pf_get_num_tc(pf)) {
8212                         netdev_info(netdev,
8213                                     "TC count greater than enabled on link for adapter\n");
8214                         return ret;
8215                 }
8216                 break;
8217         case TC_MQPRIO_MODE_CHANNEL:
8218                 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8219                         netdev_info(netdev,
8220                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8221                         return ret;
8222                 }
8223                 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
8224                         return ret;
8225                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8226                 if (ret)
8227                         return ret;
8228                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8229                        sizeof(*mqprio_qopt));
8230                 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8231                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
8232                 break;
8233         default:
8234                 return -EINVAL;
8235         }
8236
8237 config_tc:
8238         /* Generate TC map for number of tc requested */
8239         for (i = 0; i < num_tc; i++)
8240                 enabled_tc |= BIT(i);
8241
8242         /* Requesting same TC configuration as already enabled */
8243         if (enabled_tc == vsi->tc_config.enabled_tc &&
8244             mode != TC_MQPRIO_MODE_CHANNEL)
8245                 return 0;
8246
8247         /* Quiesce VSI queues */
8248         i40e_quiesce_vsi(vsi);
8249
8250         if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8251                 i40e_remove_queue_channels(vsi);
8252
8253         /* Configure VSI for enabled TCs */
8254         ret = i40e_vsi_config_tc(vsi, enabled_tc);
8255         if (ret) {
8256                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8257                             vsi->seid);
8258                 need_reset = true;
8259                 goto exit;
8260         } else if (enabled_tc &&
8261                    (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8262                 netdev_info(netdev,
8263                             "Failed to create channel. Override queues (%u) not power of 2\n",
8264                             vsi->tc_config.tc_info[0].qcount);
8265                 ret = -EINVAL;
8266                 need_reset = true;
8267                 goto exit;
8268         }
8269
8270         dev_info(&vsi->back->pdev->dev,
8271                  "Setup channel (id:%u) utilizing num_queues %d\n",
8272                  vsi->seid, vsi->tc_config.tc_info[0].qcount);
8273
8274         if (i40e_is_tc_mqprio_enabled(pf)) {
8275                 if (vsi->mqprio_qopt.max_rate[0]) {
8276                         u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8277                                                   vsi->mqprio_qopt.max_rate[0]);
8278
8279                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8280                         if (!ret) {
8281                                 u64 credits = max_tx_rate;
8282
8283                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
8284                                 dev_dbg(&vsi->back->pdev->dev,
8285                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8286                                         max_tx_rate,
8287                                         credits,
8288                                         vsi->seid);
8289                         } else {
8290                                 need_reset = true;
8291                                 goto exit;
8292                         }
8293                 }
8294                 ret = i40e_configure_queue_channels(vsi);
8295                 if (ret) {
8296                         vsi->num_queue_pairs = old_queue_pairs;
8297                         netdev_info(netdev,
8298                                     "Failed configuring queue channels\n");
8299                         need_reset = true;
8300                         goto exit;
8301                 }
8302         }
8303
8304 exit:
8305         /* Reset the configuration data to defaults, only TC0 is enabled */
8306         if (need_reset) {
8307                 i40e_vsi_set_default_tc_config(vsi);
8308                 need_reset = false;
8309         }
8310
8311         /* Unquiesce VSI */
8312         i40e_unquiesce_vsi(vsi);
8313         return ret;
8314 }
8315
8316 /**
8317  * i40e_set_cld_element - sets cloud filter element data
8318  * @filter: cloud filter rule
8319  * @cld: ptr to cloud filter element data
8320  *
8321  * This is helper function to copy data into cloud filter element
8322  **/
8323 static inline void
8324 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8325                      struct i40e_aqc_cloud_filters_element_data *cld)
8326 {
8327         u32 ipa;
8328         int i;
8329
8330         memset(cld, 0, sizeof(*cld));
8331         ether_addr_copy(cld->outer_mac, filter->dst_mac);
8332         ether_addr_copy(cld->inner_mac, filter->src_mac);
8333
8334         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8335                 return;
8336
8337         if (filter->n_proto == ETH_P_IPV6) {
8338 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
8339                 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8340                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8341
8342                         *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8343                 }
8344         } else {
8345                 ipa = be32_to_cpu(filter->dst_ipv4);
8346
8347                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8348         }
8349
8350         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8351
8352         /* tenant_id is not supported by FW now, once the support is enabled
8353          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8354          */
8355         if (filter->tenant_id)
8356                 return;
8357 }
8358
8359 /**
8360  * i40e_add_del_cloud_filter - Add/del cloud filter
8361  * @vsi: pointer to VSI
8362  * @filter: cloud filter rule
8363  * @add: if true, add, if false, delete
8364  *
8365  * Add or delete a cloud filter for a specific flow spec.
8366  * Returns 0 if the filter were successfully added.
8367  **/
8368 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8369                               struct i40e_cloud_filter *filter, bool add)
8370 {
8371         struct i40e_aqc_cloud_filters_element_data cld_filter;
8372         struct i40e_pf *pf = vsi->back;
8373         int ret;
8374         static const u16 flag_table[128] = {
8375                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8376                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8377                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8378                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8379                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8380                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8381                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8382                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8383                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8384                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8385                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8386                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8387                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8388                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
8389         };
8390
8391         if (filter->flags >= ARRAY_SIZE(flag_table))
8392                 return -EIO;
8393
8394         memset(&cld_filter, 0, sizeof(cld_filter));
8395
8396         /* copy element needed to add cloud filter from filter */
8397         i40e_set_cld_element(filter, &cld_filter);
8398
8399         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8400                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8401                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8402
8403         if (filter->n_proto == ETH_P_IPV6)
8404                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8405                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8406         else
8407                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8408                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8409
8410         if (add)
8411                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8412                                                 &cld_filter, 1);
8413         else
8414                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8415                                                 &cld_filter, 1);
8416         if (ret)
8417                 dev_dbg(&pf->pdev->dev,
8418                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8419                         add ? "add" : "delete", filter->dst_port, ret,
8420                         pf->hw.aq.asq_last_status);
8421         else
8422                 dev_info(&pf->pdev->dev,
8423                          "%s cloud filter for VSI: %d\n",
8424                          add ? "Added" : "Deleted", filter->seid);
8425         return ret;
8426 }
8427
8428 /**
8429  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8430  * @vsi: pointer to VSI
8431  * @filter: cloud filter rule
8432  * @add: if true, add, if false, delete
8433  *
8434  * Add or delete a cloud filter for a specific flow spec using big buffer.
8435  * Returns 0 if the filter were successfully added.
8436  **/
8437 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8438                                       struct i40e_cloud_filter *filter,
8439                                       bool add)
8440 {
8441         struct i40e_aqc_cloud_filters_element_bb cld_filter;
8442         struct i40e_pf *pf = vsi->back;
8443         int ret;
8444
8445         /* Both (src/dst) valid mac_addr are not supported */
8446         if ((is_valid_ether_addr(filter->dst_mac) &&
8447              is_valid_ether_addr(filter->src_mac)) ||
8448             (is_multicast_ether_addr(filter->dst_mac) &&
8449              is_multicast_ether_addr(filter->src_mac)))
8450                 return -EOPNOTSUPP;
8451
8452         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8453          * ports are not supported via big buffer now.
8454          */
8455         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8456                 return -EOPNOTSUPP;
8457
8458         /* adding filter using src_port/src_ip is not supported at this stage */
8459         if (filter->src_port ||
8460             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8461             !ipv6_addr_any(&filter->ip.v6.src_ip6))
8462                 return -EOPNOTSUPP;
8463
8464         memset(&cld_filter, 0, sizeof(cld_filter));
8465
8466         /* copy element needed to add cloud filter from filter */
8467         i40e_set_cld_element(filter, &cld_filter.element);
8468
8469         if (is_valid_ether_addr(filter->dst_mac) ||
8470             is_valid_ether_addr(filter->src_mac) ||
8471             is_multicast_ether_addr(filter->dst_mac) ||
8472             is_multicast_ether_addr(filter->src_mac)) {
8473                 /* MAC + IP : unsupported mode */
8474                 if (filter->dst_ipv4)
8475                         return -EOPNOTSUPP;
8476
8477                 /* since we validated that L4 port must be valid before
8478                  * we get here, start with respective "flags" value
8479                  * and update if vlan is present or not
8480                  */
8481                 cld_filter.element.flags =
8482                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8483
8484                 if (filter->vlan_id) {
8485                         cld_filter.element.flags =
8486                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8487                 }
8488
8489         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8490                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8491                 cld_filter.element.flags =
8492                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8493                 if (filter->n_proto == ETH_P_IPV6)
8494                         cld_filter.element.flags |=
8495                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8496                 else
8497                         cld_filter.element.flags |=
8498                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8499         } else {
8500                 dev_err(&pf->pdev->dev,
8501                         "either mac or ip has to be valid for cloud filter\n");
8502                 return -EINVAL;
8503         }
8504
8505         /* Now copy L4 port in Byte 6..7 in general fields */
8506         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8507                                                 be16_to_cpu(filter->dst_port);
8508
8509         if (add) {
8510                 /* Validate current device switch mode, change if necessary */
8511                 ret = i40e_validate_and_set_switch_mode(vsi);
8512                 if (ret) {
8513                         dev_err(&pf->pdev->dev,
8514                                 "failed to set switch mode, ret %d\n",
8515                                 ret);
8516                         return ret;
8517                 }
8518
8519                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8520                                                    &cld_filter, 1);
8521         } else {
8522                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8523                                                    &cld_filter, 1);
8524         }
8525
8526         if (ret)
8527                 dev_dbg(&pf->pdev->dev,
8528                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8529                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8530         else
8531                 dev_info(&pf->pdev->dev,
8532                          "%s cloud filter for VSI: %d, L4 port: %d\n",
8533                          add ? "add" : "delete", filter->seid,
8534                          ntohs(filter->dst_port));
8535         return ret;
8536 }
8537
8538 /**
8539  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8540  * @vsi: Pointer to VSI
8541  * @f: Pointer to struct flow_cls_offload
8542  * @filter: Pointer to cloud filter structure
8543  *
8544  **/
8545 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8546                                  struct flow_cls_offload *f,
8547                                  struct i40e_cloud_filter *filter)
8548 {
8549         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8550         struct flow_dissector *dissector = rule->match.dissector;
8551         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8552         struct i40e_pf *pf = vsi->back;
8553         u8 field_flags = 0;
8554
8555         if (dissector->used_keys &
8556             ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
8557               BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
8558               BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8559               BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
8560               BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8561               BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8562               BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
8563               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8564                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",
8565                         dissector->used_keys);
8566                 return -EOPNOTSUPP;
8567         }
8568
8569         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8570                 struct flow_match_enc_keyid match;
8571
8572                 flow_rule_match_enc_keyid(rule, &match);
8573                 if (match.mask->keyid != 0)
8574                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8575
8576                 filter->tenant_id = be32_to_cpu(match.key->keyid);
8577         }
8578
8579         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8580                 struct flow_match_basic match;
8581
8582                 flow_rule_match_basic(rule, &match);
8583                 n_proto_key = ntohs(match.key->n_proto);
8584                 n_proto_mask = ntohs(match.mask->n_proto);
8585
8586                 if (n_proto_key == ETH_P_ALL) {
8587                         n_proto_key = 0;
8588                         n_proto_mask = 0;
8589                 }
8590                 filter->n_proto = n_proto_key & n_proto_mask;
8591                 filter->ip_proto = match.key->ip_proto;
8592         }
8593
8594         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8595                 struct flow_match_eth_addrs match;
8596
8597                 flow_rule_match_eth_addrs(rule, &match);
8598
8599                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8600                 if (!is_zero_ether_addr(match.mask->dst)) {
8601                         if (is_broadcast_ether_addr(match.mask->dst)) {
8602                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
8603                         } else {
8604                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8605                                         match.mask->dst);
8606                                 return -EIO;
8607                         }
8608                 }
8609
8610                 if (!is_zero_ether_addr(match.mask->src)) {
8611                         if (is_broadcast_ether_addr(match.mask->src)) {
8612                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
8613                         } else {
8614                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8615                                         match.mask->src);
8616                                 return -EIO;
8617                         }
8618                 }
8619                 ether_addr_copy(filter->dst_mac, match.key->dst);
8620                 ether_addr_copy(filter->src_mac, match.key->src);
8621         }
8622
8623         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8624                 struct flow_match_vlan match;
8625
8626                 flow_rule_match_vlan(rule, &match);
8627                 if (match.mask->vlan_id) {
8628                         if (match.mask->vlan_id == VLAN_VID_MASK) {
8629                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8630
8631                         } else {
8632                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8633                                         match.mask->vlan_id);
8634                                 return -EIO;
8635                         }
8636                 }
8637
8638                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8639         }
8640
8641         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8642                 struct flow_match_control match;
8643
8644                 flow_rule_match_control(rule, &match);
8645                 addr_type = match.key->addr_type;
8646         }
8647
8648         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8649                 struct flow_match_ipv4_addrs match;
8650
8651                 flow_rule_match_ipv4_addrs(rule, &match);
8652                 if (match.mask->dst) {
8653                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8654                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8655                         } else {
8656                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8657                                         &match.mask->dst);
8658                                 return -EIO;
8659                         }
8660                 }
8661
8662                 if (match.mask->src) {
8663                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
8664                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8665                         } else {
8666                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8667                                         &match.mask->src);
8668                                 return -EIO;
8669                         }
8670                 }
8671
8672                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8673                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8674                         return -EIO;
8675                 }
8676                 filter->dst_ipv4 = match.key->dst;
8677                 filter->src_ipv4 = match.key->src;
8678         }
8679
8680         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8681                 struct flow_match_ipv6_addrs match;
8682
8683                 flow_rule_match_ipv6_addrs(rule, &match);
8684
8685                 /* src and dest IPV6 address should not be LOOPBACK
8686                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
8687                  */
8688                 if (ipv6_addr_loopback(&match.key->dst) ||
8689                     ipv6_addr_loopback(&match.key->src)) {
8690                         dev_err(&pf->pdev->dev,
8691                                 "Bad ipv6, addr is LOOPBACK\n");
8692                         return -EIO;
8693                 }
8694                 if (!ipv6_addr_any(&match.mask->dst) ||
8695                     !ipv6_addr_any(&match.mask->src))
8696                         field_flags |= I40E_CLOUD_FIELD_IIP;
8697
8698                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8699                        sizeof(filter->src_ipv6));
8700                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8701                        sizeof(filter->dst_ipv6));
8702         }
8703
8704         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8705                 struct flow_match_ports match;
8706
8707                 flow_rule_match_ports(rule, &match);
8708                 if (match.mask->src) {
8709                         if (match.mask->src == cpu_to_be16(0xffff)) {
8710                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8711                         } else {
8712                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8713                                         be16_to_cpu(match.mask->src));
8714                                 return -EIO;
8715                         }
8716                 }
8717
8718                 if (match.mask->dst) {
8719                         if (match.mask->dst == cpu_to_be16(0xffff)) {
8720                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8721                         } else {
8722                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8723                                         be16_to_cpu(match.mask->dst));
8724                                 return -EIO;
8725                         }
8726                 }
8727
8728                 filter->dst_port = match.key->dst;
8729                 filter->src_port = match.key->src;
8730
8731                 switch (filter->ip_proto) {
8732                 case IPPROTO_TCP:
8733                 case IPPROTO_UDP:
8734                         break;
8735                 default:
8736                         dev_err(&pf->pdev->dev,
8737                                 "Only UDP and TCP transport are supported\n");
8738                         return -EINVAL;
8739                 }
8740         }
8741         filter->flags = field_flags;
8742         return 0;
8743 }
8744
8745 /**
8746  * i40e_handle_tclass: Forward to a traffic class on the device
8747  * @vsi: Pointer to VSI
8748  * @tc: traffic class index on the device
8749  * @filter: Pointer to cloud filter structure
8750  *
8751  **/
8752 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8753                               struct i40e_cloud_filter *filter)
8754 {
8755         struct i40e_channel *ch, *ch_tmp;
8756
8757         /* direct to a traffic class on the same device */
8758         if (tc == 0) {
8759                 filter->seid = vsi->seid;
8760                 return 0;
8761         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8762                 if (!filter->dst_port) {
8763                         dev_err(&vsi->back->pdev->dev,
8764                                 "Specify destination port to direct to traffic class that is not default\n");
8765                         return -EINVAL;
8766                 }
8767                 if (list_empty(&vsi->ch_list))
8768                         return -EINVAL;
8769                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8770                                          list) {
8771                         if (ch->seid == vsi->tc_seid_map[tc])
8772                                 filter->seid = ch->seid;
8773                 }
8774                 return 0;
8775         }
8776         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8777         return -EINVAL;
8778 }
8779
8780 /**
8781  * i40e_configure_clsflower - Configure tc flower filters
8782  * @vsi: Pointer to VSI
8783  * @cls_flower: Pointer to struct flow_cls_offload
8784  *
8785  **/
8786 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8787                                     struct flow_cls_offload *cls_flower)
8788 {
8789         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8790         struct i40e_cloud_filter *filter = NULL;
8791         struct i40e_pf *pf = vsi->back;
8792         int err = 0;
8793
8794         if (tc < 0) {
8795                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8796                 return -EOPNOTSUPP;
8797         }
8798
8799         if (!tc) {
8800                 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8801                 return -EINVAL;
8802         }
8803
8804         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8805             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8806                 return -EBUSY;
8807
8808         if (pf->fdir_pf_active_filters ||
8809             (!hlist_empty(&pf->fdir_filter_list))) {
8810                 dev_err(&vsi->back->pdev->dev,
8811                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8812                 return -EINVAL;
8813         }
8814
8815         if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
8816                 dev_err(&vsi->back->pdev->dev,
8817                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8818                 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);
8819                 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);
8820         }
8821
8822         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8823         if (!filter)
8824                 return -ENOMEM;
8825
8826         filter->cookie = cls_flower->cookie;
8827
8828         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8829         if (err < 0)
8830                 goto err;
8831
8832         err = i40e_handle_tclass(vsi, tc, filter);
8833         if (err < 0)
8834                 goto err;
8835
8836         /* Add cloud filter */
8837         if (filter->dst_port)
8838                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8839         else
8840                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8841
8842         if (err) {
8843                 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8844                         err);
8845                 goto err;
8846         }
8847
8848         /* add filter to the ordered list */
8849         INIT_HLIST_NODE(&filter->cloud_node);
8850
8851         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8852
8853         pf->num_cloud_filters++;
8854
8855         return err;
8856 err:
8857         kfree(filter);
8858         return err;
8859 }
8860
8861 /**
8862  * i40e_find_cloud_filter - Find the could filter in the list
8863  * @vsi: Pointer to VSI
8864  * @cookie: filter specific cookie
8865  *
8866  **/
8867 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8868                                                         unsigned long *cookie)
8869 {
8870         struct i40e_cloud_filter *filter = NULL;
8871         struct hlist_node *node2;
8872
8873         hlist_for_each_entry_safe(filter, node2,
8874                                   &vsi->back->cloud_filter_list, cloud_node)
8875                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8876                         return filter;
8877         return NULL;
8878 }
8879
8880 /**
8881  * i40e_delete_clsflower - Remove tc flower filters
8882  * @vsi: Pointer to VSI
8883  * @cls_flower: Pointer to struct flow_cls_offload
8884  *
8885  **/
8886 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8887                                  struct flow_cls_offload *cls_flower)
8888 {
8889         struct i40e_cloud_filter *filter = NULL;
8890         struct i40e_pf *pf = vsi->back;
8891         int err = 0;
8892
8893         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8894
8895         if (!filter)
8896                 return -EINVAL;
8897
8898         hash_del(&filter->cloud_node);
8899
8900         if (filter->dst_port)
8901                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8902         else
8903                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8904
8905         kfree(filter);
8906         if (err) {
8907                 dev_err(&pf->pdev->dev,
8908                         "Failed to delete cloud filter, err %pe\n",
8909                         ERR_PTR(err));
8910                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8911         }
8912
8913         pf->num_cloud_filters--;
8914         if (!pf->num_cloud_filters)
8915                 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
8916                     !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
8917                         set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
8918                         clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
8919                         clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
8920                 }
8921         return 0;
8922 }
8923
8924 /**
8925  * i40e_setup_tc_cls_flower - flower classifier offloads
8926  * @np: net device to configure
8927  * @cls_flower: offload data
8928  **/
8929 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8930                                     struct flow_cls_offload *cls_flower)
8931 {
8932         struct i40e_vsi *vsi = np->vsi;
8933
8934         switch (cls_flower->command) {
8935         case FLOW_CLS_REPLACE:
8936                 return i40e_configure_clsflower(vsi, cls_flower);
8937         case FLOW_CLS_DESTROY:
8938                 return i40e_delete_clsflower(vsi, cls_flower);
8939         case FLOW_CLS_STATS:
8940                 return -EOPNOTSUPP;
8941         default:
8942                 return -EOPNOTSUPP;
8943         }
8944 }
8945
8946 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8947                                   void *cb_priv)
8948 {
8949         struct i40e_netdev_priv *np = cb_priv;
8950
8951         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8952                 return -EOPNOTSUPP;
8953
8954         switch (type) {
8955         case TC_SETUP_CLSFLOWER:
8956                 return i40e_setup_tc_cls_flower(np, type_data);
8957
8958         default:
8959                 return -EOPNOTSUPP;
8960         }
8961 }
8962
8963 static LIST_HEAD(i40e_block_cb_list);
8964
8965 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8966                            void *type_data)
8967 {
8968         struct i40e_netdev_priv *np = netdev_priv(netdev);
8969
8970         switch (type) {
8971         case TC_SETUP_QDISC_MQPRIO:
8972                 return i40e_setup_tc(netdev, type_data);
8973         case TC_SETUP_BLOCK:
8974                 return flow_block_cb_setup_simple(type_data,
8975                                                   &i40e_block_cb_list,
8976                                                   i40e_setup_tc_block_cb,
8977                                                   np, np, true);
8978         default:
8979                 return -EOPNOTSUPP;
8980         }
8981 }
8982
8983 /**
8984  * i40e_open - Called when a network interface is made active
8985  * @netdev: network interface device structure
8986  *
8987  * The open entry point is called when a network interface is made
8988  * active by the system (IFF_UP).  At this point all resources needed
8989  * for transmit and receive operations are allocated, the interrupt
8990  * handler is registered with the OS, the netdev watchdog subtask is
8991  * enabled, and the stack is notified that the interface is ready.
8992  *
8993  * Returns 0 on success, negative value on failure
8994  **/
8995 int i40e_open(struct net_device *netdev)
8996 {
8997         struct i40e_netdev_priv *np = netdev_priv(netdev);
8998         struct i40e_vsi *vsi = np->vsi;
8999         struct i40e_pf *pf = vsi->back;
9000         int err;
9001
9002         /* disallow open during test or if eeprom is broken */
9003         if (test_bit(__I40E_TESTING, pf->state) ||
9004             test_bit(__I40E_BAD_EEPROM, pf->state))
9005                 return -EBUSY;
9006
9007         netif_carrier_off(netdev);
9008
9009         if (i40e_force_link_state(pf, true))
9010                 return -EAGAIN;
9011
9012         err = i40e_vsi_open(vsi);
9013         if (err)
9014                 return err;
9015
9016         /* configure global TSO hardware offload settings */
9017         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
9018                                                        TCP_FLAG_FIN) >> 16);
9019         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
9020                                                        TCP_FLAG_FIN |
9021                                                        TCP_FLAG_CWR) >> 16);
9022         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
9023         udp_tunnel_get_rx_info(netdev);
9024
9025         return 0;
9026 }
9027
9028 /**
9029  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9030  * @vsi: vsi structure
9031  *
9032  * This updates netdev's number of tx/rx queues
9033  *
9034  * Returns status of setting tx/rx queues
9035  **/
9036 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9037 {
9038         int ret;
9039
9040         ret = netif_set_real_num_rx_queues(vsi->netdev,
9041                                            vsi->num_queue_pairs);
9042         if (ret)
9043                 return ret;
9044
9045         return netif_set_real_num_tx_queues(vsi->netdev,
9046                                             vsi->num_queue_pairs);
9047 }
9048
9049 /**
9050  * i40e_vsi_open -
9051  * @vsi: the VSI to open
9052  *
9053  * Finish initialization of the VSI.
9054  *
9055  * Returns 0 on success, negative value on failure
9056  *
9057  * Note: expects to be called while under rtnl_lock()
9058  **/
9059 int i40e_vsi_open(struct i40e_vsi *vsi)
9060 {
9061         struct i40e_pf *pf = vsi->back;
9062         char int_name[I40E_INT_NAME_STR_LEN];
9063         int err;
9064
9065         /* allocate descriptors */
9066         err = i40e_vsi_setup_tx_resources(vsi);
9067         if (err)
9068                 goto err_setup_tx;
9069         err = i40e_vsi_setup_rx_resources(vsi);
9070         if (err)
9071                 goto err_setup_rx;
9072
9073         err = i40e_vsi_configure(vsi);
9074         if (err)
9075                 goto err_setup_rx;
9076
9077         if (vsi->netdev) {
9078                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9079                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9080                 err = i40e_vsi_request_irq(vsi, int_name);
9081                 if (err)
9082                         goto err_setup_rx;
9083
9084                 /* Notify the stack of the actual queue counts. */
9085                 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9086                 if (err)
9087                         goto err_set_queues;
9088
9089         } else if (vsi->type == I40E_VSI_FDIR) {
9090                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9091                          dev_driver_string(&pf->pdev->dev),
9092                          dev_name(&pf->pdev->dev));
9093                 err = i40e_vsi_request_irq(vsi, int_name);
9094                 if (err)
9095                         goto err_setup_rx;
9096
9097         } else {
9098                 err = -EINVAL;
9099                 goto err_setup_rx;
9100         }
9101
9102         err = i40e_up_complete(vsi);
9103         if (err)
9104                 goto err_up_complete;
9105
9106         return 0;
9107
9108 err_up_complete:
9109         i40e_down(vsi);
9110 err_set_queues:
9111         i40e_vsi_free_irq(vsi);
9112 err_setup_rx:
9113         i40e_vsi_free_rx_resources(vsi);
9114 err_setup_tx:
9115         i40e_vsi_free_tx_resources(vsi);
9116         if (vsi == pf->vsi[pf->lan_vsi])
9117                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9118
9119         return err;
9120 }
9121
9122 /**
9123  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9124  * @pf: Pointer to PF
9125  *
9126  * This function destroys the hlist where all the Flow Director
9127  * filters were saved.
9128  **/
9129 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9130 {
9131         struct i40e_fdir_filter *filter;
9132         struct i40e_flex_pit *pit_entry, *tmp;
9133         struct hlist_node *node2;
9134
9135         hlist_for_each_entry_safe(filter, node2,
9136                                   &pf->fdir_filter_list, fdir_node) {
9137                 hlist_del(&filter->fdir_node);
9138                 kfree(filter);
9139         }
9140
9141         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9142                 list_del(&pit_entry->list);
9143                 kfree(pit_entry);
9144         }
9145         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9146
9147         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9148                 list_del(&pit_entry->list);
9149                 kfree(pit_entry);
9150         }
9151         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9152
9153         pf->fdir_pf_active_filters = 0;
9154         i40e_reset_fdir_filter_cnt(pf);
9155
9156         /* Reprogram the default input set for TCP/IPv4 */
9157         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9158                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9159                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9160
9161         /* Reprogram the default input set for TCP/IPv6 */
9162         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9163                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9164                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9165
9166         /* Reprogram the default input set for UDP/IPv4 */
9167         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9168                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9169                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9170
9171         /* Reprogram the default input set for UDP/IPv6 */
9172         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9173                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9174                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9175
9176         /* Reprogram the default input set for SCTP/IPv4 */
9177         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9178                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9179                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9180
9181         /* Reprogram the default input set for SCTP/IPv6 */
9182         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9183                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9184                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9185
9186         /* Reprogram the default input set for Other/IPv4 */
9187         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9188                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9189
9190         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9191                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9192
9193         /* Reprogram the default input set for Other/IPv6 */
9194         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9195                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9196
9197         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9198                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9199 }
9200
9201 /**
9202  * i40e_cloud_filter_exit - Cleans up the cloud filters
9203  * @pf: Pointer to PF
9204  *
9205  * This function destroys the hlist where all the cloud filters
9206  * were saved.
9207  **/
9208 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9209 {
9210         struct i40e_cloud_filter *cfilter;
9211         struct hlist_node *node;
9212
9213         hlist_for_each_entry_safe(cfilter, node,
9214                                   &pf->cloud_filter_list, cloud_node) {
9215                 hlist_del(&cfilter->cloud_node);
9216                 kfree(cfilter);
9217         }
9218         pf->num_cloud_filters = 0;
9219
9220         if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
9221             !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
9222                 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
9223                 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
9224                 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
9225         }
9226 }
9227
9228 /**
9229  * i40e_close - Disables a network interface
9230  * @netdev: network interface device structure
9231  *
9232  * The close entry point is called when an interface is de-activated
9233  * by the OS.  The hardware is still under the driver's control, but
9234  * this netdev interface is disabled.
9235  *
9236  * Returns 0, this is not allowed to fail
9237  **/
9238 int i40e_close(struct net_device *netdev)
9239 {
9240         struct i40e_netdev_priv *np = netdev_priv(netdev);
9241         struct i40e_vsi *vsi = np->vsi;
9242
9243         i40e_vsi_close(vsi);
9244
9245         return 0;
9246 }
9247
9248 /**
9249  * i40e_do_reset - Start a PF or Core Reset sequence
9250  * @pf: board private structure
9251  * @reset_flags: which reset is requested
9252  * @lock_acquired: indicates whether or not the lock has been acquired
9253  * before this function was called.
9254  *
9255  * The essential difference in resets is that the PF Reset
9256  * doesn't clear the packet buffers, doesn't reset the PE
9257  * firmware, and doesn't bother the other PFs on the chip.
9258  **/
9259 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9260 {
9261         struct i40e_vsi *vsi;
9262         u32 val;
9263         int i;
9264
9265         /* do the biggest reset indicated */
9266         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9267
9268                 /* Request a Global Reset
9269                  *
9270                  * This will start the chip's countdown to the actual full
9271                  * chip reset event, and a warning interrupt to be sent
9272                  * to all PFs, including the requestor.  Our handler
9273                  * for the warning interrupt will deal with the shutdown
9274                  * and recovery of the switch setup.
9275                  */
9276                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9277                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9278                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9279                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9280
9281         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9282
9283                 /* Request a Core Reset
9284                  *
9285                  * Same as Global Reset, except does *not* include the MAC/PHY
9286                  */
9287                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9288                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9289                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
9290                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9291                 i40e_flush(&pf->hw);
9292
9293         } else if (reset_flags & I40E_PF_RESET_FLAG) {
9294
9295                 /* Request a PF Reset
9296                  *
9297                  * Resets only the PF-specific registers
9298                  *
9299                  * This goes directly to the tear-down and rebuild of
9300                  * the switch, since we need to do all the recovery as
9301                  * for the Core Reset.
9302                  */
9303                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
9304                 i40e_handle_reset_warning(pf, lock_acquired);
9305
9306         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9307                 /* Request a PF Reset
9308                  *
9309                  * Resets PF and reinitializes PFs VSI.
9310                  */
9311                 i40e_prep_for_reset(pf);
9312                 i40e_reset_and_rebuild(pf, true, lock_acquired);
9313                 dev_info(&pf->pdev->dev,
9314                          test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
9315                          "FW LLDP is disabled\n" :
9316                          "FW LLDP is enabled\n");
9317
9318         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9319                 /* Find the VSI(s) that requested a re-init */
9320                 dev_info(&pf->pdev->dev, "VSI reinit requested\n");
9321
9322                 i40e_pf_for_each_vsi(pf, i, vsi) {
9323                         if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9324                                                vsi->state))
9325                                 i40e_vsi_reinit_locked(vsi);
9326                 }
9327         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9328                 /* Find the VSI(s) that needs to be brought down */
9329                 dev_info(&pf->pdev->dev, "VSI down requested\n");
9330
9331                 i40e_pf_for_each_vsi(pf, i, vsi) {
9332                         if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9333                                                vsi->state)) {
9334                                 set_bit(__I40E_VSI_DOWN, vsi->state);
9335                                 i40e_down(vsi);
9336                         }
9337                 }
9338         } else {
9339                 dev_info(&pf->pdev->dev,
9340                          "bad reset request 0x%08x\n", reset_flags);
9341         }
9342 }
9343
9344 #ifdef CONFIG_I40E_DCB
9345 /**
9346  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9347  * @pf: board private structure
9348  * @old_cfg: current DCB config
9349  * @new_cfg: new DCB config
9350  **/
9351 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9352                             struct i40e_dcbx_config *old_cfg,
9353                             struct i40e_dcbx_config *new_cfg)
9354 {
9355         bool need_reconfig = false;
9356
9357         /* Check if ETS configuration has changed */
9358         if (memcmp(&new_cfg->etscfg,
9359                    &old_cfg->etscfg,
9360                    sizeof(new_cfg->etscfg))) {
9361                 /* If Priority Table has changed reconfig is needed */
9362                 if (memcmp(&new_cfg->etscfg.prioritytable,
9363                            &old_cfg->etscfg.prioritytable,
9364                            sizeof(new_cfg->etscfg.prioritytable))) {
9365                         need_reconfig = true;
9366                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9367                 }
9368
9369                 if (memcmp(&new_cfg->etscfg.tcbwtable,
9370                            &old_cfg->etscfg.tcbwtable,
9371                            sizeof(new_cfg->etscfg.tcbwtable)))
9372                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9373
9374                 if (memcmp(&new_cfg->etscfg.tsatable,
9375                            &old_cfg->etscfg.tsatable,
9376                            sizeof(new_cfg->etscfg.tsatable)))
9377                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9378         }
9379
9380         /* Check if PFC configuration has changed */
9381         if (memcmp(&new_cfg->pfc,
9382                    &old_cfg->pfc,
9383                    sizeof(new_cfg->pfc))) {
9384                 need_reconfig = true;
9385                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9386         }
9387
9388         /* Check if APP Table has changed */
9389         if (memcmp(&new_cfg->app,
9390                    &old_cfg->app,
9391                    sizeof(new_cfg->app))) {
9392                 need_reconfig = true;
9393                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9394         }
9395
9396         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9397         return need_reconfig;
9398 }
9399
9400 /**
9401  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9402  * @pf: board private structure
9403  * @e: event info posted on ARQ
9404  **/
9405 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9406                                   struct i40e_arq_event_info *e)
9407 {
9408         struct i40e_aqc_lldp_get_mib *mib =
9409                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9410         struct i40e_hw *hw = &pf->hw;
9411         struct i40e_dcbx_config tmp_dcbx_cfg;
9412         bool need_reconfig = false;
9413         int ret = 0;
9414         u8 type;
9415
9416         /* X710-T*L 2.5G and 5G speeds don't support DCB */
9417         if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9418             (hw->phy.link_info.link_speed &
9419              ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9420              !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9421                 /* let firmware decide if the DCB should be disabled */
9422                 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9423
9424         /* Not DCB capable or capability disabled */
9425         if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9426                 return ret;
9427
9428         /* Ignore if event is not for Nearest Bridge */
9429         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9430                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9431         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9432         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9433                 return ret;
9434
9435         /* Check MIB Type and return if event for Remote MIB update */
9436         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9437         dev_dbg(&pf->pdev->dev,
9438                 "LLDP event mib type %s\n", type ? "remote" : "local");
9439         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9440                 /* Update the remote cached instance and return */
9441                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9442                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9443                                 &hw->remote_dcbx_config);
9444                 goto exit;
9445         }
9446
9447         /* Store the old configuration */
9448         tmp_dcbx_cfg = hw->local_dcbx_config;
9449
9450         /* Reset the old DCBx configuration data */
9451         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9452         /* Get updated DCBX data from firmware */
9453         ret = i40e_get_dcb_config(&pf->hw);
9454         if (ret) {
9455                 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9456                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9457                     (hw->phy.link_info.link_speed &
9458                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9459                         dev_warn(&pf->pdev->dev,
9460                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9461                         clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9462                 } else {
9463                         dev_info(&pf->pdev->dev,
9464                                  "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9465                                  ERR_PTR(ret),
9466                                  i40e_aq_str(&pf->hw,
9467                                              pf->hw.aq.asq_last_status));
9468                 }
9469                 goto exit;
9470         }
9471
9472         /* No change detected in DCBX configs */
9473         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9474                     sizeof(tmp_dcbx_cfg))) {
9475                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9476                 goto exit;
9477         }
9478
9479         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9480                                                &hw->local_dcbx_config);
9481
9482         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9483
9484         if (!need_reconfig)
9485                 goto exit;
9486
9487         /* Enable DCB tagging only when more than one TC */
9488         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9489                 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
9490         else
9491                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9492
9493         set_bit(__I40E_PORT_SUSPENDED, pf->state);
9494         /* Reconfiguration needed quiesce all VSIs */
9495         i40e_pf_quiesce_all_vsi(pf);
9496
9497         /* Changes in configuration update VEB/VSI */
9498         i40e_dcb_reconfigure(pf);
9499
9500         ret = i40e_resume_port_tx(pf);
9501
9502         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9503         /* In case of error no point in resuming VSIs */
9504         if (ret)
9505                 goto exit;
9506
9507         /* Wait for the PF's queues to be disabled */
9508         ret = i40e_pf_wait_queues_disabled(pf);
9509         if (ret) {
9510                 /* Schedule PF reset to recover */
9511                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9512                 i40e_service_event_schedule(pf);
9513         } else {
9514                 i40e_pf_unquiesce_all_vsi(pf);
9515                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9516                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9517         }
9518
9519 exit:
9520         return ret;
9521 }
9522 #endif /* CONFIG_I40E_DCB */
9523
9524 /**
9525  * i40e_do_reset_safe - Protected reset path for userland calls.
9526  * @pf: board private structure
9527  * @reset_flags: which reset is requested
9528  *
9529  **/
9530 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9531 {
9532         rtnl_lock();
9533         i40e_do_reset(pf, reset_flags, true);
9534         rtnl_unlock();
9535 }
9536
9537 /**
9538  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9539  * @pf: board private structure
9540  * @e: event info posted on ARQ
9541  *
9542  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9543  * and VF queues
9544  **/
9545 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9546                                            struct i40e_arq_event_info *e)
9547 {
9548         struct i40e_aqc_lan_overflow *data =
9549                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9550         u32 queue = le32_to_cpu(data->prtdcb_rupto);
9551         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9552         struct i40e_hw *hw = &pf->hw;
9553         struct i40e_vf *vf;
9554         u16 vf_id;
9555
9556         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9557                 queue, qtx_ctl);
9558
9559         if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=
9560             I40E_QTX_CTL_VF_QUEUE)
9561                 return;
9562
9563         /* Queue belongs to VF, find the VF and issue VF reset */
9564         vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);
9565         vf_id -= hw->func_caps.vf_base_id;
9566         vf = &pf->vf[vf_id];
9567         i40e_vc_notify_vf_reset(vf);
9568         /* Allow VF to process pending reset notification */
9569         msleep(20);
9570         i40e_reset_vf(vf, false);
9571 }
9572
9573 /**
9574  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9575  * @pf: board private structure
9576  **/
9577 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9578 {
9579         u32 val, fcnt_prog;
9580
9581         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9582         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9583         return fcnt_prog;
9584 }
9585
9586 /**
9587  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9588  * @pf: board private structure
9589  **/
9590 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9591 {
9592         u32 val, fcnt_prog;
9593
9594         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9595         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9596                     FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);
9597         return fcnt_prog;
9598 }
9599
9600 /**
9601  * i40e_get_global_fd_count - Get total FD filters programmed on device
9602  * @pf: board private structure
9603  **/
9604 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9605 {
9606         u32 val, fcnt_prog;
9607
9608         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9609         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9610                     FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);
9611         return fcnt_prog;
9612 }
9613
9614 /**
9615  * i40e_reenable_fdir_sb - Restore FDir SB capability
9616  * @pf: board private structure
9617  **/
9618 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9619 {
9620         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9621                 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
9622                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9623                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9624 }
9625
9626 /**
9627  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9628  * @pf: board private structure
9629  **/
9630 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9631 {
9632         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9633                 /* ATR uses the same filtering logic as SB rules. It only
9634                  * functions properly if the input set mask is at the default
9635                  * settings. It is safe to restore the default input set
9636                  * because there are no active TCPv4 filter rules.
9637                  */
9638                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9639                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9640                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9641
9642                 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
9643                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9644                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9645         }
9646 }
9647
9648 /**
9649  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9650  * @pf: board private structure
9651  * @filter: FDir filter to remove
9652  */
9653 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9654                                        struct i40e_fdir_filter *filter)
9655 {
9656         /* Update counters */
9657         pf->fdir_pf_active_filters--;
9658         pf->fd_inv = 0;
9659
9660         switch (filter->flow_type) {
9661         case TCP_V4_FLOW:
9662                 pf->fd_tcp4_filter_cnt--;
9663                 break;
9664         case UDP_V4_FLOW:
9665                 pf->fd_udp4_filter_cnt--;
9666                 break;
9667         case SCTP_V4_FLOW:
9668                 pf->fd_sctp4_filter_cnt--;
9669                 break;
9670         case TCP_V6_FLOW:
9671                 pf->fd_tcp6_filter_cnt--;
9672                 break;
9673         case UDP_V6_FLOW:
9674                 pf->fd_udp6_filter_cnt--;
9675                 break;
9676         case SCTP_V6_FLOW:
9677                 pf->fd_udp6_filter_cnt--;
9678                 break;
9679         case IP_USER_FLOW:
9680                 switch (filter->ipl4_proto) {
9681                 case IPPROTO_TCP:
9682                         pf->fd_tcp4_filter_cnt--;
9683                         break;
9684                 case IPPROTO_UDP:
9685                         pf->fd_udp4_filter_cnt--;
9686                         break;
9687                 case IPPROTO_SCTP:
9688                         pf->fd_sctp4_filter_cnt--;
9689                         break;
9690                 case IPPROTO_IP:
9691                         pf->fd_ip4_filter_cnt--;
9692                         break;
9693                 }
9694                 break;
9695         case IPV6_USER_FLOW:
9696                 switch (filter->ipl4_proto) {
9697                 case IPPROTO_TCP:
9698                         pf->fd_tcp6_filter_cnt--;
9699                         break;
9700                 case IPPROTO_UDP:
9701                         pf->fd_udp6_filter_cnt--;
9702                         break;
9703                 case IPPROTO_SCTP:
9704                         pf->fd_sctp6_filter_cnt--;
9705                         break;
9706                 case IPPROTO_IP:
9707                         pf->fd_ip6_filter_cnt--;
9708                         break;
9709                 }
9710                 break;
9711         }
9712
9713         /* Remove the filter from the list and free memory */
9714         hlist_del(&filter->fdir_node);
9715         kfree(filter);
9716 }
9717
9718 /**
9719  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9720  * @pf: board private structure
9721  **/
9722 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9723 {
9724         struct i40e_fdir_filter *filter;
9725         u32 fcnt_prog, fcnt_avail;
9726         struct hlist_node *node;
9727
9728         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9729                 return;
9730
9731         /* Check if we have enough room to re-enable FDir SB capability. */
9732         fcnt_prog = i40e_get_global_fd_count(pf);
9733         fcnt_avail = pf->fdir_pf_filter_count;
9734         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9735             (pf->fd_add_err == 0) ||
9736             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9737                 i40e_reenable_fdir_sb(pf);
9738
9739         /* We should wait for even more space before re-enabling ATR.
9740          * Additionally, we cannot enable ATR as long as we still have TCP SB
9741          * rules active.
9742          */
9743         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9744             pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9745                 i40e_reenable_fdir_atr(pf);
9746
9747         /* if hw had a problem adding a filter, delete it */
9748         if (pf->fd_inv > 0) {
9749                 hlist_for_each_entry_safe(filter, node,
9750                                           &pf->fdir_filter_list, fdir_node)
9751                         if (filter->fd_id == pf->fd_inv)
9752                                 i40e_delete_invalid_filter(pf, filter);
9753         }
9754 }
9755
9756 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9757 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9758 /**
9759  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9760  * @pf: board private structure
9761  **/
9762 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9763 {
9764         unsigned long min_flush_time;
9765         int flush_wait_retry = 50;
9766         bool disable_atr = false;
9767         int fd_room;
9768         int reg;
9769
9770         if (!time_after(jiffies, pf->fd_flush_timestamp +
9771                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9772                 return;
9773
9774         /* If the flush is happening too quick and we have mostly SB rules we
9775          * should not re-enable ATR for some time.
9776          */
9777         min_flush_time = pf->fd_flush_timestamp +
9778                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9779         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9780
9781         if (!(time_after(jiffies, min_flush_time)) &&
9782             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9783                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9784                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9785                 disable_atr = true;
9786         }
9787
9788         pf->fd_flush_timestamp = jiffies;
9789         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9790         /* flush all filters */
9791         wr32(&pf->hw, I40E_PFQF_CTL_1,
9792              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9793         i40e_flush(&pf->hw);
9794         pf->fd_flush_cnt++;
9795         pf->fd_add_err = 0;
9796         do {
9797                 /* Check FD flush status every 5-6msec */
9798                 usleep_range(5000, 6000);
9799                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9800                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9801                         break;
9802         } while (flush_wait_retry--);
9803         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9804                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9805         } else {
9806                 /* replay sideband filters */
9807                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9808                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9809                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9810                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9811                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9812                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9813         }
9814 }
9815
9816 /**
9817  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9818  * @pf: board private structure
9819  **/
9820 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9821 {
9822         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9823 }
9824
9825 /**
9826  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9827  * @pf: board private structure
9828  **/
9829 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9830 {
9831
9832         /* if interface is down do nothing */
9833         if (test_bit(__I40E_DOWN, pf->state))
9834                 return;
9835
9836         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9837                 i40e_fdir_flush_and_replay(pf);
9838
9839         i40e_fdir_check_and_reenable(pf);
9840
9841 }
9842
9843 /**
9844  * i40e_vsi_link_event - notify VSI of a link event
9845  * @vsi: vsi to be notified
9846  * @link_up: link up or down
9847  **/
9848 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9849 {
9850         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9851                 return;
9852
9853         switch (vsi->type) {
9854         case I40E_VSI_MAIN:
9855                 if (!vsi->netdev || !vsi->netdev_registered)
9856                         break;
9857
9858                 if (link_up) {
9859                         netif_carrier_on(vsi->netdev);
9860                         netif_tx_wake_all_queues(vsi->netdev);
9861                 } else {
9862                         netif_carrier_off(vsi->netdev);
9863                         netif_tx_stop_all_queues(vsi->netdev);
9864                 }
9865                 break;
9866
9867         case I40E_VSI_SRIOV:
9868         case I40E_VSI_VMDQ2:
9869         case I40E_VSI_CTRL:
9870         case I40E_VSI_IWARP:
9871         case I40E_VSI_MIRROR:
9872         default:
9873                 /* there is no notification for other VSIs */
9874                 break;
9875         }
9876 }
9877
9878 /**
9879  * i40e_veb_link_event - notify elements on the veb of a link event
9880  * @veb: veb to be notified
9881  * @link_up: link up or down
9882  **/
9883 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9884 {
9885         struct i40e_vsi *vsi;
9886         struct i40e_pf *pf;
9887         int i;
9888
9889         if (!veb || !veb->pf)
9890                 return;
9891         pf = veb->pf;
9892
9893         /* Send link event to contained VSIs */
9894         i40e_pf_for_each_vsi(pf, i, vsi)
9895                 if (vsi->uplink_seid == veb->seid)
9896                         i40e_vsi_link_event(vsi, link_up);
9897 }
9898
9899 /**
9900  * i40e_link_event - Update netif_carrier status
9901  * @pf: board private structure
9902  **/
9903 static void i40e_link_event(struct i40e_pf *pf)
9904 {
9905         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9906         u8 new_link_speed, old_link_speed;
9907         bool new_link, old_link;
9908         int status;
9909 #ifdef CONFIG_I40E_DCB
9910         int err;
9911 #endif /* CONFIG_I40E_DCB */
9912
9913         /* set this to force the get_link_status call to refresh state */
9914         pf->hw.phy.get_link_info = true;
9915         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9916         status = i40e_get_link_status(&pf->hw, &new_link);
9917
9918         /* On success, disable temp link polling */
9919         if (status == 0) {
9920                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9921         } else {
9922                 /* Enable link polling temporarily until i40e_get_link_status
9923                  * returns 0
9924                  */
9925                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9926                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9927                         status);
9928                 return;
9929         }
9930
9931         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9932         new_link_speed = pf->hw.phy.link_info.link_speed;
9933
9934         if (new_link == old_link &&
9935             new_link_speed == old_link_speed &&
9936             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9937              new_link == netif_carrier_ok(vsi->netdev)))
9938                 return;
9939
9940         i40e_print_link_message(vsi, new_link);
9941
9942         /* Notify the base of the switch tree connected to
9943          * the link.  Floating VEBs are not notified.
9944          */
9945         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9946                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9947         else
9948                 i40e_vsi_link_event(vsi, new_link);
9949
9950         if (pf->vf)
9951                 i40e_vc_notify_link_state(pf);
9952
9953         if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
9954                 i40e_ptp_set_increment(pf);
9955 #ifdef CONFIG_I40E_DCB
9956         if (new_link == old_link)
9957                 return;
9958         /* Not SW DCB so firmware will take care of default settings */
9959         if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9960                 return;
9961
9962         /* We cover here only link down, as after link up in case of SW DCB
9963          * SW LLDP agent will take care of setting it up
9964          */
9965         if (!new_link) {
9966                 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9967                 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9968                 err = i40e_dcb_sw_default_config(pf);
9969                 if (err) {
9970                         clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9971                         clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9972                 } else {
9973                         pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9974                                        DCB_CAP_DCBX_VER_IEEE;
9975                         set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9976                         clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9977                 }
9978         }
9979 #endif /* CONFIG_I40E_DCB */
9980 }
9981
9982 /**
9983  * i40e_watchdog_subtask - periodic checks not using event driven response
9984  * @pf: board private structure
9985  **/
9986 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9987 {
9988         struct i40e_vsi *vsi;
9989         struct i40e_veb *veb;
9990         int i;
9991
9992         /* if interface is down do nothing */
9993         if (test_bit(__I40E_DOWN, pf->state) ||
9994             test_bit(__I40E_CONFIG_BUSY, pf->state))
9995                 return;
9996
9997         /* make sure we don't do these things too often */
9998         if (time_before(jiffies, (pf->service_timer_previous +
9999                                   pf->service_timer_period)))
10000                 return;
10001         pf->service_timer_previous = jiffies;
10002
10003         if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
10004             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
10005                 i40e_link_event(pf);
10006
10007         /* Update the stats for active netdevs so the network stack
10008          * can look at updated numbers whenever it cares to
10009          */
10010         i40e_pf_for_each_vsi(pf, i, vsi)
10011                 if (vsi->netdev)
10012                         i40e_update_stats(vsi);
10013
10014         if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
10015                 /* Update the stats for the active switching components */
10016                 i40e_pf_for_each_veb(pf, i, veb)
10017                         i40e_update_veb_stats(veb);
10018         }
10019
10020         i40e_ptp_rx_hang(pf);
10021         i40e_ptp_tx_hang(pf);
10022 }
10023
10024 /**
10025  * i40e_reset_subtask - Set up for resetting the device and driver
10026  * @pf: board private structure
10027  **/
10028 static void i40e_reset_subtask(struct i40e_pf *pf)
10029 {
10030         u32 reset_flags = 0;
10031
10032         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10033                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10034                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10035         }
10036         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10037                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10038                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10039         }
10040         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10041                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10042                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10043         }
10044         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10045                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10046                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10047         }
10048         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10049                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10050                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10051         }
10052
10053         /* If there's a recovery already waiting, it takes
10054          * precedence before starting a new reset sequence.
10055          */
10056         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10057                 i40e_prep_for_reset(pf);
10058                 i40e_reset(pf);
10059                 i40e_rebuild(pf, false, false);
10060         }
10061
10062         /* If we're already down or resetting, just bail */
10063         if (reset_flags &&
10064             !test_bit(__I40E_DOWN, pf->state) &&
10065             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10066                 i40e_do_reset(pf, reset_flags, false);
10067         }
10068 }
10069
10070 /**
10071  * i40e_handle_link_event - Handle link event
10072  * @pf: board private structure
10073  * @e: event info posted on ARQ
10074  **/
10075 static void i40e_handle_link_event(struct i40e_pf *pf,
10076                                    struct i40e_arq_event_info *e)
10077 {
10078         struct i40e_aqc_get_link_status *status =
10079                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10080
10081         /* Do a new status request to re-enable LSE reporting
10082          * and load new status information into the hw struct
10083          * This completely ignores any state information
10084          * in the ARQ event info, instead choosing to always
10085          * issue the AQ update link status command.
10086          */
10087         i40e_link_event(pf);
10088
10089         /* Check if module meets thermal requirements */
10090         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10091                 dev_err(&pf->pdev->dev,
10092                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10093                 dev_err(&pf->pdev->dev,
10094                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10095         } else {
10096                 /* check for unqualified module, if link is down, suppress
10097                  * the message if link was forced to be down.
10098                  */
10099                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10100                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10101                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
10102                     (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
10103                         dev_err(&pf->pdev->dev,
10104                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10105                         dev_err(&pf->pdev->dev,
10106                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10107                 }
10108         }
10109 }
10110
10111 /**
10112  * i40e_clean_adminq_subtask - Clean the AdminQ rings
10113  * @pf: board private structure
10114  **/
10115 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10116 {
10117         struct i40e_arq_event_info event;
10118         struct i40e_hw *hw = &pf->hw;
10119         u16 pending, i = 0;
10120         u16 opcode;
10121         u32 oldval;
10122         int ret;
10123         u32 val;
10124
10125         /* Do not run clean AQ when PF reset fails */
10126         if (test_bit(__I40E_RESET_FAILED, pf->state))
10127                 return;
10128
10129         /* check for error indications */
10130         val = rd32(&pf->hw, I40E_PF_ARQLEN);
10131         oldval = val;
10132         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10133                 if (hw->debug_mask & I40E_DEBUG_AQ)
10134                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10135                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10136         }
10137         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10138                 if (hw->debug_mask & I40E_DEBUG_AQ)
10139                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10140                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10141                 pf->arq_overflows++;
10142         }
10143         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10144                 if (hw->debug_mask & I40E_DEBUG_AQ)
10145                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10146                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10147         }
10148         if (oldval != val)
10149                 wr32(&pf->hw, I40E_PF_ARQLEN, val);
10150
10151         val = rd32(&pf->hw, I40E_PF_ATQLEN);
10152         oldval = val;
10153         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10154                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10155                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10156                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10157         }
10158         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10159                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10160                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10161                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10162         }
10163         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10164                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10165                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10166                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10167         }
10168         if (oldval != val)
10169                 wr32(&pf->hw, I40E_PF_ATQLEN, val);
10170
10171         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10172         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10173         if (!event.msg_buf)
10174                 return;
10175
10176         do {
10177                 ret = i40e_clean_arq_element(hw, &event, &pending);
10178                 if (ret == -EALREADY)
10179                         break;
10180                 else if (ret) {
10181                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10182                         break;
10183                 }
10184
10185                 opcode = le16_to_cpu(event.desc.opcode);
10186                 switch (opcode) {
10187
10188                 case i40e_aqc_opc_get_link_status:
10189                         rtnl_lock();
10190                         i40e_handle_link_event(pf, &event);
10191                         rtnl_unlock();
10192                         break;
10193                 case i40e_aqc_opc_send_msg_to_pf:
10194                         ret = i40e_vc_process_vf_msg(pf,
10195                                         le16_to_cpu(event.desc.retval),
10196                                         le32_to_cpu(event.desc.cookie_high),
10197                                         le32_to_cpu(event.desc.cookie_low),
10198                                         event.msg_buf,
10199                                         event.msg_len);
10200                         break;
10201                 case i40e_aqc_opc_lldp_update_mib:
10202                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10203 #ifdef CONFIG_I40E_DCB
10204                         rtnl_lock();
10205                         i40e_handle_lldp_event(pf, &event);
10206                         rtnl_unlock();
10207 #endif /* CONFIG_I40E_DCB */
10208                         break;
10209                 case i40e_aqc_opc_event_lan_overflow:
10210                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10211                         i40e_handle_lan_overflow_event(pf, &event);
10212                         break;
10213                 case i40e_aqc_opc_send_msg_to_peer:
10214                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10215                         break;
10216                 case i40e_aqc_opc_nvm_erase:
10217                 case i40e_aqc_opc_nvm_update:
10218                 case i40e_aqc_opc_oem_post_update:
10219                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10220                                    "ARQ NVM operation 0x%04x completed\n",
10221                                    opcode);
10222                         break;
10223                 default:
10224                         dev_info(&pf->pdev->dev,
10225                                  "ARQ: Unknown event 0x%04x ignored\n",
10226                                  opcode);
10227                         break;
10228                 }
10229         } while (i++ < I40E_AQ_WORK_LIMIT);
10230
10231         if (i < I40E_AQ_WORK_LIMIT)
10232                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10233
10234         /* re-enable Admin queue interrupt cause */
10235         val = rd32(hw, I40E_PFINT_ICR0_ENA);
10236         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10237         wr32(hw, I40E_PFINT_ICR0_ENA, val);
10238         i40e_flush(hw);
10239
10240         kfree(event.msg_buf);
10241 }
10242
10243 /**
10244  * i40e_verify_eeprom - make sure eeprom is good to use
10245  * @pf: board private structure
10246  **/
10247 static void i40e_verify_eeprom(struct i40e_pf *pf)
10248 {
10249         int err;
10250
10251         err = i40e_diag_eeprom_test(&pf->hw);
10252         if (err) {
10253                 /* retry in case of garbage read */
10254                 err = i40e_diag_eeprom_test(&pf->hw);
10255                 if (err) {
10256                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10257                                  err);
10258                         set_bit(__I40E_BAD_EEPROM, pf->state);
10259                 }
10260         }
10261
10262         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10263                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10264                 clear_bit(__I40E_BAD_EEPROM, pf->state);
10265         }
10266 }
10267
10268 /**
10269  * i40e_enable_pf_switch_lb
10270  * @pf: pointer to the PF structure
10271  *
10272  * enable switch loop back or die - no point in a return value
10273  **/
10274 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10275 {
10276         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10277         struct i40e_vsi_context ctxt;
10278         int ret;
10279
10280         ctxt.seid = pf->main_vsi_seid;
10281         ctxt.pf_num = pf->hw.pf_id;
10282         ctxt.vf_num = 0;
10283         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10284         if (ret) {
10285                 dev_info(&pf->pdev->dev,
10286                          "couldn't get PF vsi config, err %pe aq_err %s\n",
10287                          ERR_PTR(ret),
10288                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10289                 return;
10290         }
10291         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10292         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10293         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10294
10295         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10296         if (ret) {
10297                 dev_info(&pf->pdev->dev,
10298                          "update vsi switch failed, err %pe aq_err %s\n",
10299                          ERR_PTR(ret),
10300                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10301         }
10302 }
10303
10304 /**
10305  * i40e_disable_pf_switch_lb
10306  * @pf: pointer to the PF structure
10307  *
10308  * disable switch loop back or die - no point in a return value
10309  **/
10310 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10311 {
10312         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10313         struct i40e_vsi_context ctxt;
10314         int ret;
10315
10316         ctxt.seid = pf->main_vsi_seid;
10317         ctxt.pf_num = pf->hw.pf_id;
10318         ctxt.vf_num = 0;
10319         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10320         if (ret) {
10321                 dev_info(&pf->pdev->dev,
10322                          "couldn't get PF vsi config, err %pe aq_err %s\n",
10323                          ERR_PTR(ret),
10324                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10325                 return;
10326         }
10327         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10328         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10329         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10330
10331         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10332         if (ret) {
10333                 dev_info(&pf->pdev->dev,
10334                          "update vsi switch failed, err %pe aq_err %s\n",
10335                          ERR_PTR(ret),
10336                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10337         }
10338 }
10339
10340 /**
10341  * i40e_config_bridge_mode - Configure the HW bridge mode
10342  * @veb: pointer to the bridge instance
10343  *
10344  * Configure the loop back mode for the LAN VSI that is downlink to the
10345  * specified HW bridge instance. It is expected this function is called
10346  * when a new HW bridge is instantiated.
10347  **/
10348 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10349 {
10350         struct i40e_pf *pf = veb->pf;
10351
10352         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10353                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10354                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10355         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10356                 i40e_disable_pf_switch_lb(pf);
10357         else
10358                 i40e_enable_pf_switch_lb(pf);
10359 }
10360
10361 /**
10362  * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10363  * @veb: pointer to the VEB instance
10364  *
10365  * This is a function that builds the attached VSIs. We track the connections
10366  * through our own index numbers because the seid's from the HW could change
10367  * across the reset.
10368  **/
10369 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10370 {
10371         struct i40e_vsi *ctl_vsi = NULL;
10372         struct i40e_pf *pf = veb->pf;
10373         struct i40e_vsi *vsi;
10374         int v, ret;
10375
10376         /* As we do not maintain PV (port virtualizer) switch element then
10377          * there can be only one non-floating VEB that have uplink to MAC SEID
10378          * and its control VSI is the main one.
10379          */
10380         if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {
10381                 dev_err(&pf->pdev->dev,
10382                         "Invalid uplink SEID for VEB %d\n", veb->idx);
10383                 return -ENOENT;
10384         }
10385
10386         if (veb->uplink_seid == pf->mac_seid) {
10387                 /* Check that the LAN VSI has VEB owning flag set */
10388                 ctl_vsi = pf->vsi[pf->lan_vsi];
10389
10390                 if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||
10391                             !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {
10392                         dev_err(&pf->pdev->dev,
10393                                 "Invalid control VSI for VEB %d\n", veb->idx);
10394                         return -ENOENT;
10395                 }
10396
10397                 /* Add the control VSI to switch */
10398                 ret = i40e_add_vsi(ctl_vsi);
10399                 if (ret) {
10400                         dev_err(&pf->pdev->dev,
10401                                 "Rebuild of owner VSI for VEB %d failed: %d\n",
10402                                 veb->idx, ret);
10403                         return ret;
10404                 }
10405
10406                 i40e_vsi_reset_stats(ctl_vsi);
10407         }
10408
10409         /* create the VEB in the switch and move the VSI onto the VEB */
10410         ret = i40e_add_veb(veb, ctl_vsi);
10411         if (ret)
10412                 return ret;
10413
10414         if (veb->uplink_seid) {
10415                 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
10416                         veb->bridge_mode = BRIDGE_MODE_VEB;
10417                 else
10418                         veb->bridge_mode = BRIDGE_MODE_VEPA;
10419                 i40e_config_bridge_mode(veb);
10420         }
10421
10422         /* create the remaining VSIs attached to this VEB */
10423         i40e_pf_for_each_vsi(pf, v, vsi) {
10424                 if (vsi == ctl_vsi)
10425                         continue;
10426
10427                 if (vsi->veb_idx == veb->idx) {
10428                         vsi->uplink_seid = veb->seid;
10429                         ret = i40e_add_vsi(vsi);
10430                         if (ret) {
10431                                 dev_info(&pf->pdev->dev,
10432                                          "rebuild of vsi_idx %d failed: %d\n",
10433                                          v, ret);
10434                                 return ret;
10435                         }
10436                         i40e_vsi_reset_stats(vsi);
10437                 }
10438         }
10439
10440         return ret;
10441 }
10442
10443 /**
10444  * i40e_get_capabilities - get info about the HW
10445  * @pf: the PF struct
10446  * @list_type: AQ capability to be queried
10447  **/
10448 static int i40e_get_capabilities(struct i40e_pf *pf,
10449                                  enum i40e_admin_queue_opc list_type)
10450 {
10451         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10452         u16 data_size;
10453         int buf_len;
10454         int err;
10455
10456         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10457         do {
10458                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10459                 if (!cap_buf)
10460                         return -ENOMEM;
10461
10462                 /* this loads the data into the hw struct for us */
10463                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10464                                                     &data_size, list_type,
10465                                                     NULL);
10466                 /* data loaded, buffer no longer needed */
10467                 kfree(cap_buf);
10468
10469                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10470                         /* retry with a larger buffer */
10471                         buf_len = data_size;
10472                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10473                         dev_info(&pf->pdev->dev,
10474                                  "capability discovery failed, err %pe aq_err %s\n",
10475                                  ERR_PTR(err),
10476                                  i40e_aq_str(&pf->hw,
10477                                              pf->hw.aq.asq_last_status));
10478                         return -ENODEV;
10479                 }
10480         } while (err);
10481
10482         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10483                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10484                         dev_info(&pf->pdev->dev,
10485                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10486                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10487                                  pf->hw.func_caps.num_msix_vectors,
10488                                  pf->hw.func_caps.num_msix_vectors_vf,
10489                                  pf->hw.func_caps.fd_filters_guaranteed,
10490                                  pf->hw.func_caps.fd_filters_best_effort,
10491                                  pf->hw.func_caps.num_tx_qp,
10492                                  pf->hw.func_caps.num_vsis);
10493                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10494                         dev_info(&pf->pdev->dev,
10495                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
10496                                  pf->hw.dev_caps.switch_mode,
10497                                  pf->hw.dev_caps.valid_functions);
10498                         dev_info(&pf->pdev->dev,
10499                                  "SR-IOV=%d, num_vfs for all function=%u\n",
10500                                  pf->hw.dev_caps.sr_iov_1_1,
10501                                  pf->hw.dev_caps.num_vfs);
10502                         dev_info(&pf->pdev->dev,
10503                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10504                                  pf->hw.dev_caps.num_vsis,
10505                                  pf->hw.dev_caps.num_rx_qp,
10506                                  pf->hw.dev_caps.num_tx_qp);
10507                 }
10508         }
10509         if (list_type == i40e_aqc_opc_list_func_capabilities) {
10510 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10511                        + pf->hw.func_caps.num_vfs)
10512                 if (pf->hw.revision_id == 0 &&
10513                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10514                         dev_info(&pf->pdev->dev,
10515                                  "got num_vsis %d, setting num_vsis to %d\n",
10516                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10517                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10518                 }
10519         }
10520         return 0;
10521 }
10522
10523 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10524
10525 /**
10526  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10527  * @pf: board private structure
10528  **/
10529 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10530 {
10531         struct i40e_vsi *vsi;
10532
10533         /* quick workaround for an NVM issue that leaves a critical register
10534          * uninitialized
10535          */
10536         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10537                 static const u32 hkey[] = {
10538                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10539                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10540                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10541                         0x95b3a76d};
10542                 int i;
10543
10544                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10545                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10546         }
10547
10548         if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
10549                 return;
10550
10551         /* find existing VSI and see if it needs configuring */
10552         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10553
10554         /* create a new VSI if none exists */
10555         if (!vsi) {
10556                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10557                                      pf->vsi[pf->lan_vsi]->seid, 0);
10558                 if (!vsi) {
10559                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10560                         clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
10561                         set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
10562                         return;
10563                 }
10564         }
10565
10566         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10567 }
10568
10569 /**
10570  * i40e_fdir_teardown - release the Flow Director resources
10571  * @pf: board private structure
10572  **/
10573 static void i40e_fdir_teardown(struct i40e_pf *pf)
10574 {
10575         struct i40e_vsi *vsi;
10576
10577         i40e_fdir_filter_exit(pf);
10578         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10579         if (vsi)
10580                 i40e_vsi_release(vsi);
10581 }
10582
10583 /**
10584  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10585  * @vsi: PF main vsi
10586  * @seid: seid of main or channel VSIs
10587  *
10588  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10589  * existed before reset
10590  **/
10591 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10592 {
10593         struct i40e_cloud_filter *cfilter;
10594         struct i40e_pf *pf = vsi->back;
10595         struct hlist_node *node;
10596         int ret;
10597
10598         /* Add cloud filters back if they exist */
10599         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10600                                   cloud_node) {
10601                 if (cfilter->seid != seid)
10602                         continue;
10603
10604                 if (cfilter->dst_port)
10605                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10606                                                                 true);
10607                 else
10608                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10609
10610                 if (ret) {
10611                         dev_dbg(&pf->pdev->dev,
10612                                 "Failed to rebuild cloud filter, err %pe aq_err %s\n",
10613                                 ERR_PTR(ret),
10614                                 i40e_aq_str(&pf->hw,
10615                                             pf->hw.aq.asq_last_status));
10616                         return ret;
10617                 }
10618         }
10619         return 0;
10620 }
10621
10622 /**
10623  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10624  * @vsi: PF main vsi
10625  *
10626  * Rebuilds channel VSIs if they existed before reset
10627  **/
10628 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10629 {
10630         struct i40e_channel *ch, *ch_tmp;
10631         int ret;
10632
10633         if (list_empty(&vsi->ch_list))
10634                 return 0;
10635
10636         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10637                 if (!ch->initialized)
10638                         break;
10639                 /* Proceed with creation of channel (VMDq2) VSI */
10640                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10641                 if (ret) {
10642                         dev_info(&vsi->back->pdev->dev,
10643                                  "failed to rebuild channels using uplink_seid %u\n",
10644                                  vsi->uplink_seid);
10645                         return ret;
10646                 }
10647                 /* Reconfigure TX queues using QTX_CTL register */
10648                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10649                 if (ret) {
10650                         dev_info(&vsi->back->pdev->dev,
10651                                  "failed to configure TX rings for channel %u\n",
10652                                  ch->seid);
10653                         return ret;
10654                 }
10655                 /* update 'next_base_queue' */
10656                 vsi->next_base_queue = vsi->next_base_queue +
10657                                                         ch->num_queue_pairs;
10658                 if (ch->max_tx_rate) {
10659                         u64 credits = ch->max_tx_rate;
10660
10661                         if (i40e_set_bw_limit(vsi, ch->seid,
10662                                               ch->max_tx_rate))
10663                                 return -EINVAL;
10664
10665                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
10666                         dev_dbg(&vsi->back->pdev->dev,
10667                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10668                                 ch->max_tx_rate,
10669                                 credits,
10670                                 ch->seid);
10671                 }
10672                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10673                 if (ret) {
10674                         dev_dbg(&vsi->back->pdev->dev,
10675                                 "Failed to rebuild cloud filters for channel VSI %u\n",
10676                                 ch->seid);
10677                         return ret;
10678                 }
10679         }
10680         return 0;
10681 }
10682
10683 /**
10684  * i40e_clean_xps_state - clean xps state for every tx_ring
10685  * @vsi: ptr to the VSI
10686  **/
10687 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10688 {
10689         int i;
10690
10691         if (vsi->tx_rings)
10692                 for (i = 0; i < vsi->num_queue_pairs; i++)
10693                         if (vsi->tx_rings[i])
10694                                 clear_bit(__I40E_TX_XPS_INIT_DONE,
10695                                           vsi->tx_rings[i]->state);
10696 }
10697
10698 /**
10699  * i40e_prep_for_reset - prep for the core to reset
10700  * @pf: board private structure
10701  *
10702  * Close up the VFs and other things in prep for PF Reset.
10703   **/
10704 static void i40e_prep_for_reset(struct i40e_pf *pf)
10705 {
10706         struct i40e_hw *hw = &pf->hw;
10707         struct i40e_vsi *vsi;
10708         int ret = 0;
10709         u32 v;
10710
10711         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10712         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10713                 return;
10714         if (i40e_check_asq_alive(&pf->hw))
10715                 i40e_vc_notify_reset(pf);
10716
10717         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10718
10719         /* quiesce the VSIs and their queues that are not already DOWN */
10720         i40e_pf_quiesce_all_vsi(pf);
10721
10722         i40e_pf_for_each_vsi(pf, v, vsi) {
10723                 i40e_clean_xps_state(vsi);
10724                 vsi->seid = 0;
10725         }
10726
10727         i40e_shutdown_adminq(&pf->hw);
10728
10729         /* call shutdown HMC */
10730         if (hw->hmc.hmc_obj) {
10731                 ret = i40e_shutdown_lan_hmc(hw);
10732                 if (ret)
10733                         dev_warn(&pf->pdev->dev,
10734                                  "shutdown_lan_hmc failed: %d\n", ret);
10735         }
10736
10737         /* Save the current PTP time so that we can restore the time after the
10738          * reset completes.
10739          */
10740         i40e_ptp_save_hw_time(pf);
10741 }
10742
10743 /**
10744  * i40e_send_version - update firmware with driver version
10745  * @pf: PF struct
10746  */
10747 static void i40e_send_version(struct i40e_pf *pf)
10748 {
10749         struct i40e_driver_version dv;
10750
10751         dv.major_version = 0xff;
10752         dv.minor_version = 0xff;
10753         dv.build_version = 0xff;
10754         dv.subbuild_version = 0;
10755         strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10756         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10757 }
10758
10759 /**
10760  * i40e_get_oem_version - get OEM specific version information
10761  * @hw: pointer to the hardware structure
10762  **/
10763 static void i40e_get_oem_version(struct i40e_hw *hw)
10764 {
10765         u16 block_offset = 0xffff;
10766         u16 block_length = 0;
10767         u16 capabilities = 0;
10768         u16 gen_snap = 0;
10769         u16 release = 0;
10770
10771 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
10772 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
10773 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
10774 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
10775 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
10776 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
10777 #define I40E_NVM_OEM_LENGTH                     3
10778
10779         /* Check if pointer to OEM version block is valid. */
10780         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10781         if (block_offset == 0xffff)
10782                 return;
10783
10784         /* Check if OEM version block has correct length. */
10785         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10786                            &block_length);
10787         if (block_length < I40E_NVM_OEM_LENGTH)
10788                 return;
10789
10790         /* Check if OEM version format is as expected. */
10791         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10792                            &capabilities);
10793         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10794                 return;
10795
10796         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10797                            &gen_snap);
10798         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10799                            &release);
10800         hw->nvm.oem_ver =
10801                 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |
10802                 FIELD_PREP(I40E_OEM_RELEASE_MASK, release);
10803         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10804 }
10805
10806 /**
10807  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10808  * @pf: board private structure
10809  **/
10810 static int i40e_reset(struct i40e_pf *pf)
10811 {
10812         struct i40e_hw *hw = &pf->hw;
10813         int ret;
10814
10815         ret = i40e_pf_reset(hw);
10816         if (ret) {
10817                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10818                 set_bit(__I40E_RESET_FAILED, pf->state);
10819                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10820         } else {
10821                 pf->pfr_count++;
10822         }
10823         return ret;
10824 }
10825
10826 /**
10827  * i40e_rebuild - rebuild using a saved config
10828  * @pf: board private structure
10829  * @reinit: if the Main VSI needs to re-initialized.
10830  * @lock_acquired: indicates whether or not the lock has been acquired
10831  * before this function was called.
10832  **/
10833 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10834 {
10835         const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10836         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10837         struct i40e_hw *hw = &pf->hw;
10838         struct i40e_veb *veb;
10839         int ret;
10840         u32 val;
10841         int v;
10842
10843         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10844             is_recovery_mode_reported)
10845                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10846
10847         if (test_bit(__I40E_DOWN, pf->state) &&
10848             !test_bit(__I40E_RECOVERY_MODE, pf->state))
10849                 goto clear_recovery;
10850         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10851
10852         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10853         ret = i40e_init_adminq(&pf->hw);
10854         if (ret) {
10855                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10856                          ERR_PTR(ret),
10857                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10858                 goto clear_recovery;
10859         }
10860         i40e_get_oem_version(&pf->hw);
10861
10862         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10863                 /* The following delay is necessary for firmware update. */
10864                 mdelay(1000);
10865         }
10866
10867         /* re-verify the eeprom if we just had an EMP reset */
10868         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10869                 i40e_verify_eeprom(pf);
10870
10871         /* if we are going out of or into recovery mode we have to act
10872          * accordingly with regard to resources initialization
10873          * and deinitialization
10874          */
10875         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10876                 if (i40e_get_capabilities(pf,
10877                                           i40e_aqc_opc_list_func_capabilities))
10878                         goto end_unlock;
10879
10880                 if (is_recovery_mode_reported) {
10881                         /* we're staying in recovery mode so we'll reinitialize
10882                          * misc vector here
10883                          */
10884                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10885                                 goto end_unlock;
10886                 } else {
10887                         if (!lock_acquired)
10888                                 rtnl_lock();
10889                         /* we're going out of recovery mode so we'll free
10890                          * the IRQ allocated specifically for recovery mode
10891                          * and restore the interrupt scheme
10892                          */
10893                         free_irq(pf->pdev->irq, pf);
10894                         i40e_clear_interrupt_scheme(pf);
10895                         if (i40e_restore_interrupt_scheme(pf))
10896                                 goto end_unlock;
10897                 }
10898
10899                 /* tell the firmware that we're starting */
10900                 i40e_send_version(pf);
10901
10902                 /* bail out in case recovery mode was detected, as there is
10903                  * no need for further configuration.
10904                  */
10905                 goto end_unlock;
10906         }
10907
10908         i40e_clear_pxe_mode(hw);
10909         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10910         if (ret)
10911                 goto end_core_reset;
10912
10913         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10914                                 hw->func_caps.num_rx_qp, 0, 0);
10915         if (ret) {
10916                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10917                 goto end_core_reset;
10918         }
10919         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10920         if (ret) {
10921                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10922                 goto end_core_reset;
10923         }
10924
10925 #ifdef CONFIG_I40E_DCB
10926         /* Enable FW to write a default DCB config on link-up
10927          * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10928          * is not supported with new link speed
10929          */
10930         if (i40e_is_tc_mqprio_enabled(pf)) {
10931                 i40e_aq_set_dcb_parameters(hw, false, NULL);
10932         } else {
10933                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10934                     (hw->phy.link_info.link_speed &
10935                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10936                         i40e_aq_set_dcb_parameters(hw, false, NULL);
10937                         dev_warn(&pf->pdev->dev,
10938                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10939                         clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10940                 } else {
10941                         i40e_aq_set_dcb_parameters(hw, true, NULL);
10942                         ret = i40e_init_pf_dcb(pf);
10943                         if (ret) {
10944                                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10945                                          ret);
10946                                 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10947                                 /* Continue without DCB enabled */
10948                         }
10949                 }
10950         }
10951
10952 #endif /* CONFIG_I40E_DCB */
10953         if (!lock_acquired)
10954                 rtnl_lock();
10955         ret = i40e_setup_pf_switch(pf, reinit, true);
10956         if (ret)
10957                 goto end_unlock;
10958
10959         /* The driver only wants link up/down and module qualification
10960          * reports from firmware.  Note the negative logic.
10961          */
10962         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10963                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10964                                          I40E_AQ_EVENT_MEDIA_NA |
10965                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10966         if (ret)
10967                 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
10968                          ERR_PTR(ret),
10969                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10970
10971         /* Rebuild the VSIs and VEBs that existed before reset.
10972          * They are still in our local switch element arrays, so only
10973          * need to rebuild the switch model in the HW.
10974          *
10975          * If there were VEBs but the reconstitution failed, we'll try
10976          * to recover minimal use by getting the basic PF VSI working.
10977          */
10978         if (vsi->uplink_seid != pf->mac_seid) {
10979                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10980
10981                 /* Rebuild VEBs */
10982                 i40e_pf_for_each_veb(pf, v, veb) {
10983                         ret = i40e_reconstitute_veb(veb);
10984                         if (!ret)
10985                                 continue;
10986
10987                         /* If Main VEB failed, we're in deep doodoo,
10988                          * so give up rebuilding the switch and set up
10989                          * for minimal rebuild of PF VSI.
10990                          * If orphan failed, we'll report the error
10991                          * but try to keep going.
10992                          */
10993                         if (veb->uplink_seid == pf->mac_seid) {
10994                                 dev_info(&pf->pdev->dev,
10995                                          "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10996                                          ret);
10997                                 vsi->uplink_seid = pf->mac_seid;
10998                                 break;
10999                         } else if (veb->uplink_seid == 0) {
11000                                 dev_info(&pf->pdev->dev,
11001                                          "rebuild of orphan VEB failed: %d\n",
11002                                          ret);
11003                         }
11004                 }
11005         }
11006
11007         if (vsi->uplink_seid == pf->mac_seid) {
11008                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11009                 /* no VEB, so rebuild only the Main VSI */
11010                 ret = i40e_add_vsi(vsi);
11011                 if (ret) {
11012                         dev_info(&pf->pdev->dev,
11013                                  "rebuild of Main VSI failed: %d\n", ret);
11014                         goto end_unlock;
11015                 }
11016         }
11017
11018         if (vsi->mqprio_qopt.max_rate[0]) {
11019                 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11020                                                   vsi->mqprio_qopt.max_rate[0]);
11021                 u64 credits = 0;
11022
11023                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11024                 if (ret)
11025                         goto end_unlock;
11026
11027                 credits = max_tx_rate;
11028                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
11029                 dev_dbg(&vsi->back->pdev->dev,
11030                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11031                         max_tx_rate,
11032                         credits,
11033                         vsi->seid);
11034         }
11035
11036         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11037         if (ret)
11038                 goto end_unlock;
11039
11040         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11041          * for this main VSI if they exist
11042          */
11043         ret = i40e_rebuild_channels(vsi);
11044         if (ret)
11045                 goto end_unlock;
11046
11047         /* Reconfigure hardware for allowing smaller MSS in the case
11048          * of TSO, so that we avoid the MDD being fired and causing
11049          * a reset in the case of small MSS+TSO.
11050          */
11051 #define I40E_REG_MSS          0x000E64DC
11052 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11053 #define I40E_64BYTE_MSS       0x400000
11054         val = rd32(hw, I40E_REG_MSS);
11055         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11056                 val &= ~I40E_REG_MSS_MIN_MASK;
11057                 val |= I40E_64BYTE_MSS;
11058                 wr32(hw, I40E_REG_MSS, val);
11059         }
11060
11061         if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
11062                 msleep(75);
11063                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11064                 if (ret)
11065                         dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11066                                  ERR_PTR(ret),
11067                                  i40e_aq_str(&pf->hw,
11068                                              pf->hw.aq.asq_last_status));
11069         }
11070         /* reinit the misc interrupt */
11071         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
11072                 ret = i40e_setup_misc_vector(pf);
11073                 if (ret)
11074                         goto end_unlock;
11075         }
11076
11077         /* Add a filter to drop all Flow control frames from any VSI from being
11078          * transmitted. By doing so we stop a malicious VF from sending out
11079          * PAUSE or PFC frames and potentially controlling traffic for other
11080          * PF/VF VSIs.
11081          * The FW can still send Flow control frames if enabled.
11082          */
11083         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11084                                                        pf->main_vsi_seid);
11085
11086         /* restart the VSIs that were rebuilt and running before the reset */
11087         i40e_pf_unquiesce_all_vsi(pf);
11088
11089         /* Release the RTNL lock before we start resetting VFs */
11090         if (!lock_acquired)
11091                 rtnl_unlock();
11092
11093         /* Restore promiscuous settings */
11094         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11095         if (ret)
11096                 dev_warn(&pf->pdev->dev,
11097                          "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11098                          pf->cur_promisc ? "on" : "off",
11099                          ERR_PTR(ret),
11100                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11101
11102         i40e_reset_all_vfs(pf, true);
11103
11104         /* tell the firmware that we're starting */
11105         i40e_send_version(pf);
11106
11107         /* We've already released the lock, so don't do it again */
11108         goto end_core_reset;
11109
11110 end_unlock:
11111         if (!lock_acquired)
11112                 rtnl_unlock();
11113 end_core_reset:
11114         clear_bit(__I40E_RESET_FAILED, pf->state);
11115 clear_recovery:
11116         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11117         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11118 }
11119
11120 /**
11121  * i40e_reset_and_rebuild - reset and rebuild using a saved config
11122  * @pf: board private structure
11123  * @reinit: if the Main VSI needs to re-initialized.
11124  * @lock_acquired: indicates whether or not the lock has been acquired
11125  * before this function was called.
11126  **/
11127 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11128                                    bool lock_acquired)
11129 {
11130         int ret;
11131
11132         if (test_bit(__I40E_IN_REMOVE, pf->state))
11133                 return;
11134         /* Now we wait for GRST to settle out.
11135          * We don't have to delete the VEBs or VSIs from the hw switch
11136          * because the reset will make them disappear.
11137          */
11138         ret = i40e_reset(pf);
11139         if (!ret)
11140                 i40e_rebuild(pf, reinit, lock_acquired);
11141 }
11142
11143 /**
11144  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11145  * @pf: board private structure
11146  *
11147  * Close up the VFs and other things in prep for a Core Reset,
11148  * then get ready to rebuild the world.
11149  * @lock_acquired: indicates whether or not the lock has been acquired
11150  * before this function was called.
11151  **/
11152 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11153 {
11154         i40e_prep_for_reset(pf);
11155         i40e_reset_and_rebuild(pf, false, lock_acquired);
11156 }
11157
11158 /**
11159  * i40e_handle_mdd_event
11160  * @pf: pointer to the PF structure
11161  *
11162  * Called from the MDD irq handler to identify possibly malicious vfs
11163  **/
11164 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11165 {
11166         struct i40e_hw *hw = &pf->hw;
11167         bool mdd_detected = false;
11168         struct i40e_vf *vf;
11169         u32 reg;
11170         int i;
11171
11172         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11173                 return;
11174
11175         /* find what triggered the MDD event */
11176         reg = rd32(hw, I40E_GL_MDET_TX);
11177         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11178                 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);
11179                 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);
11180                 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);
11181                 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -
11182                                 pf->hw.func_caps.base_queue;
11183                 if (netif_msg_tx_err(pf))
11184                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11185                                  event, queue, pf_num, vf_num);
11186                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11187                 mdd_detected = true;
11188         }
11189         reg = rd32(hw, I40E_GL_MDET_RX);
11190         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11191                 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);
11192                 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);
11193                 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -
11194                                 pf->hw.func_caps.base_queue;
11195                 if (netif_msg_rx_err(pf))
11196                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11197                                  event, queue, func);
11198                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11199                 mdd_detected = true;
11200         }
11201
11202         if (mdd_detected) {
11203                 reg = rd32(hw, I40E_PF_MDET_TX);
11204                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11205                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11206                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11207                 }
11208                 reg = rd32(hw, I40E_PF_MDET_RX);
11209                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11210                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11211                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11212                 }
11213         }
11214
11215         /* see if one of the VFs needs its hand slapped */
11216         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11217                 vf = &(pf->vf[i]);
11218                 reg = rd32(hw, I40E_VP_MDET_TX(i));
11219                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11220                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11221                         vf->num_mdd_events++;
11222                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11223                                  i);
11224                         dev_info(&pf->pdev->dev,
11225                                  "Use PF Control I/F to re-enable the VF\n");
11226                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11227                 }
11228
11229                 reg = rd32(hw, I40E_VP_MDET_RX(i));
11230                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11231                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11232                         vf->num_mdd_events++;
11233                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11234                                  i);
11235                         dev_info(&pf->pdev->dev,
11236                                  "Use PF Control I/F to re-enable the VF\n");
11237                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11238                 }
11239         }
11240
11241         /* re-enable mdd interrupt cause */
11242         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
11243         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11244         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11245         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11246         i40e_flush(hw);
11247 }
11248
11249 /**
11250  * i40e_service_task - Run the driver's async subtasks
11251  * @work: pointer to work_struct containing our data
11252  **/
11253 static void i40e_service_task(struct work_struct *work)
11254 {
11255         struct i40e_pf *pf = container_of(work,
11256                                           struct i40e_pf,
11257                                           service_task);
11258         unsigned long start_time = jiffies;
11259
11260         /* don't bother with service tasks if a reset is in progress */
11261         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11262             test_bit(__I40E_SUSPENDED, pf->state))
11263                 return;
11264
11265         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11266                 return;
11267
11268         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11269                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
11270                 i40e_sync_filters_subtask(pf);
11271                 i40e_reset_subtask(pf);
11272                 i40e_handle_mdd_event(pf);
11273                 i40e_vc_process_vflr_event(pf);
11274                 i40e_watchdog_subtask(pf);
11275                 i40e_fdir_reinit_subtask(pf);
11276                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11277                         /* Client subtask will reopen next time through. */
11278                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
11279                                                            true);
11280                 } else {
11281                         i40e_client_subtask(pf);
11282                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11283                                                pf->state))
11284                                 i40e_notify_client_of_l2_param_changes(
11285                                                                 pf->vsi[pf->lan_vsi]);
11286                 }
11287                 i40e_sync_filters_subtask(pf);
11288         } else {
11289                 i40e_reset_subtask(pf);
11290         }
11291
11292         i40e_clean_adminq_subtask(pf);
11293
11294         /* flush memory to make sure state is correct before next watchdog */
11295         smp_mb__before_atomic();
11296         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11297
11298         /* If the tasks have taken longer than one timer cycle or there
11299          * is more work to be done, reschedule the service task now
11300          * rather than wait for the timer to tick again.
11301          */
11302         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11303             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
11304             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
11305             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11306                 i40e_service_event_schedule(pf);
11307 }
11308
11309 /**
11310  * i40e_service_timer - timer callback
11311  * @t: timer list pointer
11312  **/
11313 static void i40e_service_timer(struct timer_list *t)
11314 {
11315         struct i40e_pf *pf = from_timer(pf, t, service_timer);
11316
11317         mod_timer(&pf->service_timer,
11318                   round_jiffies(jiffies + pf->service_timer_period));
11319         i40e_service_event_schedule(pf);
11320 }
11321
11322 /**
11323  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11324  * @vsi: the VSI being configured
11325  **/
11326 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11327 {
11328         struct i40e_pf *pf = vsi->back;
11329
11330         switch (vsi->type) {
11331         case I40E_VSI_MAIN:
11332                 vsi->alloc_queue_pairs = pf->num_lan_qps;
11333                 if (!vsi->num_tx_desc)
11334                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11335                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11336                 if (!vsi->num_rx_desc)
11337                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11338                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11339                 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11340                         vsi->num_q_vectors = pf->num_lan_msix;
11341                 else
11342                         vsi->num_q_vectors = 1;
11343
11344                 break;
11345
11346         case I40E_VSI_FDIR:
11347                 vsi->alloc_queue_pairs = 1;
11348                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11349                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11350                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11351                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11352                 vsi->num_q_vectors = pf->num_fdsb_msix;
11353                 break;
11354
11355         case I40E_VSI_VMDQ2:
11356                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11357                 if (!vsi->num_tx_desc)
11358                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11359                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11360                 if (!vsi->num_rx_desc)
11361                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11362                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11363                 vsi->num_q_vectors = pf->num_vmdq_msix;
11364                 break;
11365
11366         case I40E_VSI_SRIOV:
11367                 vsi->alloc_queue_pairs = pf->num_vf_qps;
11368                 if (!vsi->num_tx_desc)
11369                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11370                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11371                 if (!vsi->num_rx_desc)
11372                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11373                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11374                 break;
11375
11376         default:
11377                 WARN_ON(1);
11378                 return -ENODATA;
11379         }
11380
11381         if (is_kdump_kernel()) {
11382                 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11383                 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11384         }
11385
11386         return 0;
11387 }
11388
11389 /**
11390  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11391  * @vsi: VSI pointer
11392  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11393  *
11394  * On error: returns error code (negative)
11395  * On success: returns 0
11396  **/
11397 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11398 {
11399         struct i40e_ring **next_rings;
11400         int size;
11401         int ret = 0;
11402
11403         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11404         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11405                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11406         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11407         if (!vsi->tx_rings)
11408                 return -ENOMEM;
11409         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11410         if (i40e_enabled_xdp_vsi(vsi)) {
11411                 vsi->xdp_rings = next_rings;
11412                 next_rings += vsi->alloc_queue_pairs;
11413         }
11414         vsi->rx_rings = next_rings;
11415
11416         if (alloc_qvectors) {
11417                 /* allocate memory for q_vector pointers */
11418                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11419                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11420                 if (!vsi->q_vectors) {
11421                         ret = -ENOMEM;
11422                         goto err_vectors;
11423                 }
11424         }
11425         return ret;
11426
11427 err_vectors:
11428         kfree(vsi->tx_rings);
11429         return ret;
11430 }
11431
11432 /**
11433  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11434  * @pf: board private structure
11435  * @type: type of VSI
11436  *
11437  * On error: returns error code (negative)
11438  * On success: returns vsi index in PF (positive)
11439  **/
11440 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11441 {
11442         int ret = -ENODEV;
11443         struct i40e_vsi *vsi;
11444         int vsi_idx;
11445         int i;
11446
11447         /* Need to protect the allocation of the VSIs at the PF level */
11448         mutex_lock(&pf->switch_mutex);
11449
11450         /* VSI list may be fragmented if VSI creation/destruction has
11451          * been happening.  We can afford to do a quick scan to look
11452          * for any free VSIs in the list.
11453          *
11454          * find next empty vsi slot, looping back around if necessary
11455          */
11456         i = pf->next_vsi;
11457         while (i < pf->num_alloc_vsi && pf->vsi[i])
11458                 i++;
11459         if (i >= pf->num_alloc_vsi) {
11460                 i = 0;
11461                 while (i < pf->next_vsi && pf->vsi[i])
11462                         i++;
11463         }
11464
11465         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11466                 vsi_idx = i;             /* Found one! */
11467         } else {
11468                 ret = -ENODEV;
11469                 goto unlock_pf;  /* out of VSI slots! */
11470         }
11471         pf->next_vsi = ++i;
11472
11473         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11474         if (!vsi) {
11475                 ret = -ENOMEM;
11476                 goto unlock_pf;
11477         }
11478         vsi->type = type;
11479         vsi->back = pf;
11480         set_bit(__I40E_VSI_DOWN, vsi->state);
11481         vsi->flags = 0;
11482         vsi->idx = vsi_idx;
11483         vsi->int_rate_limit = 0;
11484         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11485                                 pf->rss_table_size : 64;
11486         vsi->netdev_registered = false;
11487         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11488         hash_init(vsi->mac_filter_hash);
11489         vsi->irqs_ready = false;
11490
11491         if (type == I40E_VSI_MAIN) {
11492                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11493                 if (!vsi->af_xdp_zc_qps)
11494                         goto err_rings;
11495         }
11496
11497         ret = i40e_set_num_rings_in_vsi(vsi);
11498         if (ret)
11499                 goto err_rings;
11500
11501         ret = i40e_vsi_alloc_arrays(vsi, true);
11502         if (ret)
11503                 goto err_rings;
11504
11505         /* Setup default MSIX irq handler for VSI */
11506         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11507
11508         /* Initialize VSI lock */
11509         spin_lock_init(&vsi->mac_filter_hash_lock);
11510         pf->vsi[vsi_idx] = vsi;
11511         ret = vsi_idx;
11512         goto unlock_pf;
11513
11514 err_rings:
11515         bitmap_free(vsi->af_xdp_zc_qps);
11516         pf->next_vsi = i - 1;
11517         kfree(vsi);
11518 unlock_pf:
11519         mutex_unlock(&pf->switch_mutex);
11520         return ret;
11521 }
11522
11523 /**
11524  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11525  * @vsi: VSI pointer
11526  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11527  *
11528  * On error: returns error code (negative)
11529  * On success: returns 0
11530  **/
11531 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11532 {
11533         /* free the ring and vector containers */
11534         if (free_qvectors) {
11535                 kfree(vsi->q_vectors);
11536                 vsi->q_vectors = NULL;
11537         }
11538         kfree(vsi->tx_rings);
11539         vsi->tx_rings = NULL;
11540         vsi->rx_rings = NULL;
11541         vsi->xdp_rings = NULL;
11542 }
11543
11544 /**
11545  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11546  * and lookup table
11547  * @vsi: Pointer to VSI structure
11548  */
11549 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11550 {
11551         if (!vsi)
11552                 return;
11553
11554         kfree(vsi->rss_hkey_user);
11555         vsi->rss_hkey_user = NULL;
11556
11557         kfree(vsi->rss_lut_user);
11558         vsi->rss_lut_user = NULL;
11559 }
11560
11561 /**
11562  * i40e_vsi_clear - Deallocate the VSI provided
11563  * @vsi: the VSI being un-configured
11564  **/
11565 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11566 {
11567         struct i40e_pf *pf;
11568
11569         if (!vsi)
11570                 return 0;
11571
11572         if (!vsi->back)
11573                 goto free_vsi;
11574         pf = vsi->back;
11575
11576         mutex_lock(&pf->switch_mutex);
11577         if (!pf->vsi[vsi->idx]) {
11578                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11579                         vsi->idx, vsi->idx, vsi->type);
11580                 goto unlock_vsi;
11581         }
11582
11583         if (pf->vsi[vsi->idx] != vsi) {
11584                 dev_err(&pf->pdev->dev,
11585                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11586                         pf->vsi[vsi->idx]->idx,
11587                         pf->vsi[vsi->idx]->type,
11588                         vsi->idx, vsi->type);
11589                 goto unlock_vsi;
11590         }
11591
11592         /* updates the PF for this cleared vsi */
11593         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11594         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11595
11596         bitmap_free(vsi->af_xdp_zc_qps);
11597         i40e_vsi_free_arrays(vsi, true);
11598         i40e_clear_rss_config_user(vsi);
11599
11600         pf->vsi[vsi->idx] = NULL;
11601         if (vsi->idx < pf->next_vsi)
11602                 pf->next_vsi = vsi->idx;
11603
11604 unlock_vsi:
11605         mutex_unlock(&pf->switch_mutex);
11606 free_vsi:
11607         kfree(vsi);
11608
11609         return 0;
11610 }
11611
11612 /**
11613  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11614  * @vsi: the VSI being cleaned
11615  **/
11616 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11617 {
11618         int i;
11619
11620         if (vsi->tx_rings && vsi->tx_rings[0]) {
11621                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11622                         kfree_rcu(vsi->tx_rings[i], rcu);
11623                         WRITE_ONCE(vsi->tx_rings[i], NULL);
11624                         WRITE_ONCE(vsi->rx_rings[i], NULL);
11625                         if (vsi->xdp_rings)
11626                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11627                 }
11628         }
11629 }
11630
11631 /**
11632  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11633  * @vsi: the VSI being configured
11634  **/
11635 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11636 {
11637         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11638         struct i40e_pf *pf = vsi->back;
11639         struct i40e_ring *ring;
11640
11641         /* Set basic values in the rings to be used later during open() */
11642         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11643                 /* allocate space for both Tx and Rx in one shot */
11644                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11645                 if (!ring)
11646                         goto err_out;
11647
11648                 ring->queue_index = i;
11649                 ring->reg_idx = vsi->base_queue + i;
11650                 ring->ring_active = false;
11651                 ring->vsi = vsi;
11652                 ring->netdev = vsi->netdev;
11653                 ring->dev = &pf->pdev->dev;
11654                 ring->count = vsi->num_tx_desc;
11655                 ring->size = 0;
11656                 ring->dcb_tc = 0;
11657                 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11658                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11659                 ring->itr_setting = pf->tx_itr_default;
11660                 WRITE_ONCE(vsi->tx_rings[i], ring++);
11661
11662                 if (!i40e_enabled_xdp_vsi(vsi))
11663                         goto setup_rx;
11664
11665                 ring->queue_index = vsi->alloc_queue_pairs + i;
11666                 ring->reg_idx = vsi->base_queue + ring->queue_index;
11667                 ring->ring_active = false;
11668                 ring->vsi = vsi;
11669                 ring->netdev = NULL;
11670                 ring->dev = &pf->pdev->dev;
11671                 ring->count = vsi->num_tx_desc;
11672                 ring->size = 0;
11673                 ring->dcb_tc = 0;
11674                 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11675                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11676                 set_ring_xdp(ring);
11677                 ring->itr_setting = pf->tx_itr_default;
11678                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11679
11680 setup_rx:
11681                 ring->queue_index = i;
11682                 ring->reg_idx = vsi->base_queue + i;
11683                 ring->ring_active = false;
11684                 ring->vsi = vsi;
11685                 ring->netdev = vsi->netdev;
11686                 ring->dev = &pf->pdev->dev;
11687                 ring->count = vsi->num_rx_desc;
11688                 ring->size = 0;
11689                 ring->dcb_tc = 0;
11690                 ring->itr_setting = pf->rx_itr_default;
11691                 WRITE_ONCE(vsi->rx_rings[i], ring);
11692         }
11693
11694         return 0;
11695
11696 err_out:
11697         i40e_vsi_clear_rings(vsi);
11698         return -ENOMEM;
11699 }
11700
11701 /**
11702  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11703  * @pf: board private structure
11704  * @vectors: the number of MSI-X vectors to request
11705  *
11706  * Returns the number of vectors reserved, or error
11707  **/
11708 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11709 {
11710         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11711                                         I40E_MIN_MSIX, vectors);
11712         if (vectors < 0) {
11713                 dev_info(&pf->pdev->dev,
11714                          "MSI-X vector reservation failed: %d\n", vectors);
11715                 vectors = 0;
11716         }
11717
11718         return vectors;
11719 }
11720
11721 /**
11722  * i40e_init_msix - Setup the MSIX capability
11723  * @pf: board private structure
11724  *
11725  * Work with the OS to set up the MSIX vectors needed.
11726  *
11727  * Returns the number of vectors reserved or negative on failure
11728  **/
11729 static int i40e_init_msix(struct i40e_pf *pf)
11730 {
11731         struct i40e_hw *hw = &pf->hw;
11732         int cpus, extra_vectors;
11733         int vectors_left;
11734         int v_budget, i;
11735         int v_actual;
11736         int iwarp_requested = 0;
11737
11738         if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11739                 return -ENODEV;
11740
11741         /* The number of vectors we'll request will be comprised of:
11742          *   - Add 1 for "other" cause for Admin Queue events, etc.
11743          *   - The number of LAN queue pairs
11744          *      - Queues being used for RSS.
11745          *              We don't need as many as max_rss_size vectors.
11746          *              use rss_size instead in the calculation since that
11747          *              is governed by number of cpus in the system.
11748          *      - assumes symmetric Tx/Rx pairing
11749          *   - The number of VMDq pairs
11750          *   - The CPU count within the NUMA node if iWARP is enabled
11751          * Once we count this up, try the request.
11752          *
11753          * If we can't get what we want, we'll simplify to nearly nothing
11754          * and try again.  If that still fails, we punt.
11755          */
11756         vectors_left = hw->func_caps.num_msix_vectors;
11757         v_budget = 0;
11758
11759         /* reserve one vector for miscellaneous handler */
11760         if (vectors_left) {
11761                 v_budget++;
11762                 vectors_left--;
11763         }
11764
11765         /* reserve some vectors for the main PF traffic queues. Initially we
11766          * only reserve at most 50% of the available vectors, in the case that
11767          * the number of online CPUs is large. This ensures that we can enable
11768          * extra features as well. Once we've enabled the other features, we
11769          * will use any remaining vectors to reach as close as we can to the
11770          * number of online CPUs.
11771          */
11772         cpus = num_online_cpus();
11773         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11774         vectors_left -= pf->num_lan_msix;
11775
11776         /* reserve one vector for sideband flow director */
11777         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11778                 if (vectors_left) {
11779                         pf->num_fdsb_msix = 1;
11780                         v_budget++;
11781                         vectors_left--;
11782                 } else {
11783                         pf->num_fdsb_msix = 0;
11784                 }
11785         }
11786
11787         /* can we reserve enough for iWARP? */
11788         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11789                 iwarp_requested = pf->num_iwarp_msix;
11790
11791                 if (!vectors_left)
11792                         pf->num_iwarp_msix = 0;
11793                 else if (vectors_left < pf->num_iwarp_msix)
11794                         pf->num_iwarp_msix = 1;
11795                 v_budget += pf->num_iwarp_msix;
11796                 vectors_left -= pf->num_iwarp_msix;
11797         }
11798
11799         /* any vectors left over go for VMDq support */
11800         if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
11801                 if (!vectors_left) {
11802                         pf->num_vmdq_msix = 0;
11803                         pf->num_vmdq_qps = 0;
11804                 } else {
11805                         int vmdq_vecs_wanted =
11806                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11807                         int vmdq_vecs =
11808                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11809
11810                         /* if we're short on vectors for what's desired, we limit
11811                          * the queues per vmdq.  If this is still more than are
11812                          * available, the user will need to change the number of
11813                          * queues/vectors used by the PF later with the ethtool
11814                          * channels command
11815                          */
11816                         if (vectors_left < vmdq_vecs_wanted) {
11817                                 pf->num_vmdq_qps = 1;
11818                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11819                                 vmdq_vecs = min_t(int,
11820                                                   vectors_left,
11821                                                   vmdq_vecs_wanted);
11822                         }
11823                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11824
11825                         v_budget += vmdq_vecs;
11826                         vectors_left -= vmdq_vecs;
11827                 }
11828         }
11829
11830         /* On systems with a large number of SMP cores, we previously limited
11831          * the number of vectors for num_lan_msix to be at most 50% of the
11832          * available vectors, to allow for other features. Now, we add back
11833          * the remaining vectors. However, we ensure that the total
11834          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11835          * calculate the number of vectors we can add without going over the
11836          * cap of CPUs. For systems with a small number of CPUs this will be
11837          * zero.
11838          */
11839         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11840         pf->num_lan_msix += extra_vectors;
11841         vectors_left -= extra_vectors;
11842
11843         WARN(vectors_left < 0,
11844              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11845
11846         v_budget += pf->num_lan_msix;
11847         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11848                                    GFP_KERNEL);
11849         if (!pf->msix_entries)
11850                 return -ENOMEM;
11851
11852         for (i = 0; i < v_budget; i++)
11853                 pf->msix_entries[i].entry = i;
11854         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11855
11856         if (v_actual < I40E_MIN_MSIX) {
11857                 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
11858                 kfree(pf->msix_entries);
11859                 pf->msix_entries = NULL;
11860                 pci_disable_msix(pf->pdev);
11861                 return -ENODEV;
11862
11863         } else if (v_actual == I40E_MIN_MSIX) {
11864                 /* Adjust for minimal MSIX use */
11865                 pf->num_vmdq_vsis = 0;
11866                 pf->num_vmdq_qps = 0;
11867                 pf->num_lan_qps = 1;
11868                 pf->num_lan_msix = 1;
11869
11870         } else if (v_actual != v_budget) {
11871                 /* If we have limited resources, we will start with no vectors
11872                  * for the special features and then allocate vectors to some
11873                  * of these features based on the policy and at the end disable
11874                  * the features that did not get any vectors.
11875                  */
11876                 int vec;
11877
11878                 dev_info(&pf->pdev->dev,
11879                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11880                          v_actual, v_budget);
11881                 /* reserve the misc vector */
11882                 vec = v_actual - 1;
11883
11884                 /* Scale vector usage down */
11885                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11886                 pf->num_vmdq_vsis = 1;
11887                 pf->num_vmdq_qps = 1;
11888
11889                 /* partition out the remaining vectors */
11890                 switch (vec) {
11891                 case 2:
11892                         pf->num_lan_msix = 1;
11893                         break;
11894                 case 3:
11895                         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11896                                 pf->num_lan_msix = 1;
11897                                 pf->num_iwarp_msix = 1;
11898                         } else {
11899                                 pf->num_lan_msix = 2;
11900                         }
11901                         break;
11902                 default:
11903                         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11904                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11905                                                  iwarp_requested);
11906                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11907                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11908                         } else {
11909                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11910                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11911                         }
11912                         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11913                                 pf->num_fdsb_msix = 1;
11914                                 vec--;
11915                         }
11916                         pf->num_lan_msix = min_t(int,
11917                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11918                                                               pf->num_lan_msix);
11919                         pf->num_lan_qps = pf->num_lan_msix;
11920                         break;
11921                 }
11922         }
11923
11924         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
11925                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11926                 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
11927                 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
11928         }
11929         if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
11930                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11931                 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
11932         }
11933
11934         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
11935             pf->num_iwarp_msix == 0) {
11936                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11937                 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
11938         }
11939         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11940                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11941                    pf->num_lan_msix,
11942                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11943                    pf->num_fdsb_msix,
11944                    pf->num_iwarp_msix);
11945
11946         return v_actual;
11947 }
11948
11949 /**
11950  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11951  * @vsi: the VSI being configured
11952  * @v_idx: index of the vector in the vsi struct
11953  *
11954  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11955  **/
11956 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11957 {
11958         struct i40e_q_vector *q_vector;
11959
11960         /* allocate q_vector */
11961         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11962         if (!q_vector)
11963                 return -ENOMEM;
11964
11965         q_vector->vsi = vsi;
11966         q_vector->v_idx = v_idx;
11967         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11968
11969         if (vsi->netdev)
11970                 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
11971
11972         /* tie q_vector and vsi together */
11973         vsi->q_vectors[v_idx] = q_vector;
11974
11975         return 0;
11976 }
11977
11978 /**
11979  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11980  * @vsi: the VSI being configured
11981  *
11982  * We allocate one q_vector per queue interrupt.  If allocation fails we
11983  * return -ENOMEM.
11984  **/
11985 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11986 {
11987         struct i40e_pf *pf = vsi->back;
11988         int err, v_idx, num_q_vectors;
11989
11990         /* if not MSIX, give the one vector only to the LAN VSI */
11991         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11992                 num_q_vectors = vsi->num_q_vectors;
11993         else if (vsi == pf->vsi[pf->lan_vsi])
11994                 num_q_vectors = 1;
11995         else
11996                 return -EINVAL;
11997
11998         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11999                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12000                 if (err)
12001                         goto err_out;
12002         }
12003
12004         return 0;
12005
12006 err_out:
12007         while (v_idx--)
12008                 i40e_free_q_vector(vsi, v_idx);
12009
12010         return err;
12011 }
12012
12013 /**
12014  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12015  * @pf: board private structure to initialize
12016  **/
12017 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12018 {
12019         int vectors = 0;
12020         ssize_t size;
12021
12022         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12023                 vectors = i40e_init_msix(pf);
12024                 if (vectors < 0) {
12025                         clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12026                         clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12027                         clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
12028                         clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
12029                         clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
12030                         clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12031                         clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12032                         clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12033                         clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12034                         set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12035
12036                         /* rework the queue expectations without MSIX */
12037                         i40e_determine_queue_usage(pf);
12038                 }
12039         }
12040
12041         if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
12042             test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
12043                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12044                 vectors = pci_enable_msi(pf->pdev);
12045                 if (vectors < 0) {
12046                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12047                                  vectors);
12048                         clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
12049                 }
12050                 vectors = 1;  /* one MSI or Legacy vector */
12051         }
12052
12053         if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
12054             !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12055                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12056
12057         /* set up vector assignment tracking */
12058         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12059         pf->irq_pile = kzalloc(size, GFP_KERNEL);
12060         if (!pf->irq_pile)
12061                 return -ENOMEM;
12062
12063         pf->irq_pile->num_entries = vectors;
12064
12065         /* track first vector for misc interrupts, ignore return */
12066         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12067
12068         return 0;
12069 }
12070
12071 /**
12072  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12073  * @pf: private board data structure
12074  *
12075  * Restore the interrupt scheme that was cleared when we suspended the
12076  * device. This should be called during resume to re-allocate the q_vectors
12077  * and reacquire IRQs.
12078  */
12079 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12080 {
12081         struct i40e_vsi *vsi;
12082         int err, i;
12083
12084         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
12085          * scheme. We need to re-enabled them here in order to attempt to
12086          * re-acquire the MSI or MSI-X vectors
12087          */
12088         set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12089         set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12090
12091         err = i40e_init_interrupt_scheme(pf);
12092         if (err)
12093                 return err;
12094
12095         /* Now that we've re-acquired IRQs, we need to remap the vectors and
12096          * rings together again.
12097          */
12098         i40e_pf_for_each_vsi(pf, i, vsi) {
12099                 err = i40e_vsi_alloc_q_vectors(vsi);
12100                 if (err)
12101                         goto err_unwind;
12102
12103                 i40e_vsi_map_rings_to_vectors(vsi);
12104         }
12105
12106         err = i40e_setup_misc_vector(pf);
12107         if (err)
12108                 goto err_unwind;
12109
12110         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
12111                 i40e_client_update_msix_info(pf);
12112
12113         return 0;
12114
12115 err_unwind:
12116         while (i--) {
12117                 if (pf->vsi[i])
12118                         i40e_vsi_free_q_vectors(pf->vsi[i]);
12119         }
12120
12121         return err;
12122 }
12123
12124 /**
12125  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12126  * non queue events in recovery mode
12127  * @pf: board private structure
12128  *
12129  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12130  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12131  * This is handled differently than in recovery mode since no Tx/Rx resources
12132  * are being allocated.
12133  **/
12134 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12135 {
12136         int err;
12137
12138         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12139                 err = i40e_setup_misc_vector(pf);
12140
12141                 if (err) {
12142                         dev_info(&pf->pdev->dev,
12143                                  "MSI-X misc vector request failed, error %d\n",
12144                                  err);
12145                         return err;
12146                 }
12147         } else {
12148                 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
12149
12150                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
12151                                   pf->int_name, pf);
12152
12153                 if (err) {
12154                         dev_info(&pf->pdev->dev,
12155                                  "MSI/legacy misc vector request failed, error %d\n",
12156                                  err);
12157                         return err;
12158                 }
12159                 i40e_enable_misc_int_causes(pf);
12160                 i40e_irq_dynamic_enable_icr0(pf);
12161         }
12162
12163         return 0;
12164 }
12165
12166 /**
12167  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12168  * @pf: board private structure
12169  *
12170  * This sets up the handler for MSIX 0, which is used to manage the
12171  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
12172  * when in MSI or Legacy interrupt mode.
12173  **/
12174 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12175 {
12176         struct i40e_hw *hw = &pf->hw;
12177         int err = 0;
12178
12179         /* Only request the IRQ once, the first time through. */
12180         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12181                 err = request_irq(pf->msix_entries[0].vector,
12182                                   i40e_intr, 0, pf->int_name, pf);
12183                 if (err) {
12184                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12185                         dev_info(&pf->pdev->dev,
12186                                  "request_irq for %s failed: %d\n",
12187                                  pf->int_name, err);
12188                         return -EFAULT;
12189                 }
12190         }
12191
12192         i40e_enable_misc_int_causes(pf);
12193
12194         /* associate no queues to the misc vector */
12195         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12196         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12197
12198         i40e_flush(hw);
12199
12200         i40e_irq_dynamic_enable_icr0(pf);
12201
12202         return err;
12203 }
12204
12205 /**
12206  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12207  * @vsi: Pointer to vsi structure
12208  * @seed: Buffter to store the hash keys
12209  * @lut: Buffer to store the lookup table entries
12210  * @lut_size: Size of buffer to store the lookup table entries
12211  *
12212  * Return 0 on success, negative on failure
12213  */
12214 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12215                            u8 *lut, u16 lut_size)
12216 {
12217         struct i40e_pf *pf = vsi->back;
12218         struct i40e_hw *hw = &pf->hw;
12219         int ret = 0;
12220
12221         if (seed) {
12222                 ret = i40e_aq_get_rss_key(hw, vsi->id,
12223                         (struct i40e_aqc_get_set_rss_key_data *)seed);
12224                 if (ret) {
12225                         dev_info(&pf->pdev->dev,
12226                                  "Cannot get RSS key, err %pe aq_err %s\n",
12227                                  ERR_PTR(ret),
12228                                  i40e_aq_str(&pf->hw,
12229                                              pf->hw.aq.asq_last_status));
12230                         return ret;
12231                 }
12232         }
12233
12234         if (lut) {
12235                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
12236
12237                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12238                 if (ret) {
12239                         dev_info(&pf->pdev->dev,
12240                                  "Cannot get RSS lut, err %pe aq_err %s\n",
12241                                  ERR_PTR(ret),
12242                                  i40e_aq_str(&pf->hw,
12243                                              pf->hw.aq.asq_last_status));
12244                         return ret;
12245                 }
12246         }
12247
12248         return ret;
12249 }
12250
12251 /**
12252  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12253  * @vsi: Pointer to vsi structure
12254  * @seed: RSS hash seed
12255  * @lut: Lookup table
12256  * @lut_size: Lookup table size
12257  *
12258  * Returns 0 on success, negative on failure
12259  **/
12260 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12261                                const u8 *lut, u16 lut_size)
12262 {
12263         struct i40e_pf *pf = vsi->back;
12264         struct i40e_hw *hw = &pf->hw;
12265         u16 vf_id = vsi->vf_id;
12266         u8 i;
12267
12268         /* Fill out hash function seed */
12269         if (seed) {
12270                 u32 *seed_dw = (u32 *)seed;
12271
12272                 if (vsi->type == I40E_VSI_MAIN) {
12273                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12274                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12275                 } else if (vsi->type == I40E_VSI_SRIOV) {
12276                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12277                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12278                 } else {
12279                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12280                 }
12281         }
12282
12283         if (lut) {
12284                 u32 *lut_dw = (u32 *)lut;
12285
12286                 if (vsi->type == I40E_VSI_MAIN) {
12287                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
12288                                 return -EINVAL;
12289                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12290                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12291                 } else if (vsi->type == I40E_VSI_SRIOV) {
12292                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12293                                 return -EINVAL;
12294                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12295                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12296                 } else {
12297                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12298                 }
12299         }
12300         i40e_flush(hw);
12301
12302         return 0;
12303 }
12304
12305 /**
12306  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12307  * @vsi: Pointer to VSI structure
12308  * @seed: Buffer to store the keys
12309  * @lut: Buffer to store the lookup table entries
12310  * @lut_size: Size of buffer to store the lookup table entries
12311  *
12312  * Returns 0 on success, negative on failure
12313  */
12314 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12315                             u8 *lut, u16 lut_size)
12316 {
12317         struct i40e_pf *pf = vsi->back;
12318         struct i40e_hw *hw = &pf->hw;
12319         u16 i;
12320
12321         if (seed) {
12322                 u32 *seed_dw = (u32 *)seed;
12323
12324                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12325                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12326         }
12327         if (lut) {
12328                 u32 *lut_dw = (u32 *)lut;
12329
12330                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12331                         return -EINVAL;
12332                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12333                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12334         }
12335
12336         return 0;
12337 }
12338
12339 /**
12340  * i40e_config_rss - Configure RSS keys and lut
12341  * @vsi: Pointer to VSI structure
12342  * @seed: RSS hash seed
12343  * @lut: Lookup table
12344  * @lut_size: Lookup table size
12345  *
12346  * Returns 0 on success, negative on failure
12347  */
12348 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12349 {
12350         struct i40e_pf *pf = vsi->back;
12351
12352         if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12353                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12354         else
12355                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12356 }
12357
12358 /**
12359  * i40e_get_rss - Get RSS keys and lut
12360  * @vsi: Pointer to VSI structure
12361  * @seed: Buffer to store the keys
12362  * @lut: Buffer to store the lookup table entries
12363  * @lut_size: Size of buffer to store the lookup table entries
12364  *
12365  * Returns 0 on success, negative on failure
12366  */
12367 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12368 {
12369         struct i40e_pf *pf = vsi->back;
12370
12371         if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12372                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12373         else
12374                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12375 }
12376
12377 /**
12378  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12379  * @pf: Pointer to board private structure
12380  * @lut: Lookup table
12381  * @rss_table_size: Lookup table size
12382  * @rss_size: Range of queue number for hashing
12383  */
12384 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12385                        u16 rss_table_size, u16 rss_size)
12386 {
12387         u16 i;
12388
12389         for (i = 0; i < rss_table_size; i++)
12390                 lut[i] = i % rss_size;
12391 }
12392
12393 /**
12394  * i40e_pf_config_rss - Prepare for RSS if used
12395  * @pf: board private structure
12396  **/
12397 static int i40e_pf_config_rss(struct i40e_pf *pf)
12398 {
12399         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12400         u8 seed[I40E_HKEY_ARRAY_SIZE];
12401         u8 *lut;
12402         struct i40e_hw *hw = &pf->hw;
12403         u32 reg_val;
12404         u64 hena;
12405         int ret;
12406
12407         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12408         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12409                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12410         hena |= i40e_pf_get_default_rss_hena(pf);
12411
12412         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12413         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12414
12415         /* Determine the RSS table size based on the hardware capabilities */
12416         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12417         reg_val = (pf->rss_table_size == 512) ?
12418                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12419                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12420         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12421
12422         /* Determine the RSS size of the VSI */
12423         if (!vsi->rss_size) {
12424                 u16 qcount;
12425                 /* If the firmware does something weird during VSI init, we
12426                  * could end up with zero TCs. Check for that to avoid
12427                  * divide-by-zero. It probably won't pass traffic, but it also
12428                  * won't panic.
12429                  */
12430                 qcount = vsi->num_queue_pairs /
12431                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12432                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12433         }
12434         if (!vsi->rss_size)
12435                 return -EINVAL;
12436
12437         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12438         if (!lut)
12439                 return -ENOMEM;
12440
12441         /* Use user configured lut if there is one, otherwise use default */
12442         if (vsi->rss_lut_user)
12443                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12444         else
12445                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12446
12447         /* Use user configured hash key if there is one, otherwise
12448          * use default.
12449          */
12450         if (vsi->rss_hkey_user)
12451                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12452         else
12453                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12454         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12455         kfree(lut);
12456
12457         return ret;
12458 }
12459
12460 /**
12461  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12462  * @pf: board private structure
12463  * @queue_count: the requested queue count for rss.
12464  *
12465  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12466  * count which may be different from the requested queue count.
12467  * Note: expects to be called while under rtnl_lock()
12468  **/
12469 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12470 {
12471         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12472         int new_rss_size;
12473
12474         if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
12475                 return 0;
12476
12477         queue_count = min_t(int, queue_count, num_online_cpus());
12478         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12479
12480         if (queue_count != vsi->num_queue_pairs) {
12481                 u16 qcount;
12482
12483                 vsi->req_queue_pairs = queue_count;
12484                 i40e_prep_for_reset(pf);
12485                 if (test_bit(__I40E_IN_REMOVE, pf->state))
12486                         return pf->alloc_rss_size;
12487
12488                 pf->alloc_rss_size = new_rss_size;
12489
12490                 i40e_reset_and_rebuild(pf, true, true);
12491
12492                 /* Discard the user configured hash keys and lut, if less
12493                  * queues are enabled.
12494                  */
12495                 if (queue_count < vsi->rss_size) {
12496                         i40e_clear_rss_config_user(vsi);
12497                         dev_dbg(&pf->pdev->dev,
12498                                 "discard user configured hash keys and lut\n");
12499                 }
12500
12501                 /* Reset vsi->rss_size, as number of enabled queues changed */
12502                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12503                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12504
12505                 i40e_pf_config_rss(pf);
12506         }
12507         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12508                  vsi->req_queue_pairs, pf->rss_size_max);
12509         return pf->alloc_rss_size;
12510 }
12511
12512 /**
12513  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12514  * @pf: board private structure
12515  **/
12516 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12517 {
12518         bool min_valid, max_valid;
12519         u32 max_bw, min_bw;
12520         int status;
12521
12522         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12523                                            &min_valid, &max_valid);
12524
12525         if (!status) {
12526                 if (min_valid)
12527                         pf->min_bw = min_bw;
12528                 if (max_valid)
12529                         pf->max_bw = max_bw;
12530         }
12531
12532         return status;
12533 }
12534
12535 /**
12536  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12537  * @pf: board private structure
12538  **/
12539 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12540 {
12541         struct i40e_aqc_configure_partition_bw_data bw_data;
12542         int status;
12543
12544         memset(&bw_data, 0, sizeof(bw_data));
12545
12546         /* Set the valid bit for this PF */
12547         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12548         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12549         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12550
12551         /* Set the new bandwidths */
12552         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12553
12554         return status;
12555 }
12556
12557 /**
12558  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12559  * @pf: board private structure
12560  **/
12561 int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12562 {
12563         /* Commit temporary BW setting to permanent NVM image */
12564         enum i40e_admin_queue_err last_aq_status;
12565         u16 nvm_word;
12566         int ret;
12567
12568         if (pf->hw.partition_id != 1) {
12569                 dev_info(&pf->pdev->dev,
12570                          "Commit BW only works on partition 1! This is partition %d",
12571                          pf->hw.partition_id);
12572                 ret = -EOPNOTSUPP;
12573                 goto bw_commit_out;
12574         }
12575
12576         /* Acquire NVM for read access */
12577         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12578         last_aq_status = pf->hw.aq.asq_last_status;
12579         if (ret) {
12580                 dev_info(&pf->pdev->dev,
12581                          "Cannot acquire NVM for read access, err %pe aq_err %s\n",
12582                          ERR_PTR(ret),
12583                          i40e_aq_str(&pf->hw, last_aq_status));
12584                 goto bw_commit_out;
12585         }
12586
12587         /* Read word 0x10 of NVM - SW compatibility word 1 */
12588         ret = i40e_aq_read_nvm(&pf->hw,
12589                                I40E_SR_NVM_CONTROL_WORD,
12590                                0x10, sizeof(nvm_word), &nvm_word,
12591                                false, NULL);
12592         /* Save off last admin queue command status before releasing
12593          * the NVM
12594          */
12595         last_aq_status = pf->hw.aq.asq_last_status;
12596         i40e_release_nvm(&pf->hw);
12597         if (ret) {
12598                 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",
12599                          ERR_PTR(ret),
12600                          i40e_aq_str(&pf->hw, last_aq_status));
12601                 goto bw_commit_out;
12602         }
12603
12604         /* Wait a bit for NVM release to complete */
12605         msleep(50);
12606
12607         /* Acquire NVM for write access */
12608         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12609         last_aq_status = pf->hw.aq.asq_last_status;
12610         if (ret) {
12611                 dev_info(&pf->pdev->dev,
12612                          "Cannot acquire NVM for write access, err %pe aq_err %s\n",
12613                          ERR_PTR(ret),
12614                          i40e_aq_str(&pf->hw, last_aq_status));
12615                 goto bw_commit_out;
12616         }
12617         /* Write it back out unchanged to initiate update NVM,
12618          * which will force a write of the shadow (alt) RAM to
12619          * the NVM - thus storing the bandwidth values permanently.
12620          */
12621         ret = i40e_aq_update_nvm(&pf->hw,
12622                                  I40E_SR_NVM_CONTROL_WORD,
12623                                  0x10, sizeof(nvm_word),
12624                                  &nvm_word, true, 0, NULL);
12625         /* Save off last admin queue command status before releasing
12626          * the NVM
12627          */
12628         last_aq_status = pf->hw.aq.asq_last_status;
12629         i40e_release_nvm(&pf->hw);
12630         if (ret)
12631                 dev_info(&pf->pdev->dev,
12632                          "BW settings NOT SAVED, err %pe aq_err %s\n",
12633                          ERR_PTR(ret),
12634                          i40e_aq_str(&pf->hw, last_aq_status));
12635 bw_commit_out:
12636
12637         return ret;
12638 }
12639
12640 /**
12641  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12642  * if total port shutdown feature is enabled for this PF
12643  * @pf: board private structure
12644  **/
12645 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12646 {
12647 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED        BIT(4)
12648 #define I40E_FEATURES_ENABLE_PTR                0x2A
12649 #define I40E_CURRENT_SETTING_PTR                0x2B
12650 #define I40E_LINK_BEHAVIOR_WORD_OFFSET          0x2D
12651 #define I40E_LINK_BEHAVIOR_WORD_LENGTH          0x1
12652 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED    BIT(0)
12653 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH      4
12654         u16 sr_emp_sr_settings_ptr = 0;
12655         u16 features_enable = 0;
12656         u16 link_behavior = 0;
12657         int read_status = 0;
12658         bool ret = false;
12659
12660         read_status = i40e_read_nvm_word(&pf->hw,
12661                                          I40E_SR_EMP_SR_SETTINGS_PTR,
12662                                          &sr_emp_sr_settings_ptr);
12663         if (read_status)
12664                 goto err_nvm;
12665         read_status = i40e_read_nvm_word(&pf->hw,
12666                                          sr_emp_sr_settings_ptr +
12667                                          I40E_FEATURES_ENABLE_PTR,
12668                                          &features_enable);
12669         if (read_status)
12670                 goto err_nvm;
12671         if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12672                 read_status = i40e_read_nvm_module_data(&pf->hw,
12673                                                         I40E_SR_EMP_SR_SETTINGS_PTR,
12674                                                         I40E_CURRENT_SETTING_PTR,
12675                                                         I40E_LINK_BEHAVIOR_WORD_OFFSET,
12676                                                         I40E_LINK_BEHAVIOR_WORD_LENGTH,
12677                                                         &link_behavior);
12678                 if (read_status)
12679                         goto err_nvm;
12680                 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12681                 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12682         }
12683         return ret;
12684
12685 err_nvm:
12686         dev_warn(&pf->pdev->dev,
12687                  "total-port-shutdown feature is off due to read nvm error: %pe\n",
12688                  ERR_PTR(read_status));
12689         return ret;
12690 }
12691
12692 /**
12693  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12694  * @pf: board private structure to initialize
12695  *
12696  * i40e_sw_init initializes the Adapter private data structure.
12697  * Fields are initialized based on PCI device information and
12698  * OS network device settings (MTU size).
12699  **/
12700 static int i40e_sw_init(struct i40e_pf *pf)
12701 {
12702         int err = 0;
12703         int size;
12704         u16 pow;
12705
12706         /* Set default capability flags */
12707         bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);
12708         set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12709         set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12710
12711         /* Set default ITR */
12712         pf->rx_itr_default = I40E_ITR_RX_DEF;
12713         pf->tx_itr_default = I40E_ITR_TX_DEF;
12714
12715         /* Depending on PF configurations, it is possible that the RSS
12716          * maximum might end up larger than the available queues
12717          */
12718         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12719         pf->alloc_rss_size = 1;
12720         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12721         pf->rss_size_max = min_t(int, pf->rss_size_max,
12722                                  pf->hw.func_caps.num_tx_qp);
12723
12724         /* find the next higher power-of-2 of num cpus */
12725         pow = roundup_pow_of_two(num_online_cpus());
12726         pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12727
12728         if (pf->hw.func_caps.rss) {
12729                 set_bit(I40E_FLAG_RSS_ENA, pf->flags);
12730                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12731                                            num_online_cpus());
12732         }
12733
12734         /* MFP mode enabled */
12735         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12736                 set_bit(I40E_FLAG_MFP_ENA, pf->flags);
12737                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12738                 if (i40e_get_partition_bw_setting(pf)) {
12739                         dev_warn(&pf->pdev->dev,
12740                                  "Could not get partition bw settings\n");
12741                 } else {
12742                         dev_info(&pf->pdev->dev,
12743                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
12744                                  pf->min_bw, pf->max_bw);
12745
12746                         /* nudge the Tx scheduler */
12747                         i40e_set_partition_bw_setting(pf);
12748                 }
12749         }
12750
12751         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12752             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12753                 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12754                 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
12755                     pf->hw.num_partitions > 1)
12756                         dev_info(&pf->pdev->dev,
12757                                  "Flow Director Sideband mode Disabled in MFP mode\n");
12758                 else
12759                         set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12760                 pf->fdir_pf_filter_count =
12761                                  pf->hw.func_caps.fd_filters_guaranteed;
12762                 pf->hw.fdir_shared_filter_count =
12763                                  pf->hw.func_caps.fd_filters_best_effort;
12764         }
12765
12766         /* Enable HW ATR eviction if possible */
12767         if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
12768                 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);
12769
12770         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12771                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12772                 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12773                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12774         }
12775
12776         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12777                 set_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12778                 /* IWARP needs one extra vector for CQP just like MISC.*/
12779                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12780         }
12781         /* Stopping FW LLDP engine is supported on XL710 and X722
12782          * starting from FW versions determined in i40e_init_adminq.
12783          * Stopping the FW LLDP engine is not supported on XL710
12784          * if NPAR is functioning so unset this hw flag in this case.
12785          */
12786         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12787             pf->hw.func_caps.npar_enable)
12788                 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);
12789
12790 #ifdef CONFIG_PCI_IOV
12791         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12792                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12793                 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12794                 pf->num_req_vfs = min_t(int,
12795                                         pf->hw.func_caps.num_vfs,
12796                                         I40E_MAX_VF_COUNT);
12797         }
12798 #endif /* CONFIG_PCI_IOV */
12799         pf->lan_veb = I40E_NO_VEB;
12800         pf->lan_vsi = I40E_NO_VSI;
12801
12802         /* By default FW has this off for performance reasons */
12803         clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
12804
12805         /* set up queue assignment tracking */
12806         size = sizeof(struct i40e_lump_tracking)
12807                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12808         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12809         if (!pf->qp_pile) {
12810                 err = -ENOMEM;
12811                 goto sw_init_done;
12812         }
12813         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12814
12815         pf->tx_timeout_recovery_level = 1;
12816
12817         if (pf->hw.mac.type != I40E_MAC_X722 &&
12818             i40e_is_total_port_shutdown_enabled(pf)) {
12819                 /* Link down on close must be on when total port shutdown
12820                  * is enabled for a given port
12821                  */
12822                 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
12823                 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
12824                 dev_info(&pf->pdev->dev,
12825                          "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12826         }
12827         mutex_init(&pf->switch_mutex);
12828
12829 sw_init_done:
12830         return err;
12831 }
12832
12833 /**
12834  * i40e_set_ntuple - set the ntuple feature flag and take action
12835  * @pf: board private structure to initialize
12836  * @features: the feature set that the stack is suggesting
12837  *
12838  * returns a bool to indicate if reset needs to happen
12839  **/
12840 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12841 {
12842         bool need_reset = false;
12843
12844         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12845          * the state changed, we need to reset.
12846          */
12847         if (features & NETIF_F_NTUPLE) {
12848                 /* Enable filters and mark for reset */
12849                 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
12850                         need_reset = true;
12851                 /* enable FD_SB only if there is MSI-X vector and no cloud
12852                  * filters exist
12853                  */
12854                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12855                         set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12856                         clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12857                 }
12858         } else {
12859                 /* turn off filters, mark for reset and clear SW filter list */
12860                 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12861                         need_reset = true;
12862                         i40e_fdir_filter_exit(pf);
12863                 }
12864                 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12865                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12866                 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12867
12868                 /* reset fd counters */
12869                 pf->fd_add_err = 0;
12870                 pf->fd_atr_cnt = 0;
12871                 /* if ATR was auto disabled it can be re-enabled. */
12872                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12873                         if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
12874                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12875                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12876         }
12877         return need_reset;
12878 }
12879
12880 /**
12881  * i40e_clear_rss_lut - clear the rx hash lookup table
12882  * @vsi: the VSI being configured
12883  **/
12884 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12885 {
12886         struct i40e_pf *pf = vsi->back;
12887         struct i40e_hw *hw = &pf->hw;
12888         u16 vf_id = vsi->vf_id;
12889         u8 i;
12890
12891         if (vsi->type == I40E_VSI_MAIN) {
12892                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12893                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12894         } else if (vsi->type == I40E_VSI_SRIOV) {
12895                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12896                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12897         } else {
12898                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12899         }
12900 }
12901
12902 /**
12903  * i40e_set_loopback - turn on/off loopback mode on underlying PF
12904  * @vsi: ptr to VSI
12905  * @ena: flag to indicate the on/off setting
12906  */
12907 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12908 {
12909         bool if_running = netif_running(vsi->netdev) &&
12910                           !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);
12911         int ret;
12912
12913         if (if_running)
12914                 i40e_down(vsi);
12915
12916         ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
12917         if (ret)
12918                 netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
12919         if (if_running)
12920                 i40e_up(vsi);
12921
12922         return ret;
12923 }
12924
12925 /**
12926  * i40e_set_features - set the netdev feature flags
12927  * @netdev: ptr to the netdev being adjusted
12928  * @features: the feature set that the stack is suggesting
12929  * Note: expects to be called while under rtnl_lock()
12930  **/
12931 static int i40e_set_features(struct net_device *netdev,
12932                              netdev_features_t features)
12933 {
12934         struct i40e_netdev_priv *np = netdev_priv(netdev);
12935         struct i40e_vsi *vsi = np->vsi;
12936         struct i40e_pf *pf = vsi->back;
12937         bool need_reset;
12938
12939         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12940                 i40e_pf_config_rss(pf);
12941         else if (!(features & NETIF_F_RXHASH) &&
12942                  netdev->features & NETIF_F_RXHASH)
12943                 i40e_clear_rss_lut(vsi);
12944
12945         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12946                 i40e_vlan_stripping_enable(vsi);
12947         else
12948                 i40e_vlan_stripping_disable(vsi);
12949
12950         if (!(features & NETIF_F_HW_TC) &&
12951             (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12952                 dev_err(&pf->pdev->dev,
12953                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12954                 return -EINVAL;
12955         }
12956
12957         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12958                 i40e_del_all_macvlans(vsi);
12959
12960         need_reset = i40e_set_ntuple(pf, features);
12961
12962         if (need_reset)
12963                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12964
12965         if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
12966                 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
12967
12968         return 0;
12969 }
12970
12971 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12972                                     unsigned int table, unsigned int idx,
12973                                     struct udp_tunnel_info *ti)
12974 {
12975         struct i40e_netdev_priv *np = netdev_priv(netdev);
12976         struct i40e_hw *hw = &np->vsi->back->hw;
12977         u8 type, filter_index;
12978         int ret;
12979
12980         type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12981                                                    I40E_AQC_TUNNEL_TYPE_NGE;
12982
12983         ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
12984                                      NULL);
12985         if (ret) {
12986                 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",
12987                             ERR_PTR(ret),
12988                             i40e_aq_str(hw, hw->aq.asq_last_status));
12989                 return -EIO;
12990         }
12991
12992         udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
12993         return 0;
12994 }
12995
12996 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12997                                       unsigned int table, unsigned int idx,
12998                                       struct udp_tunnel_info *ti)
12999 {
13000         struct i40e_netdev_priv *np = netdev_priv(netdev);
13001         struct i40e_hw *hw = &np->vsi->back->hw;
13002         int ret;
13003
13004         ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13005         if (ret) {
13006                 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",
13007                             ERR_PTR(ret),
13008                             i40e_aq_str(hw, hw->aq.asq_last_status));
13009                 return -EIO;
13010         }
13011
13012         return 0;
13013 }
13014
13015 static int i40e_get_phys_port_id(struct net_device *netdev,
13016                                  struct netdev_phys_item_id *ppid)
13017 {
13018         struct i40e_netdev_priv *np = netdev_priv(netdev);
13019         struct i40e_pf *pf = np->vsi->back;
13020         struct i40e_hw *hw = &pf->hw;
13021
13022         if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
13023                 return -EOPNOTSUPP;
13024
13025         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13026         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13027
13028         return 0;
13029 }
13030
13031 /**
13032  * i40e_ndo_fdb_add - add an entry to the hardware database
13033  * @ndm: the input from the stack
13034  * @tb: pointer to array of nladdr (unused)
13035  * @dev: the net device pointer
13036  * @addr: the MAC address entry being added
13037  * @vid: VLAN ID
13038  * @flags: instructions from stack about fdb operation
13039  * @extack: netlink extended ack, unused currently
13040  */
13041 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13042                             struct net_device *dev,
13043                             const unsigned char *addr, u16 vid,
13044                             u16 flags,
13045                             struct netlink_ext_ack *extack)
13046 {
13047         struct i40e_netdev_priv *np = netdev_priv(dev);
13048         struct i40e_pf *pf = np->vsi->back;
13049         int err = 0;
13050
13051         if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
13052                 return -EOPNOTSUPP;
13053
13054         if (vid) {
13055                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13056                 return -EINVAL;
13057         }
13058
13059         /* Hardware does not support aging addresses so if a
13060          * ndm_state is given only allow permanent addresses
13061          */
13062         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13063                 netdev_info(dev, "FDB only supports static addresses\n");
13064                 return -EINVAL;
13065         }
13066
13067         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13068                 err = dev_uc_add_excl(dev, addr);
13069         else if (is_multicast_ether_addr(addr))
13070                 err = dev_mc_add_excl(dev, addr);
13071         else
13072                 err = -EINVAL;
13073
13074         /* Only return duplicate errors if NLM_F_EXCL is set */
13075         if (err == -EEXIST && !(flags & NLM_F_EXCL))
13076                 err = 0;
13077
13078         return err;
13079 }
13080
13081 /**
13082  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13083  * @dev: the netdev being configured
13084  * @nlh: RTNL message
13085  * @flags: bridge flags
13086  * @extack: netlink extended ack
13087  *
13088  * Inserts a new hardware bridge if not already created and
13089  * enables the bridging mode requested (VEB or VEPA). If the
13090  * hardware bridge has already been inserted and the request
13091  * is to change the mode then that requires a PF reset to
13092  * allow rebuild of the components with required hardware
13093  * bridge mode enabled.
13094  *
13095  * Note: expects to be called while under rtnl_lock()
13096  **/
13097 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13098                                    struct nlmsghdr *nlh,
13099                                    u16 flags,
13100                                    struct netlink_ext_ack *extack)
13101 {
13102         struct i40e_netdev_priv *np = netdev_priv(dev);
13103         struct i40e_vsi *vsi = np->vsi;
13104         struct i40e_pf *pf = vsi->back;
13105         struct nlattr *attr, *br_spec;
13106         struct i40e_veb *veb;
13107         int rem;
13108
13109         /* Only for PF VSI for now */
13110         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13111                 return -EOPNOTSUPP;
13112
13113         /* Find the HW bridge for PF VSI */
13114         veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13115
13116         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13117         if (!br_spec)
13118                 return -EINVAL;
13119
13120         nla_for_each_nested(attr, br_spec, rem) {
13121                 __u16 mode;
13122
13123                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
13124                         continue;
13125
13126                 mode = nla_get_u16(attr);
13127                 if ((mode != BRIDGE_MODE_VEPA) &&
13128                     (mode != BRIDGE_MODE_VEB))
13129                         return -EINVAL;
13130
13131                 /* Insert a new HW bridge */
13132                 if (!veb) {
13133                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13134                                              vsi->tc_config.enabled_tc);
13135                         if (veb) {
13136                                 veb->bridge_mode = mode;
13137                                 i40e_config_bridge_mode(veb);
13138                         } else {
13139                                 /* No Bridge HW offload available */
13140                                 return -ENOENT;
13141                         }
13142                         break;
13143                 } else if (mode != veb->bridge_mode) {
13144                         /* Existing HW bridge but different mode needs reset */
13145                         veb->bridge_mode = mode;
13146                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13147                         if (mode == BRIDGE_MODE_VEB)
13148                                 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13149                         else
13150                                 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13151                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13152                         break;
13153                 }
13154         }
13155
13156         return 0;
13157 }
13158
13159 /**
13160  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13161  * @skb: skb buff
13162  * @pid: process id
13163  * @seq: RTNL message seq #
13164  * @dev: the netdev being configured
13165  * @filter_mask: unused
13166  * @nlflags: netlink flags passed in
13167  *
13168  * Return the mode in which the hardware bridge is operating in
13169  * i.e VEB or VEPA.
13170  **/
13171 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13172                                    struct net_device *dev,
13173                                    u32 __always_unused filter_mask,
13174                                    int nlflags)
13175 {
13176         struct i40e_netdev_priv *np = netdev_priv(dev);
13177         struct i40e_vsi *vsi = np->vsi;
13178         struct i40e_pf *pf = vsi->back;
13179         struct i40e_veb *veb;
13180
13181         /* Only for PF VSI for now */
13182         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13183                 return -EOPNOTSUPP;
13184
13185         /* Find the HW bridge for the PF VSI */
13186         veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13187         if (!veb)
13188                 return 0;
13189
13190         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13191                                        0, 0, nlflags, filter_mask, NULL);
13192 }
13193
13194 /**
13195  * i40e_features_check - Validate encapsulated packet conforms to limits
13196  * @skb: skb buff
13197  * @dev: This physical port's netdev
13198  * @features: Offload features that the stack believes apply
13199  **/
13200 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13201                                              struct net_device *dev,
13202                                              netdev_features_t features)
13203 {
13204         size_t len;
13205
13206         /* No point in doing any of this if neither checksum nor GSO are
13207          * being requested for this frame.  We can rule out both by just
13208          * checking for CHECKSUM_PARTIAL
13209          */
13210         if (skb->ip_summed != CHECKSUM_PARTIAL)
13211                 return features;
13212
13213         /* We cannot support GSO if the MSS is going to be less than
13214          * 64 bytes.  If it is then we need to drop support for GSO.
13215          */
13216         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13217                 features &= ~NETIF_F_GSO_MASK;
13218
13219         /* MACLEN can support at most 63 words */
13220         len = skb_network_offset(skb);
13221         if (len & ~(63 * 2))
13222                 goto out_err;
13223
13224         /* IPLEN and EIPLEN can support at most 127 dwords */
13225         len = skb_network_header_len(skb);
13226         if (len & ~(127 * 4))
13227                 goto out_err;
13228
13229         if (skb->encapsulation) {
13230                 /* L4TUNLEN can support 127 words */
13231                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
13232                 if (len & ~(127 * 2))
13233                         goto out_err;
13234
13235                 /* IPLEN can support at most 127 dwords */
13236                 len = skb_inner_transport_header(skb) -
13237                       skb_inner_network_header(skb);
13238                 if (len & ~(127 * 4))
13239                         goto out_err;
13240         }
13241
13242         /* No need to validate L4LEN as TCP is the only protocol with a
13243          * flexible value and we support all possible values supported
13244          * by TCP, which is at most 15 dwords
13245          */
13246
13247         return features;
13248 out_err:
13249         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13250 }
13251
13252 /**
13253  * i40e_xdp_setup - add/remove an XDP program
13254  * @vsi: VSI to changed
13255  * @prog: XDP program
13256  * @extack: netlink extended ack
13257  **/
13258 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13259                           struct netlink_ext_ack *extack)
13260 {
13261         int frame_size = i40e_max_vsi_frame_size(vsi, prog);
13262         struct i40e_pf *pf = vsi->back;
13263         struct bpf_prog *old_prog;
13264         bool need_reset;
13265         int i;
13266
13267         /* Don't allow frames that span over multiple buffers */
13268         if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13269                 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13270                 return -EINVAL;
13271         }
13272
13273         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
13274         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13275
13276         if (need_reset)
13277                 i40e_prep_for_reset(pf);
13278
13279         /* VSI shall be deleted in a moment, just return EINVAL */
13280         if (test_bit(__I40E_IN_REMOVE, pf->state))
13281                 return -EINVAL;
13282
13283         old_prog = xchg(&vsi->xdp_prog, prog);
13284
13285         if (need_reset) {
13286                 if (!prog) {
13287                         xdp_features_clear_redirect_target(vsi->netdev);
13288                         /* Wait until ndo_xsk_wakeup completes. */
13289                         synchronize_rcu();
13290                 }
13291                 i40e_reset_and_rebuild(pf, true, true);
13292         }
13293
13294         if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13295                 if (i40e_realloc_rx_bi_zc(vsi, true))
13296                         return -ENOMEM;
13297         } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13298                 if (i40e_realloc_rx_bi_zc(vsi, false))
13299                         return -ENOMEM;
13300         }
13301
13302         for (i = 0; i < vsi->num_queue_pairs; i++)
13303                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13304
13305         if (old_prog)
13306                 bpf_prog_put(old_prog);
13307
13308         /* Kick start the NAPI context if there is an AF_XDP socket open
13309          * on that queue id. This so that receiving will start.
13310          */
13311         if (need_reset && prog) {
13312                 for (i = 0; i < vsi->num_queue_pairs; i++)
13313                         if (vsi->xdp_rings[i]->xsk_pool)
13314                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
13315                                                       XDP_WAKEUP_RX);
13316                 xdp_features_set_redirect_target(vsi->netdev, true);
13317         }
13318
13319         return 0;
13320 }
13321
13322 /**
13323  * i40e_enter_busy_conf - Enters busy config state
13324  * @vsi: vsi
13325  *
13326  * Returns 0 on success, <0 for failure.
13327  **/
13328 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13329 {
13330         struct i40e_pf *pf = vsi->back;
13331         int timeout = 50;
13332
13333         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13334                 timeout--;
13335                 if (!timeout)
13336                         return -EBUSY;
13337                 usleep_range(1000, 2000);
13338         }
13339
13340         return 0;
13341 }
13342
13343 /**
13344  * i40e_exit_busy_conf - Exits busy config state
13345  * @vsi: vsi
13346  **/
13347 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13348 {
13349         struct i40e_pf *pf = vsi->back;
13350
13351         clear_bit(__I40E_CONFIG_BUSY, pf->state);
13352 }
13353
13354 /**
13355  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13356  * @vsi: vsi
13357  * @queue_pair: queue pair
13358  **/
13359 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13360 {
13361         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13362                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13363         memset(&vsi->tx_rings[queue_pair]->stats, 0,
13364                sizeof(vsi->tx_rings[queue_pair]->stats));
13365         if (i40e_enabled_xdp_vsi(vsi)) {
13366                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13367                        sizeof(vsi->xdp_rings[queue_pair]->stats));
13368         }
13369 }
13370
13371 /**
13372  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13373  * @vsi: vsi
13374  * @queue_pair: queue pair
13375  **/
13376 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13377 {
13378         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13379         if (i40e_enabled_xdp_vsi(vsi)) {
13380                 /* Make sure that in-progress ndo_xdp_xmit calls are
13381                  * completed.
13382                  */
13383                 synchronize_rcu();
13384                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13385         }
13386         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13387 }
13388
13389 /**
13390  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13391  * @vsi: vsi
13392  * @queue_pair: queue pair
13393  * @enable: true for enable, false for disable
13394  **/
13395 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13396                                         bool enable)
13397 {
13398         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13399         struct i40e_q_vector *q_vector = rxr->q_vector;
13400
13401         if (!vsi->netdev)
13402                 return;
13403
13404         /* All rings in a qp belong to the same qvector. */
13405         if (q_vector->rx.ring || q_vector->tx.ring) {
13406                 if (enable)
13407                         napi_enable(&q_vector->napi);
13408                 else
13409                         napi_disable(&q_vector->napi);
13410         }
13411 }
13412
13413 /**
13414  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13415  * @vsi: vsi
13416  * @queue_pair: queue pair
13417  * @enable: true for enable, false for disable
13418  *
13419  * Returns 0 on success, <0 on failure.
13420  **/
13421 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13422                                         bool enable)
13423 {
13424         struct i40e_pf *pf = vsi->back;
13425         int pf_q, ret = 0;
13426
13427         pf_q = vsi->base_queue + queue_pair;
13428         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13429                                      false /*is xdp*/, enable);
13430         if (ret) {
13431                 dev_info(&pf->pdev->dev,
13432                          "VSI seid %d Tx ring %d %sable timeout\n",
13433                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13434                 return ret;
13435         }
13436
13437         i40e_control_rx_q(pf, pf_q, enable);
13438         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13439         if (ret) {
13440                 dev_info(&pf->pdev->dev,
13441                          "VSI seid %d Rx ring %d %sable timeout\n",
13442                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13443                 return ret;
13444         }
13445
13446         /* Due to HW errata, on Rx disable only, the register can
13447          * indicate done before it really is. Needs 50ms to be sure
13448          */
13449         if (!enable)
13450                 mdelay(50);
13451
13452         if (!i40e_enabled_xdp_vsi(vsi))
13453                 return ret;
13454
13455         ret = i40e_control_wait_tx_q(vsi->seid, pf,
13456                                      pf_q + vsi->alloc_queue_pairs,
13457                                      true /*is xdp*/, enable);
13458         if (ret) {
13459                 dev_info(&pf->pdev->dev,
13460                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
13461                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13462         }
13463
13464         return ret;
13465 }
13466
13467 /**
13468  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13469  * @vsi: vsi
13470  * @queue_pair: queue_pair
13471  **/
13472 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13473 {
13474         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13475         struct i40e_pf *pf = vsi->back;
13476         struct i40e_hw *hw = &pf->hw;
13477
13478         /* All rings in a qp belong to the same qvector. */
13479         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
13480                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13481         else
13482                 i40e_irq_dynamic_enable_icr0(pf);
13483
13484         i40e_flush(hw);
13485 }
13486
13487 /**
13488  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13489  * @vsi: vsi
13490  * @queue_pair: queue_pair
13491  **/
13492 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13493 {
13494         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13495         struct i40e_pf *pf = vsi->back;
13496         struct i40e_hw *hw = &pf->hw;
13497
13498         /* For simplicity, instead of removing the qp interrupt causes
13499          * from the interrupt linked list, we simply disable the interrupt, and
13500          * leave the list intact.
13501          *
13502          * All rings in a qp belong to the same qvector.
13503          */
13504         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
13505                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13506
13507                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13508                 i40e_flush(hw);
13509                 synchronize_irq(pf->msix_entries[intpf].vector);
13510         } else {
13511                 /* Legacy and MSI mode - this stops all interrupt handling */
13512                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13513                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13514                 i40e_flush(hw);
13515                 synchronize_irq(pf->pdev->irq);
13516         }
13517 }
13518
13519 /**
13520  * i40e_queue_pair_disable - Disables a queue pair
13521  * @vsi: vsi
13522  * @queue_pair: queue pair
13523  *
13524  * Returns 0 on success, <0 on failure.
13525  **/
13526 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13527 {
13528         int err;
13529
13530         err = i40e_enter_busy_conf(vsi);
13531         if (err)
13532                 return err;
13533
13534         i40e_queue_pair_disable_irq(vsi, queue_pair);
13535         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13536         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13537         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13538         i40e_queue_pair_clean_rings(vsi, queue_pair);
13539         i40e_queue_pair_reset_stats(vsi, queue_pair);
13540
13541         return err;
13542 }
13543
13544 /**
13545  * i40e_queue_pair_enable - Enables a queue pair
13546  * @vsi: vsi
13547  * @queue_pair: queue pair
13548  *
13549  * Returns 0 on success, <0 on failure.
13550  **/
13551 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13552 {
13553         int err;
13554
13555         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13556         if (err)
13557                 return err;
13558
13559         if (i40e_enabled_xdp_vsi(vsi)) {
13560                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13561                 if (err)
13562                         return err;
13563         }
13564
13565         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13566         if (err)
13567                 return err;
13568
13569         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13570         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13571         i40e_queue_pair_enable_irq(vsi, queue_pair);
13572
13573         i40e_exit_busy_conf(vsi);
13574
13575         return err;
13576 }
13577
13578 /**
13579  * i40e_xdp - implements ndo_bpf for i40e
13580  * @dev: netdevice
13581  * @xdp: XDP command
13582  **/
13583 static int i40e_xdp(struct net_device *dev,
13584                     struct netdev_bpf *xdp)
13585 {
13586         struct i40e_netdev_priv *np = netdev_priv(dev);
13587         struct i40e_vsi *vsi = np->vsi;
13588
13589         if (vsi->type != I40E_VSI_MAIN)
13590                 return -EINVAL;
13591
13592         switch (xdp->command) {
13593         case XDP_SETUP_PROG:
13594                 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13595         case XDP_SETUP_XSK_POOL:
13596                 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13597                                            xdp->xsk.queue_id);
13598         default:
13599                 return -EINVAL;
13600         }
13601 }
13602
13603 static const struct net_device_ops i40e_netdev_ops = {
13604         .ndo_open               = i40e_open,
13605         .ndo_stop               = i40e_close,
13606         .ndo_start_xmit         = i40e_lan_xmit_frame,
13607         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
13608         .ndo_set_rx_mode        = i40e_set_rx_mode,
13609         .ndo_validate_addr      = eth_validate_addr,
13610         .ndo_set_mac_address    = i40e_set_mac,
13611         .ndo_change_mtu         = i40e_change_mtu,
13612         .ndo_eth_ioctl          = i40e_ioctl,
13613         .ndo_tx_timeout         = i40e_tx_timeout,
13614         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
13615         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
13616 #ifdef CONFIG_NET_POLL_CONTROLLER
13617         .ndo_poll_controller    = i40e_netpoll,
13618 #endif
13619         .ndo_setup_tc           = __i40e_setup_tc,
13620         .ndo_select_queue       = i40e_lan_select_queue,
13621         .ndo_set_features       = i40e_set_features,
13622         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
13623         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
13624         .ndo_get_vf_stats       = i40e_get_vf_stats,
13625         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
13626         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
13627         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
13628         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
13629         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
13630         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
13631         .ndo_fdb_add            = i40e_ndo_fdb_add,
13632         .ndo_features_check     = i40e_features_check,
13633         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
13634         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
13635         .ndo_bpf                = i40e_xdp,
13636         .ndo_xdp_xmit           = i40e_xdp_xmit,
13637         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
13638         .ndo_dfwd_add_station   = i40e_fwd_add,
13639         .ndo_dfwd_del_station   = i40e_fwd_del,
13640 };
13641
13642 /**
13643  * i40e_config_netdev - Setup the netdev flags
13644  * @vsi: the VSI being configured
13645  *
13646  * Returns 0 on success, negative value on failure
13647  **/
13648 static int i40e_config_netdev(struct i40e_vsi *vsi)
13649 {
13650         struct i40e_pf *pf = vsi->back;
13651         struct i40e_hw *hw = &pf->hw;
13652         struct i40e_netdev_priv *np;
13653         struct net_device *netdev;
13654         u8 broadcast[ETH_ALEN];
13655         u8 mac_addr[ETH_ALEN];
13656         int etherdev_size;
13657         netdev_features_t hw_enc_features;
13658         netdev_features_t hw_features;
13659
13660         etherdev_size = sizeof(struct i40e_netdev_priv);
13661         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13662         if (!netdev)
13663                 return -ENOMEM;
13664
13665         vsi->netdev = netdev;
13666         np = netdev_priv(netdev);
13667         np->vsi = vsi;
13668
13669         hw_enc_features = NETIF_F_SG                    |
13670                           NETIF_F_HW_CSUM               |
13671                           NETIF_F_HIGHDMA               |
13672                           NETIF_F_SOFT_FEATURES         |
13673                           NETIF_F_TSO                   |
13674                           NETIF_F_TSO_ECN               |
13675                           NETIF_F_TSO6                  |
13676                           NETIF_F_GSO_GRE               |
13677                           NETIF_F_GSO_GRE_CSUM          |
13678                           NETIF_F_GSO_PARTIAL           |
13679                           NETIF_F_GSO_IPXIP4            |
13680                           NETIF_F_GSO_IPXIP6            |
13681                           NETIF_F_GSO_UDP_TUNNEL        |
13682                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
13683                           NETIF_F_GSO_UDP_L4            |
13684                           NETIF_F_SCTP_CRC              |
13685                           NETIF_F_RXHASH                |
13686                           NETIF_F_RXCSUM                |
13687                           0;
13688
13689         if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
13690                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13691
13692         netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13693
13694         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13695
13696         netdev->hw_enc_features |= hw_enc_features;
13697
13698         /* record features VLANs can make use of */
13699         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13700
13701 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE |            \
13702                                    NETIF_F_GSO_GRE_CSUM |       \
13703                                    NETIF_F_GSO_IPXIP4 |         \
13704                                    NETIF_F_GSO_IPXIP6 |         \
13705                                    NETIF_F_GSO_UDP_TUNNEL |     \
13706                                    NETIF_F_GSO_UDP_TUNNEL_CSUM)
13707
13708         netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13709         netdev->features |= NETIF_F_GSO_PARTIAL |
13710                             I40E_GSO_PARTIAL_FEATURES;
13711
13712         netdev->mpls_features |= NETIF_F_SG;
13713         netdev->mpls_features |= NETIF_F_HW_CSUM;
13714         netdev->mpls_features |= NETIF_F_TSO;
13715         netdev->mpls_features |= NETIF_F_TSO6;
13716         netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13717
13718         /* enable macvlan offloads */
13719         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13720
13721         hw_features = hw_enc_features           |
13722                       NETIF_F_HW_VLAN_CTAG_TX   |
13723                       NETIF_F_HW_VLAN_CTAG_RX;
13724
13725         if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
13726                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13727
13728         netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13729
13730         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13731         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13732
13733         netdev->features &= ~NETIF_F_HW_TC;
13734
13735         if (vsi->type == I40E_VSI_MAIN) {
13736                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13737                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13738                 /* The following steps are necessary for two reasons. First,
13739                  * some older NVM configurations load a default MAC-VLAN
13740                  * filter that will accept any tagged packet, and we want to
13741                  * replace this with a normal filter. Additionally, it is
13742                  * possible our MAC address was provided by the platform using
13743                  * Open Firmware or similar.
13744                  *
13745                  * Thus, we need to remove the default filter and install one
13746                  * specific to the MAC address.
13747                  */
13748                 i40e_rm_default_mac_filter(vsi, mac_addr);
13749                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13750                 i40e_add_mac_filter(vsi, mac_addr);
13751                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13752
13753                 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13754                                        NETDEV_XDP_ACT_REDIRECT |
13755                                        NETDEV_XDP_ACT_XSK_ZEROCOPY |
13756                                        NETDEV_XDP_ACT_RX_SG;
13757                 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13758         } else {
13759                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13760                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13761                  * the end, which is 4 bytes long, so force truncation of the
13762                  * original name by IFNAMSIZ - 4
13763                  */
13764                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13765                          IFNAMSIZ - 4,
13766                          pf->vsi[pf->lan_vsi]->netdev->name);
13767                 eth_random_addr(mac_addr);
13768
13769                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13770                 i40e_add_mac_filter(vsi, mac_addr);
13771                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13772         }
13773
13774         /* Add the broadcast filter so that we initially will receive
13775          * broadcast packets. Note that when a new VLAN is first added the
13776          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13777          * specific filters as part of transitioning into "vlan" operation.
13778          * When more VLANs are added, the driver will copy each existing MAC
13779          * filter and add it for the new VLAN.
13780          *
13781          * Broadcast filters are handled specially by
13782          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13783          * promiscuous bit instead of adding this directly as a MAC/VLAN
13784          * filter. The subtask will update the correct broadcast promiscuous
13785          * bits as VLANs become active or inactive.
13786          */
13787         eth_broadcast_addr(broadcast);
13788         spin_lock_bh(&vsi->mac_filter_hash_lock);
13789         i40e_add_mac_filter(vsi, broadcast);
13790         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13791
13792         eth_hw_addr_set(netdev, mac_addr);
13793         ether_addr_copy(netdev->perm_addr, mac_addr);
13794
13795         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13796         netdev->neigh_priv_len = sizeof(u32) * 4;
13797
13798         netdev->priv_flags |= IFF_UNICAST_FLT;
13799         netdev->priv_flags |= IFF_SUPP_NOFCS;
13800         /* Setup netdev TC information */
13801         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13802
13803         netdev->netdev_ops = &i40e_netdev_ops;
13804         netdev->watchdog_timeo = 5 * HZ;
13805         i40e_set_ethtool_ops(netdev);
13806
13807         /* MTU range: 68 - 9706 */
13808         netdev->min_mtu = ETH_MIN_MTU;
13809         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13810
13811         return 0;
13812 }
13813
13814 /**
13815  * i40e_vsi_delete - Delete a VSI from the switch
13816  * @vsi: the VSI being removed
13817  *
13818  * Returns 0 on success, negative value on failure
13819  **/
13820 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13821 {
13822         /* remove default VSI is not allowed */
13823         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13824                 return;
13825
13826         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13827 }
13828
13829 /**
13830  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13831  * @vsi: the VSI being queried
13832  *
13833  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13834  **/
13835 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13836 {
13837         struct i40e_veb *veb;
13838         struct i40e_pf *pf = vsi->back;
13839
13840         /* Uplink is not a bridge so default to VEB */
13841         if (vsi->veb_idx >= I40E_MAX_VEB)
13842                 return 1;
13843
13844         veb = pf->veb[vsi->veb_idx];
13845         if (!veb) {
13846                 dev_info(&pf->pdev->dev,
13847                          "There is no veb associated with the bridge\n");
13848                 return -ENOENT;
13849         }
13850
13851         /* Uplink is a bridge in VEPA mode */
13852         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13853                 return 0;
13854         } else {
13855                 /* Uplink is a bridge in VEB mode */
13856                 return 1;
13857         }
13858
13859         /* VEPA is now default bridge, so return 0 */
13860         return 0;
13861 }
13862
13863 /**
13864  * i40e_add_vsi - Add a VSI to the switch
13865  * @vsi: the VSI being configured
13866  *
13867  * This initializes a VSI context depending on the VSI type to be added and
13868  * passes it down to the add_vsi aq command.
13869  **/
13870 static int i40e_add_vsi(struct i40e_vsi *vsi)
13871 {
13872         int ret = -ENODEV;
13873         struct i40e_pf *pf = vsi->back;
13874         struct i40e_hw *hw = &pf->hw;
13875         struct i40e_vsi_context ctxt;
13876         struct i40e_mac_filter *f;
13877         struct hlist_node *h;
13878         int bkt;
13879
13880         u8 enabled_tc = 0x1; /* TC0 enabled */
13881         int f_count = 0;
13882
13883         memset(&ctxt, 0, sizeof(ctxt));
13884         switch (vsi->type) {
13885         case I40E_VSI_MAIN:
13886                 /* The PF's main VSI is already setup as part of the
13887                  * device initialization, so we'll not bother with
13888                  * the add_vsi call, but we will retrieve the current
13889                  * VSI context.
13890                  */
13891                 ctxt.seid = pf->main_vsi_seid;
13892                 ctxt.pf_num = pf->hw.pf_id;
13893                 ctxt.vf_num = 0;
13894                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13895                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13896                 if (ret) {
13897                         dev_info(&pf->pdev->dev,
13898                                  "couldn't get PF vsi config, err %pe aq_err %s\n",
13899                                  ERR_PTR(ret),
13900                                  i40e_aq_str(&pf->hw,
13901                                              pf->hw.aq.asq_last_status));
13902                         return -ENOENT;
13903                 }
13904                 vsi->info = ctxt.info;
13905                 vsi->info.valid_sections = 0;
13906
13907                 vsi->seid = ctxt.seid;
13908                 vsi->id = ctxt.vsi_number;
13909
13910                 enabled_tc = i40e_pf_get_tc_map(pf);
13911
13912                 /* Source pruning is enabled by default, so the flag is
13913                  * negative logic - if it's set, we need to fiddle with
13914                  * the VSI to disable source pruning.
13915                  */
13916                 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
13917                         memset(&ctxt, 0, sizeof(ctxt));
13918                         ctxt.seid = pf->main_vsi_seid;
13919                         ctxt.pf_num = pf->hw.pf_id;
13920                         ctxt.vf_num = 0;
13921                         ctxt.info.valid_sections |=
13922                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13923                         ctxt.info.switch_id =
13924                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13925                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13926                         if (ret) {
13927                                 dev_info(&pf->pdev->dev,
13928                                          "update vsi failed, err %d aq_err %s\n",
13929                                          ret,
13930                                          i40e_aq_str(&pf->hw,
13931                                                      pf->hw.aq.asq_last_status));
13932                                 ret = -ENOENT;
13933                                 goto err;
13934                         }
13935                 }
13936
13937                 /* MFP mode setup queue map and update VSI */
13938                 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
13939                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13940                         memset(&ctxt, 0, sizeof(ctxt));
13941                         ctxt.seid = pf->main_vsi_seid;
13942                         ctxt.pf_num = pf->hw.pf_id;
13943                         ctxt.vf_num = 0;
13944                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13945                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13946                         if (ret) {
13947                                 dev_info(&pf->pdev->dev,
13948                                          "update vsi failed, err %pe aq_err %s\n",
13949                                          ERR_PTR(ret),
13950                                          i40e_aq_str(&pf->hw,
13951                                                     pf->hw.aq.asq_last_status));
13952                                 ret = -ENOENT;
13953                                 goto err;
13954                         }
13955                         /* update the local VSI info queue map */
13956                         i40e_vsi_update_queue_map(vsi, &ctxt);
13957                         vsi->info.valid_sections = 0;
13958                 } else {
13959                         /* Default/Main VSI is only enabled for TC0
13960                          * reconfigure it to enable all TCs that are
13961                          * available on the port in SFP mode.
13962                          * For MFP case the iSCSI PF would use this
13963                          * flow to enable LAN+iSCSI TC.
13964                          */
13965                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13966                         if (ret) {
13967                                 /* Single TC condition is not fatal,
13968                                  * message and continue
13969                                  */
13970                                 dev_info(&pf->pdev->dev,
13971                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
13972                                          enabled_tc,
13973                                          ERR_PTR(ret),
13974                                          i40e_aq_str(&pf->hw,
13975                                                     pf->hw.aq.asq_last_status));
13976                         }
13977                 }
13978                 break;
13979
13980         case I40E_VSI_FDIR:
13981                 ctxt.pf_num = hw->pf_id;
13982                 ctxt.vf_num = 0;
13983                 ctxt.uplink_seid = vsi->uplink_seid;
13984                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13985                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13986                 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
13987                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13988                         ctxt.info.valid_sections |=
13989                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13990                         ctxt.info.switch_id =
13991                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13992                 }
13993                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13994                 break;
13995
13996         case I40E_VSI_VMDQ2:
13997                 ctxt.pf_num = hw->pf_id;
13998                 ctxt.vf_num = 0;
13999                 ctxt.uplink_seid = vsi->uplink_seid;
14000                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14001                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14002
14003                 /* This VSI is connected to VEB so the switch_id
14004                  * should be set to zero by default.
14005                  */
14006                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14007                         ctxt.info.valid_sections |=
14008                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14009                         ctxt.info.switch_id =
14010                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14011                 }
14012
14013                 /* Setup the VSI tx/rx queue map for TC0 only for now */
14014                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14015                 break;
14016
14017         case I40E_VSI_SRIOV:
14018                 ctxt.pf_num = hw->pf_id;
14019                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14020                 ctxt.uplink_seid = vsi->uplink_seid;
14021                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14022                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14023
14024                 /* This VSI is connected to VEB so the switch_id
14025                  * should be set to zero by default.
14026                  */
14027                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14028                         ctxt.info.valid_sections |=
14029                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14030                         ctxt.info.switch_id =
14031                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14032                 }
14033
14034                 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
14035                         ctxt.info.valid_sections |=
14036                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14037                         ctxt.info.queueing_opt_flags |=
14038                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14039                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14040                 }
14041
14042                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14043                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14044                 if (pf->vf[vsi->vf_id].spoofchk) {
14045                         ctxt.info.valid_sections |=
14046                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14047                         ctxt.info.sec_flags |=
14048                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14049                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14050                 }
14051                 /* Setup the VSI tx/rx queue map for TC0 only for now */
14052                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14053                 break;
14054
14055         case I40E_VSI_IWARP:
14056                 /* send down message to iWARP */
14057                 break;
14058
14059         default:
14060                 return -ENODEV;
14061         }
14062
14063         if (vsi->type != I40E_VSI_MAIN) {
14064                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14065                 if (ret) {
14066                         dev_info(&vsi->back->pdev->dev,
14067                                  "add vsi failed, err %pe aq_err %s\n",
14068                                  ERR_PTR(ret),
14069                                  i40e_aq_str(&pf->hw,
14070                                              pf->hw.aq.asq_last_status));
14071                         ret = -ENOENT;
14072                         goto err;
14073                 }
14074                 vsi->info = ctxt.info;
14075                 vsi->info.valid_sections = 0;
14076                 vsi->seid = ctxt.seid;
14077                 vsi->id = ctxt.vsi_number;
14078         }
14079
14080         spin_lock_bh(&vsi->mac_filter_hash_lock);
14081         vsi->active_filters = 0;
14082         /* If macvlan filters already exist, force them to get loaded */
14083         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14084                 f->state = I40E_FILTER_NEW;
14085                 f_count++;
14086         }
14087         spin_unlock_bh(&vsi->mac_filter_hash_lock);
14088         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14089
14090         if (f_count) {
14091                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14092                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14093         }
14094
14095         /* Update VSI BW information */
14096         ret = i40e_vsi_get_bw_info(vsi);
14097         if (ret) {
14098                 dev_info(&pf->pdev->dev,
14099                          "couldn't get vsi bw info, err %pe aq_err %s\n",
14100                          ERR_PTR(ret),
14101                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14102                 /* VSI is already added so not tearing that up */
14103                 ret = 0;
14104         }
14105
14106 err:
14107         return ret;
14108 }
14109
14110 /**
14111  * i40e_vsi_release - Delete a VSI and free its resources
14112  * @vsi: the VSI being removed
14113  *
14114  * Returns 0 on success or < 0 on error
14115  **/
14116 int i40e_vsi_release(struct i40e_vsi *vsi)
14117 {
14118         struct i40e_mac_filter *f;
14119         struct hlist_node *h;
14120         struct i40e_veb *veb;
14121         struct i40e_pf *pf;
14122         u16 uplink_seid;
14123         int i, n, bkt;
14124
14125         pf = vsi->back;
14126
14127         /* release of a VEB-owner or last VSI is not allowed */
14128         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14129                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14130                          vsi->seid, vsi->uplink_seid);
14131                 return -ENODEV;
14132         }
14133         if (vsi == pf->vsi[pf->lan_vsi] &&
14134             !test_bit(__I40E_DOWN, pf->state)) {
14135                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14136                 return -ENODEV;
14137         }
14138         set_bit(__I40E_VSI_RELEASING, vsi->state);
14139         uplink_seid = vsi->uplink_seid;
14140
14141         if (vsi->type != I40E_VSI_SRIOV) {
14142                 if (vsi->netdev_registered) {
14143                         vsi->netdev_registered = false;
14144                         if (vsi->netdev) {
14145                                 /* results in a call to i40e_close() */
14146                                 unregister_netdev(vsi->netdev);
14147                         }
14148                 } else {
14149                         i40e_vsi_close(vsi);
14150                 }
14151                 i40e_vsi_disable_irq(vsi);
14152         }
14153
14154         if (vsi->type == I40E_VSI_MAIN)
14155                 i40e_devlink_destroy_port(pf);
14156
14157         spin_lock_bh(&vsi->mac_filter_hash_lock);
14158
14159         /* clear the sync flag on all filters */
14160         if (vsi->netdev) {
14161                 __dev_uc_unsync(vsi->netdev, NULL);
14162                 __dev_mc_unsync(vsi->netdev, NULL);
14163         }
14164
14165         /* make sure any remaining filters are marked for deletion */
14166         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14167                 __i40e_del_filter(vsi, f);
14168
14169         spin_unlock_bh(&vsi->mac_filter_hash_lock);
14170
14171         i40e_sync_vsi_filters(vsi);
14172
14173         i40e_vsi_delete(vsi);
14174         i40e_vsi_free_q_vectors(vsi);
14175         if (vsi->netdev) {
14176                 free_netdev(vsi->netdev);
14177                 vsi->netdev = NULL;
14178         }
14179         i40e_vsi_clear_rings(vsi);
14180         i40e_vsi_clear(vsi);
14181
14182         /* If this was the last thing on the VEB, except for the
14183          * controlling VSI, remove the VEB, which puts the controlling
14184          * VSI onto the uplink port.
14185          *
14186          * Well, okay, there's one more exception here: don't remove
14187          * the floating VEBs yet.  We'll wait for an explicit remove request
14188          * from up the network stack.
14189          */
14190         veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14191         if (veb && veb->uplink_seid) {
14192                 n = 0;
14193
14194                 /* Count non-controlling VSIs present on  the VEB */
14195                 i40e_pf_for_each_vsi(pf, i, vsi)
14196                         if (vsi->uplink_seid == uplink_seid &&
14197                             (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14198                                 n++;
14199
14200                 /* If there is no VSI except the control one then release
14201                  * the VEB and put the control VSI onto VEB uplink.
14202                  */
14203                 if (!n)
14204                         i40e_veb_release(veb);
14205         }
14206
14207         return 0;
14208 }
14209
14210 /**
14211  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14212  * @vsi: ptr to the VSI
14213  *
14214  * This should only be called after i40e_vsi_mem_alloc() which allocates the
14215  * corresponding SW VSI structure and initializes num_queue_pairs for the
14216  * newly allocated VSI.
14217  *
14218  * Returns 0 on success or negative on failure
14219  **/
14220 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14221 {
14222         int ret = -ENOENT;
14223         struct i40e_pf *pf = vsi->back;
14224
14225         if (vsi->q_vectors[0]) {
14226                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14227                          vsi->seid);
14228                 return -EEXIST;
14229         }
14230
14231         if (vsi->base_vector) {
14232                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14233                          vsi->seid, vsi->base_vector);
14234                 return -EEXIST;
14235         }
14236
14237         ret = i40e_vsi_alloc_q_vectors(vsi);
14238         if (ret) {
14239                 dev_info(&pf->pdev->dev,
14240                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14241                          vsi->num_q_vectors, vsi->seid, ret);
14242                 vsi->num_q_vectors = 0;
14243                 goto vector_setup_out;
14244         }
14245
14246         /* In Legacy mode, we do not have to get any other vector since we
14247          * piggyback on the misc/ICR0 for queue interrupts.
14248         */
14249         if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
14250                 return ret;
14251         if (vsi->num_q_vectors)
14252                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14253                                                  vsi->num_q_vectors, vsi->idx);
14254         if (vsi->base_vector < 0) {
14255                 dev_info(&pf->pdev->dev,
14256                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14257                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14258                 i40e_vsi_free_q_vectors(vsi);
14259                 ret = -ENOENT;
14260                 goto vector_setup_out;
14261         }
14262
14263 vector_setup_out:
14264         return ret;
14265 }
14266
14267 /**
14268  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14269  * @vsi: pointer to the vsi.
14270  *
14271  * This re-allocates a vsi's queue resources.
14272  *
14273  * Returns pointer to the successfully allocated and configured VSI sw struct
14274  * on success, otherwise returns NULL on failure.
14275  **/
14276 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14277 {
14278         u16 alloc_queue_pairs;
14279         struct i40e_pf *pf;
14280         u8 enabled_tc;
14281         int ret;
14282
14283         if (!vsi)
14284                 return NULL;
14285
14286         pf = vsi->back;
14287
14288         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14289         i40e_vsi_clear_rings(vsi);
14290
14291         i40e_vsi_free_arrays(vsi, false);
14292         i40e_set_num_rings_in_vsi(vsi);
14293         ret = i40e_vsi_alloc_arrays(vsi, false);
14294         if (ret)
14295                 goto err_vsi;
14296
14297         alloc_queue_pairs = vsi->alloc_queue_pairs *
14298                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14299
14300         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14301         if (ret < 0) {
14302                 dev_info(&pf->pdev->dev,
14303                          "failed to get tracking for %d queues for VSI %d err %d\n",
14304                          alloc_queue_pairs, vsi->seid, ret);
14305                 goto err_vsi;
14306         }
14307         vsi->base_queue = ret;
14308
14309         /* Update the FW view of the VSI. Force a reset of TC and queue
14310          * layout configurations.
14311          */
14312         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14313         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14314         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14315         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14316         if (vsi->type == I40E_VSI_MAIN)
14317                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14318
14319         /* assign it some queues */
14320         ret = i40e_alloc_rings(vsi);
14321         if (ret)
14322                 goto err_rings;
14323
14324         /* map all of the rings to the q_vectors */
14325         i40e_vsi_map_rings_to_vectors(vsi);
14326         return vsi;
14327
14328 err_rings:
14329         i40e_vsi_free_q_vectors(vsi);
14330         if (vsi->netdev_registered) {
14331                 vsi->netdev_registered = false;
14332                 unregister_netdev(vsi->netdev);
14333                 free_netdev(vsi->netdev);
14334                 vsi->netdev = NULL;
14335         }
14336         if (vsi->type == I40E_VSI_MAIN)
14337                 i40e_devlink_destroy_port(pf);
14338         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14339 err_vsi:
14340         i40e_vsi_clear(vsi);
14341         return NULL;
14342 }
14343
14344 /**
14345  * i40e_vsi_setup - Set up a VSI by a given type
14346  * @pf: board private structure
14347  * @type: VSI type
14348  * @uplink_seid: the switch element to link to
14349  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14350  *
14351  * This allocates the sw VSI structure and its queue resources, then add a VSI
14352  * to the identified VEB.
14353  *
14354  * Returns pointer to the successfully allocated and configure VSI sw struct on
14355  * success, otherwise returns NULL on failure.
14356  **/
14357 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14358                                 u16 uplink_seid, u32 param1)
14359 {
14360         struct i40e_vsi *vsi = NULL;
14361         struct i40e_veb *veb = NULL;
14362         u16 alloc_queue_pairs;
14363         int v_idx;
14364         int ret;
14365
14366         /* The requested uplink_seid must be either
14367          *     - the PF's port seid
14368          *              no VEB is needed because this is the PF
14369          *              or this is a Flow Director special case VSI
14370          *     - seid of an existing VEB
14371          *     - seid of a VSI that owns an existing VEB
14372          *     - seid of a VSI that doesn't own a VEB
14373          *              a new VEB is created and the VSI becomes the owner
14374          *     - seid of the PF VSI, which is what creates the first VEB
14375          *              this is a special case of the previous
14376          *
14377          * Find which uplink_seid we were given and create a new VEB if needed
14378          */
14379         veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14380         if (!veb && uplink_seid != pf->mac_seid) {
14381                 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid);
14382                 if (!vsi) {
14383                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14384                                  uplink_seid);
14385                         return NULL;
14386                 }
14387
14388                 if (vsi->uplink_seid == pf->mac_seid)
14389                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14390                                              vsi->tc_config.enabled_tc);
14391                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14392                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14393                                              vsi->tc_config.enabled_tc);
14394                 if (veb) {
14395                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14396                                 dev_info(&vsi->back->pdev->dev,
14397                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
14398                                 return NULL;
14399                         }
14400                         /* We come up by default in VEPA mode if SRIOV is not
14401                          * already enabled, in which case we can't force VEPA
14402                          * mode.
14403                          */
14404                         if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
14405                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
14406                                 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
14407                         }
14408                         i40e_config_bridge_mode(veb);
14409                 }
14410                 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
14411                 if (!veb) {
14412                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14413                         return NULL;
14414                 }
14415
14416                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14417                 uplink_seid = veb->seid;
14418         }
14419
14420         /* get vsi sw struct */
14421         v_idx = i40e_vsi_mem_alloc(pf, type);
14422         if (v_idx < 0)
14423                 goto err_alloc;
14424         vsi = pf->vsi[v_idx];
14425         if (!vsi)
14426                 goto err_alloc;
14427         vsi->type = type;
14428         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14429
14430         if (type == I40E_VSI_MAIN)
14431                 pf->lan_vsi = v_idx;
14432         else if (type == I40E_VSI_SRIOV)
14433                 vsi->vf_id = param1;
14434         /* assign it some queues */
14435         alloc_queue_pairs = vsi->alloc_queue_pairs *
14436                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14437
14438         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14439         if (ret < 0) {
14440                 dev_info(&pf->pdev->dev,
14441                          "failed to get tracking for %d queues for VSI %d err=%d\n",
14442                          alloc_queue_pairs, vsi->seid, ret);
14443                 goto err_vsi;
14444         }
14445         vsi->base_queue = ret;
14446
14447         /* get a VSI from the hardware */
14448         vsi->uplink_seid = uplink_seid;
14449         ret = i40e_add_vsi(vsi);
14450         if (ret)
14451                 goto err_vsi;
14452
14453         switch (vsi->type) {
14454         /* setup the netdev if needed */
14455         case I40E_VSI_MAIN:
14456         case I40E_VSI_VMDQ2:
14457                 ret = i40e_config_netdev(vsi);
14458                 if (ret)
14459                         goto err_netdev;
14460                 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14461                 if (ret)
14462                         goto err_netdev;
14463                 if (vsi->type == I40E_VSI_MAIN) {
14464                         ret = i40e_devlink_create_port(pf);
14465                         if (ret)
14466                                 goto err_netdev;
14467                         SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
14468                 }
14469                 ret = register_netdev(vsi->netdev);
14470                 if (ret)
14471                         goto err_dl_port;
14472                 vsi->netdev_registered = true;
14473                 netif_carrier_off(vsi->netdev);
14474 #ifdef CONFIG_I40E_DCB
14475                 /* Setup DCB netlink interface */
14476                 i40e_dcbnl_setup(vsi);
14477 #endif /* CONFIG_I40E_DCB */
14478                 fallthrough;
14479         case I40E_VSI_FDIR:
14480                 /* set up vectors and rings if needed */
14481                 ret = i40e_vsi_setup_vectors(vsi);
14482                 if (ret)
14483                         goto err_msix;
14484
14485                 ret = i40e_alloc_rings(vsi);
14486                 if (ret)
14487                         goto err_rings;
14488
14489                 /* map all of the rings to the q_vectors */
14490                 i40e_vsi_map_rings_to_vectors(vsi);
14491
14492                 i40e_vsi_reset_stats(vsi);
14493                 break;
14494         default:
14495                 /* no netdev or rings for the other VSI types */
14496                 break;
14497         }
14498
14499         if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
14500             vsi->type == I40E_VSI_VMDQ2) {
14501                 ret = i40e_vsi_config_rss(vsi);
14502                 if (ret)
14503                         goto err_config;
14504         }
14505         return vsi;
14506
14507 err_config:
14508         i40e_vsi_clear_rings(vsi);
14509 err_rings:
14510         i40e_vsi_free_q_vectors(vsi);
14511 err_msix:
14512         if (vsi->netdev_registered) {
14513                 vsi->netdev_registered = false;
14514                 unregister_netdev(vsi->netdev);
14515                 free_netdev(vsi->netdev);
14516                 vsi->netdev = NULL;
14517         }
14518 err_dl_port:
14519         if (vsi->type == I40E_VSI_MAIN)
14520                 i40e_devlink_destroy_port(pf);
14521 err_netdev:
14522         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14523 err_vsi:
14524         i40e_vsi_clear(vsi);
14525 err_alloc:
14526         return NULL;
14527 }
14528
14529 /**
14530  * i40e_veb_get_bw_info - Query VEB BW information
14531  * @veb: the veb to query
14532  *
14533  * Query the Tx scheduler BW configuration data for given VEB
14534  **/
14535 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14536 {
14537         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14538         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14539         struct i40e_pf *pf = veb->pf;
14540         struct i40e_hw *hw = &pf->hw;
14541         u32 tc_bw_max;
14542         int ret = 0;
14543         int i;
14544
14545         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14546                                                   &bw_data, NULL);
14547         if (ret) {
14548                 dev_info(&pf->pdev->dev,
14549                          "query veb bw config failed, err %pe aq_err %s\n",
14550                          ERR_PTR(ret),
14551                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14552                 goto out;
14553         }
14554
14555         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14556                                                    &ets_data, NULL);
14557         if (ret) {
14558                 dev_info(&pf->pdev->dev,
14559                          "query veb bw ets config failed, err %pe aq_err %s\n",
14560                          ERR_PTR(ret),
14561                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14562                 goto out;
14563         }
14564
14565         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14566         veb->bw_max_quanta = ets_data.tc_bw_max;
14567         veb->is_abs_credits = bw_data.absolute_credits_enable;
14568         veb->enabled_tc = ets_data.tc_valid_bits;
14569         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14570                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14571         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14572                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14573                 veb->bw_tc_limit_credits[i] =
14574                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
14575                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14576         }
14577
14578 out:
14579         return ret;
14580 }
14581
14582 /**
14583  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14584  * @pf: board private structure
14585  *
14586  * On error: returns error code (negative)
14587  * On success: returns vsi index in PF (positive)
14588  **/
14589 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14590 {
14591         int ret = -ENOENT;
14592         struct i40e_veb *veb;
14593         int i;
14594
14595         /* Need to protect the allocation of switch elements at the PF level */
14596         mutex_lock(&pf->switch_mutex);
14597
14598         /* VEB list may be fragmented if VEB creation/destruction has
14599          * been happening.  We can afford to do a quick scan to look
14600          * for any free slots in the list.
14601          *
14602          * find next empty veb slot, looping back around if necessary
14603          */
14604         i = 0;
14605         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14606                 i++;
14607         if (i >= I40E_MAX_VEB) {
14608                 ret = -ENOMEM;
14609                 goto err_alloc_veb;  /* out of VEB slots! */
14610         }
14611
14612         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14613         if (!veb) {
14614                 ret = -ENOMEM;
14615                 goto err_alloc_veb;
14616         }
14617         veb->pf = pf;
14618         veb->idx = i;
14619         veb->enabled_tc = 1;
14620
14621         pf->veb[i] = veb;
14622         ret = i;
14623 err_alloc_veb:
14624         mutex_unlock(&pf->switch_mutex);
14625         return ret;
14626 }
14627
14628 /**
14629  * i40e_switch_branch_release - Delete a branch of the switch tree
14630  * @branch: where to start deleting
14631  *
14632  * This uses recursion to find the tips of the branch to be
14633  * removed, deleting until we get back to and can delete this VEB.
14634  **/
14635 static void i40e_switch_branch_release(struct i40e_veb *branch)
14636 {
14637         struct i40e_pf *pf = branch->pf;
14638         u16 branch_seid = branch->seid;
14639         u16 veb_idx = branch->idx;
14640         struct i40e_vsi *vsi;
14641         struct i40e_veb *veb;
14642         int i;
14643
14644         /* release any VEBs on this VEB - RECURSION */
14645         i40e_pf_for_each_veb(pf, i, veb)
14646                 if (veb->uplink_seid == branch->seid)
14647                         i40e_switch_branch_release(veb);
14648
14649         /* Release the VSIs on this VEB, but not the owner VSI.
14650          *
14651          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14652          *       the VEB itself, so don't use (*branch) after this loop.
14653          */
14654         i40e_pf_for_each_vsi(pf, i, vsi)
14655                 if (vsi->uplink_seid == branch_seid &&
14656                     (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14657                         i40e_vsi_release(vsi);
14658
14659         /* There's one corner case where the VEB might not have been
14660          * removed, so double check it here and remove it if needed.
14661          * This case happens if the veb was created from the debugfs
14662          * commands and no VSIs were added to it.
14663          */
14664         if (pf->veb[veb_idx])
14665                 i40e_veb_release(pf->veb[veb_idx]);
14666 }
14667
14668 /**
14669  * i40e_veb_clear - remove veb struct
14670  * @veb: the veb to remove
14671  **/
14672 static void i40e_veb_clear(struct i40e_veb *veb)
14673 {
14674         if (!veb)
14675                 return;
14676
14677         if (veb->pf) {
14678                 struct i40e_pf *pf = veb->pf;
14679
14680                 mutex_lock(&pf->switch_mutex);
14681                 if (pf->veb[veb->idx] == veb)
14682                         pf->veb[veb->idx] = NULL;
14683                 mutex_unlock(&pf->switch_mutex);
14684         }
14685
14686         kfree(veb);
14687 }
14688
14689 /**
14690  * i40e_veb_release - Delete a VEB and free its resources
14691  * @veb: the VEB being removed
14692  **/
14693 void i40e_veb_release(struct i40e_veb *veb)
14694 {
14695         struct i40e_vsi *vsi, *vsi_it;
14696         struct i40e_pf *pf;
14697         int i, n = 0;
14698
14699         pf = veb->pf;
14700
14701         /* find the remaining VSI and check for extras */
14702         i40e_pf_for_each_vsi(pf, i, vsi_it)
14703                 if (vsi_it->uplink_seid == veb->seid) {
14704                         if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)
14705                                 vsi = vsi_it;
14706                         n++;
14707                 }
14708
14709         /* Floating VEB has to be empty and regular one must have
14710          * single owner VSI.
14711          */
14712         if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {
14713                 dev_info(&pf->pdev->dev,
14714                          "can't remove VEB %d with %d VSIs left\n",
14715                          veb->seid, n);
14716                 return;
14717         }
14718
14719         /* For regular VEB move the owner VSI to uplink port */
14720         if (veb->uplink_seid) {
14721                 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14722                 vsi->uplink_seid = veb->uplink_seid;
14723                 vsi->veb_idx = I40E_NO_VEB;
14724         }
14725
14726         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14727         i40e_veb_clear(veb);
14728 }
14729
14730 /**
14731  * i40e_add_veb - create the VEB in the switch
14732  * @veb: the VEB to be instantiated
14733  * @vsi: the controlling VSI
14734  **/
14735 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14736 {
14737         struct i40e_pf *pf = veb->pf;
14738         bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
14739         int ret;
14740
14741         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,
14742                               veb->enabled_tc, vsi ? false : true,
14743                               &veb->seid, enable_stats, NULL);
14744
14745         /* get a VEB from the hardware */
14746         if (ret) {
14747                 dev_info(&pf->pdev->dev,
14748                          "couldn't add VEB, err %pe aq_err %s\n",
14749                          ERR_PTR(ret),
14750                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14751                 return -EPERM;
14752         }
14753
14754         /* get statistics counter */
14755         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14756                                          &veb->stats_idx, NULL, NULL, NULL);
14757         if (ret) {
14758                 dev_info(&pf->pdev->dev,
14759                          "couldn't get VEB statistics idx, err %pe aq_err %s\n",
14760                          ERR_PTR(ret),
14761                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14762                 return -EPERM;
14763         }
14764         ret = i40e_veb_get_bw_info(veb);
14765         if (ret) {
14766                 dev_info(&pf->pdev->dev,
14767                          "couldn't get VEB bw info, err %pe aq_err %s\n",
14768                          ERR_PTR(ret),
14769                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14770                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14771                 return -ENOENT;
14772         }
14773
14774         if (vsi) {
14775                 vsi->uplink_seid = veb->seid;
14776                 vsi->veb_idx = veb->idx;
14777                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14778         }
14779
14780         return 0;
14781 }
14782
14783 /**
14784  * i40e_veb_setup - Set up a VEB
14785  * @pf: board private structure
14786  * @flags: VEB setup flags
14787  * @uplink_seid: the switch element to link to
14788  * @vsi_seid: the initial VSI seid
14789  * @enabled_tc: Enabled TC bit-map
14790  *
14791  * This allocates the sw VEB structure and links it into the switch
14792  * It is possible and legal for this to be a duplicate of an already
14793  * existing VEB.  It is also possible for both uplink and vsi seids
14794  * to be zero, in order to create a floating VEB.
14795  *
14796  * Returns pointer to the successfully allocated VEB sw struct on
14797  * success, otherwise returns NULL on failure.
14798  **/
14799 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14800                                 u16 uplink_seid, u16 vsi_seid,
14801                                 u8 enabled_tc)
14802 {
14803         struct i40e_vsi *vsi = NULL;
14804         struct i40e_veb *veb;
14805         int veb_idx;
14806         int ret;
14807
14808         /* if one seid is 0, the other must be 0 to create a floating relay */
14809         if ((uplink_seid == 0 || vsi_seid == 0) &&
14810             (uplink_seid + vsi_seid != 0)) {
14811                 dev_info(&pf->pdev->dev,
14812                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14813                          uplink_seid, vsi_seid);
14814                 return NULL;
14815         }
14816
14817         /* make sure there is such a vsi and uplink */
14818         if (vsi_seid) {
14819                 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid);
14820                 if (!vsi) {
14821                         dev_err(&pf->pdev->dev, "vsi seid %d not found\n",
14822                                 vsi_seid);
14823                         return NULL;
14824                 }
14825         }
14826
14827         /* get veb sw struct */
14828         veb_idx = i40e_veb_mem_alloc(pf);
14829         if (veb_idx < 0)
14830                 goto err_alloc;
14831         veb = pf->veb[veb_idx];
14832         veb->flags = flags;
14833         veb->uplink_seid = uplink_seid;
14834         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14835
14836         /* create the VEB in the switch */
14837         ret = i40e_add_veb(veb, vsi);
14838         if (ret)
14839                 goto err_veb;
14840
14841         if (vsi && vsi->idx == pf->lan_vsi)
14842                 pf->lan_veb = veb->idx;
14843
14844         return veb;
14845
14846 err_veb:
14847         i40e_veb_clear(veb);
14848 err_alloc:
14849         return NULL;
14850 }
14851
14852 /**
14853  * i40e_setup_pf_switch_element - set PF vars based on switch type
14854  * @pf: board private structure
14855  * @ele: element we are building info from
14856  * @num_reported: total number of elements
14857  * @printconfig: should we print the contents
14858  *
14859  * helper function to assist in extracting a few useful SEID values.
14860  **/
14861 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14862                                 struct i40e_aqc_switch_config_element_resp *ele,
14863                                 u16 num_reported, bool printconfig)
14864 {
14865         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14866         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14867         u8 element_type = ele->element_type;
14868         u16 seid = le16_to_cpu(ele->seid);
14869         struct i40e_veb *veb;
14870
14871         if (printconfig)
14872                 dev_info(&pf->pdev->dev,
14873                          "type=%d seid=%d uplink=%d downlink=%d\n",
14874                          element_type, seid, uplink_seid, downlink_seid);
14875
14876         switch (element_type) {
14877         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14878                 pf->mac_seid = seid;
14879                 break;
14880         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14881                 /* Main VEB? */
14882                 if (uplink_seid != pf->mac_seid)
14883                         break;
14884                 if (pf->lan_veb >= I40E_MAX_VEB) {
14885                         int v;
14886
14887                         /* find existing or else empty VEB */
14888                         veb = i40e_pf_get_veb_by_seid(pf, seid);
14889                         if (veb) {
14890                                 pf->lan_veb = veb->idx;
14891                         } else {
14892                                 v = i40e_veb_mem_alloc(pf);
14893                                 if (v < 0)
14894                                         break;
14895                                 pf->lan_veb = v;
14896                         }
14897                 }
14898                 if (pf->lan_veb >= I40E_MAX_VEB)
14899                         break;
14900
14901                 pf->veb[pf->lan_veb]->seid = seid;
14902                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14903                 pf->veb[pf->lan_veb]->pf = pf;
14904                 break;
14905         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14906                 if (num_reported != 1)
14907                         break;
14908                 /* This is immediately after a reset so we can assume this is
14909                  * the PF's VSI
14910                  */
14911                 pf->mac_seid = uplink_seid;
14912                 pf->main_vsi_seid = seid;
14913                 if (printconfig)
14914                         dev_info(&pf->pdev->dev,
14915                                  "pf_seid=%d main_vsi_seid=%d\n",
14916                                  downlink_seid, pf->main_vsi_seid);
14917                 break;
14918         case I40E_SWITCH_ELEMENT_TYPE_PF:
14919         case I40E_SWITCH_ELEMENT_TYPE_VF:
14920         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14921         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14922         case I40E_SWITCH_ELEMENT_TYPE_PE:
14923         case I40E_SWITCH_ELEMENT_TYPE_PA:
14924                 /* ignore these for now */
14925                 break;
14926         default:
14927                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14928                          element_type, seid);
14929                 break;
14930         }
14931 }
14932
14933 /**
14934  * i40e_fetch_switch_configuration - Get switch config from firmware
14935  * @pf: board private structure
14936  * @printconfig: should we print the contents
14937  *
14938  * Get the current switch configuration from the device and
14939  * extract a few useful SEID values.
14940  **/
14941 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14942 {
14943         struct i40e_aqc_get_switch_config_resp *sw_config;
14944         u16 next_seid = 0;
14945         int ret = 0;
14946         u8 *aq_buf;
14947         int i;
14948
14949         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14950         if (!aq_buf)
14951                 return -ENOMEM;
14952
14953         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14954         do {
14955                 u16 num_reported, num_total;
14956
14957                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14958                                                 I40E_AQ_LARGE_BUF,
14959                                                 &next_seid, NULL);
14960                 if (ret) {
14961                         dev_info(&pf->pdev->dev,
14962                                  "get switch config failed err %d aq_err %s\n",
14963                                  ret,
14964                                  i40e_aq_str(&pf->hw,
14965                                              pf->hw.aq.asq_last_status));
14966                         kfree(aq_buf);
14967                         return -ENOENT;
14968                 }
14969
14970                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14971                 num_total = le16_to_cpu(sw_config->header.num_total);
14972
14973                 if (printconfig)
14974                         dev_info(&pf->pdev->dev,
14975                                  "header: %d reported %d total\n",
14976                                  num_reported, num_total);
14977
14978                 for (i = 0; i < num_reported; i++) {
14979                         struct i40e_aqc_switch_config_element_resp *ele =
14980                                 &sw_config->element[i];
14981
14982                         i40e_setup_pf_switch_element(pf, ele, num_reported,
14983                                                      printconfig);
14984                 }
14985         } while (next_seid != 0);
14986
14987         kfree(aq_buf);
14988         return ret;
14989 }
14990
14991 /**
14992  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14993  * @pf: board private structure
14994  * @reinit: if the Main VSI needs to re-initialized.
14995  * @lock_acquired: indicates whether or not the lock has been acquired
14996  *
14997  * Returns 0 on success, negative value on failure
14998  **/
14999 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15000 {
15001         u16 flags = 0;
15002         int ret;
15003
15004         /* find out what's out there already */
15005         ret = i40e_fetch_switch_configuration(pf, false);
15006         if (ret) {
15007                 dev_info(&pf->pdev->dev,
15008                          "couldn't fetch switch config, err %pe aq_err %s\n",
15009                          ERR_PTR(ret),
15010                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15011                 return ret;
15012         }
15013         i40e_pf_reset_stats(pf);
15014
15015         /* set the switch config bit for the whole device to
15016          * support limited promisc or true promisc
15017          * when user requests promisc. The default is limited
15018          * promisc.
15019         */
15020
15021         if ((pf->hw.pf_id == 0) &&
15022             !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
15023                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15024                 pf->last_sw_conf_flags = flags;
15025         }
15026
15027         if (pf->hw.pf_id == 0) {
15028                 u16 valid_flags;
15029
15030                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15031                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15032                                                 NULL);
15033                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15034                         dev_info(&pf->pdev->dev,
15035                                  "couldn't set switch config bits, err %pe aq_err %s\n",
15036                                  ERR_PTR(ret),
15037                                  i40e_aq_str(&pf->hw,
15038                                              pf->hw.aq.asq_last_status));
15039                         /* not a fatal problem, just keep going */
15040                 }
15041                 pf->last_sw_conf_valid_flags = valid_flags;
15042         }
15043
15044         /* first time setup */
15045         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
15046                 struct i40e_vsi *vsi = NULL;
15047                 u16 uplink_seid;
15048
15049                 /* Set up the PF VSI associated with the PF's main VSI
15050                  * that is already in the HW switch
15051                  */
15052                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
15053                         uplink_seid = pf->veb[pf->lan_veb]->seid;
15054                 else
15055                         uplink_seid = pf->mac_seid;
15056                 if (pf->lan_vsi == I40E_NO_VSI)
15057                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
15058                 else if (reinit)
15059                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
15060                 if (!vsi) {
15061                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15062                         i40e_cloud_filter_exit(pf);
15063                         i40e_fdir_teardown(pf);
15064                         return -EAGAIN;
15065                 }
15066         } else {
15067                 /* force a reset of TC and queue layout configurations */
15068                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
15069
15070                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
15071                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
15072                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
15073         }
15074         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
15075
15076         i40e_fdir_sb_setup(pf);
15077
15078         /* Setup static PF queue filter control settings */
15079         ret = i40e_setup_pf_filter_control(pf);
15080         if (ret) {
15081                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15082                          ret);
15083                 /* Failure here should not stop continuing other steps */
15084         }
15085
15086         /* enable RSS in the HW, even for only one queue, as the stack can use
15087          * the hash
15088          */
15089         if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15090                 i40e_pf_config_rss(pf);
15091
15092         /* fill in link information and enable LSE reporting */
15093         i40e_link_event(pf);
15094
15095         i40e_ptp_init(pf);
15096
15097         if (!lock_acquired)
15098                 rtnl_lock();
15099
15100         /* repopulate tunnel port filters */
15101         udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
15102
15103         if (!lock_acquired)
15104                 rtnl_unlock();
15105
15106         return ret;
15107 }
15108
15109 /**
15110  * i40e_determine_queue_usage - Work out queue distribution
15111  * @pf: board private structure
15112  **/
15113 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15114 {
15115         int queues_left;
15116         int q_max;
15117
15118         pf->num_lan_qps = 0;
15119
15120         /* Find the max queues to be put into basic use.  We'll always be
15121          * using TC0, whether or not DCB is running, and TC0 will get the
15122          * big RSS set.
15123          */
15124         queues_left = pf->hw.func_caps.num_tx_qp;
15125
15126         if ((queues_left == 1) ||
15127             !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15128                 /* one qp for PF, no queues for anything else */
15129                 queues_left = 0;
15130                 pf->alloc_rss_size = pf->num_lan_qps = 1;
15131
15132                 /* make sure all the fancies are disabled */
15133                 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15134                 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15135                 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15136                 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15137                 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15138                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15139                 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
15140                 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15141                 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15142         } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
15143                    !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
15144                    !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
15145                    !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
15146                 /* one qp for PF */
15147                 pf->alloc_rss_size = pf->num_lan_qps = 1;
15148                 queues_left -= pf->num_lan_qps;
15149
15150                 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15151                 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15152                 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15153                 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15154                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15155                 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15156                 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15157         } else {
15158                 /* Not enough queues for all TCs */
15159                 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
15160                     queues_left < I40E_MAX_TRAFFIC_CLASS) {
15161                         clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15162                         clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15163                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15164                 }
15165
15166                 /* limit lan qps to the smaller of qps, cpus or msix */
15167                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15168                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15169                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15170                 pf->num_lan_qps = q_max;
15171
15172                 queues_left -= pf->num_lan_qps;
15173         }
15174
15175         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15176                 if (queues_left > 1) {
15177                         queues_left -= 1; /* save 1 queue for FD */
15178                 } else {
15179                         clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15180                         set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15181                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15182                 }
15183         }
15184
15185         if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15186             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15187                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15188                                         (queues_left / pf->num_vf_qps));
15189                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15190         }
15191
15192         if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
15193             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15194                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15195                                           (queues_left / pf->num_vmdq_qps));
15196                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15197         }
15198
15199         pf->queues_left = queues_left;
15200         dev_dbg(&pf->pdev->dev,
15201                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15202                 pf->hw.func_caps.num_tx_qp,
15203                 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
15204                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15205                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15206                 queues_left);
15207 }
15208
15209 /**
15210  * i40e_setup_pf_filter_control - Setup PF static filter control
15211  * @pf: PF to be setup
15212  *
15213  * i40e_setup_pf_filter_control sets up a PF's initial filter control
15214  * settings. If PE/FCoE are enabled then it will also set the per PF
15215  * based filter sizes required for them. It also enables Flow director,
15216  * ethertype and macvlan type filter settings for the pf.
15217  *
15218  * Returns 0 on success, negative on failure
15219  **/
15220 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15221 {
15222         struct i40e_filter_control_settings *settings = &pf->filter_settings;
15223
15224         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15225
15226         /* Flow Director is enabled */
15227         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
15228             test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15229                 settings->enable_fdir = true;
15230
15231         /* Ethtype and MACVLAN filters enabled for PF */
15232         settings->enable_ethtype = true;
15233         settings->enable_macvlan = true;
15234
15235         if (i40e_set_filter_control(&pf->hw, settings))
15236                 return -ENOENT;
15237
15238         return 0;
15239 }
15240
15241 #define INFO_STRING_LEN 255
15242 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
15243 static void i40e_print_features(struct i40e_pf *pf)
15244 {
15245         struct i40e_hw *hw = &pf->hw;
15246         char *buf;
15247         int i;
15248
15249         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15250         if (!buf)
15251                 return;
15252
15253         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15254 #ifdef CONFIG_PCI_IOV
15255         i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15256 #endif
15257         i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15258                       pf->hw.func_caps.num_vsis,
15259                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
15260         if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15261                 i += scnprintf(&buf[i], REMAIN(i), " RSS");
15262         if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15263                 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15264         if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15265                 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15266                 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15267         }
15268         if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
15269                 i += scnprintf(&buf[i], REMAIN(i), " DCB");
15270         i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15271         i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15272         if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
15273                 i += scnprintf(&buf[i], REMAIN(i), " PTP");
15274         if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
15275                 i += scnprintf(&buf[i], REMAIN(i), " VEB");
15276         else
15277                 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15278
15279         dev_info(&pf->pdev->dev, "%s\n", buf);
15280         kfree(buf);
15281         WARN_ON(i > INFO_STRING_LEN);
15282 }
15283
15284 /**
15285  * i40e_get_platform_mac_addr - get platform-specific MAC address
15286  * @pdev: PCI device information struct
15287  * @pf: board private structure
15288  *
15289  * Look up the MAC address for the device. First we'll try
15290  * eth_platform_get_mac_address, which will check Open Firmware, or arch
15291  * specific fallback. Otherwise, we'll default to the stored value in
15292  * firmware.
15293  **/
15294 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15295 {
15296         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15297                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15298 }
15299
15300 /**
15301  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15302  * @fec_cfg: FEC option to set in flags
15303  * @flags: ptr to flags in which we set FEC option
15304  **/
15305 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
15306 {
15307         if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
15308                 set_bit(I40E_FLAG_RS_FEC, flags);
15309                 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15310         }
15311         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15312             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15313                 set_bit(I40E_FLAG_RS_FEC, flags);
15314                 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15315         }
15316         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15317             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15318                 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15319                 clear_bit(I40E_FLAG_RS_FEC, flags);
15320         }
15321         if (fec_cfg == 0) {
15322                 clear_bit(I40E_FLAG_RS_FEC, flags);
15323                 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15324         }
15325 }
15326
15327 /**
15328  * i40e_check_recovery_mode - check if we are running transition firmware
15329  * @pf: board private structure
15330  *
15331  * Check registers indicating the firmware runs in recovery mode. Sets the
15332  * appropriate driver state.
15333  *
15334  * Returns true if the recovery mode was detected, false otherwise
15335  **/
15336 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15337 {
15338         u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15339
15340         if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15341                 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15342                 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15343                 set_bit(__I40E_RECOVERY_MODE, pf->state);
15344
15345                 return true;
15346         }
15347         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15348                 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15349
15350         return false;
15351 }
15352
15353 /**
15354  * i40e_pf_loop_reset - perform reset in a loop.
15355  * @pf: board private structure
15356  *
15357  * This function is useful when a NIC is about to enter recovery mode.
15358  * When a NIC's internal data structures are corrupted the NIC's
15359  * firmware is going to enter recovery mode.
15360  * Right after a POR it takes about 7 minutes for firmware to enter
15361  * recovery mode. Until that time a NIC is in some kind of intermediate
15362  * state. After that time period the NIC almost surely enters
15363  * recovery mode. The only way for a driver to detect intermediate
15364  * state is to issue a series of pf-resets and check a return value.
15365  * If a PF reset returns success then the firmware could be in recovery
15366  * mode so the caller of this code needs to check for recovery mode
15367  * if this function returns success. There is a little chance that
15368  * firmware will hang in intermediate state forever.
15369  * Since waiting 7 minutes is quite a lot of time this function waits
15370  * 10 seconds and then gives up by returning an error.
15371  *
15372  * Return 0 on success, negative on failure.
15373  **/
15374 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15375 {
15376         /* wait max 10 seconds for PF reset to succeed */
15377         const unsigned long time_end = jiffies + 10 * HZ;
15378         struct i40e_hw *hw = &pf->hw;
15379         int ret;
15380
15381         ret = i40e_pf_reset(hw);
15382         while (ret != 0 && time_before(jiffies, time_end)) {
15383                 usleep_range(10000, 20000);
15384                 ret = i40e_pf_reset(hw);
15385         }
15386
15387         if (ret == 0)
15388                 pf->pfr_count++;
15389         else
15390                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15391
15392         return ret;
15393 }
15394
15395 /**
15396  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15397  * @pf: board private structure
15398  *
15399  * Check FW registers to determine if FW issued unexpected EMP Reset.
15400  * Every time when unexpected EMP Reset occurs the FW increments
15401  * a counter of unexpected EMP Resets. When the counter reaches 10
15402  * the FW should enter the Recovery mode
15403  *
15404  * Returns true if FW issued unexpected EMP Reset
15405  **/
15406 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15407 {
15408         const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15409                            I40E_GL_FWSTS_FWS1B_MASK;
15410         return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15411                (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15412 }
15413
15414 /**
15415  * i40e_handle_resets - handle EMP resets and PF resets
15416  * @pf: board private structure
15417  *
15418  * Handle both EMP resets and PF resets and conclude whether there are
15419  * any issues regarding these resets. If there are any issues then
15420  * generate log entry.
15421  *
15422  * Return 0 if NIC is healthy or negative value when there are issues
15423  * with resets
15424  **/
15425 static int i40e_handle_resets(struct i40e_pf *pf)
15426 {
15427         const int pfr = i40e_pf_loop_reset(pf);
15428         const bool is_empr = i40e_check_fw_empr(pf);
15429
15430         if (is_empr || pfr != 0)
15431                 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15432
15433         return is_empr ? -EIO : pfr;
15434 }
15435
15436 /**
15437  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15438  * @pf: board private structure
15439  * @hw: ptr to the hardware info
15440  *
15441  * This function does a minimal setup of all subsystems needed for running
15442  * recovery mode.
15443  *
15444  * Returns 0 on success, negative on failure
15445  **/
15446 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15447 {
15448         struct i40e_vsi *vsi;
15449         int err;
15450         int v_idx;
15451
15452         pci_set_drvdata(pf->pdev, pf);
15453         pci_save_state(pf->pdev);
15454
15455         /* set up periodic task facility */
15456         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15457         pf->service_timer_period = HZ;
15458
15459         INIT_WORK(&pf->service_task, i40e_service_task);
15460         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15461
15462         err = i40e_init_interrupt_scheme(pf);
15463         if (err)
15464                 goto err_switch_setup;
15465
15466         /* The number of VSIs reported by the FW is the minimum guaranteed
15467          * to us; HW supports far more and we share the remaining pool with
15468          * the other PFs. We allocate space for more than the guarantee with
15469          * the understanding that we might not get them all later.
15470          */
15471         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15472                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15473         else
15474                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15475
15476         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15477         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15478                           GFP_KERNEL);
15479         if (!pf->vsi) {
15480                 err = -ENOMEM;
15481                 goto err_switch_setup;
15482         }
15483
15484         /* We allocate one VSI which is needed as absolute minimum
15485          * in order to register the netdev
15486          */
15487         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15488         if (v_idx < 0) {
15489                 err = v_idx;
15490                 goto err_switch_setup;
15491         }
15492         pf->lan_vsi = v_idx;
15493         vsi = pf->vsi[v_idx];
15494         if (!vsi) {
15495                 err = -EFAULT;
15496                 goto err_switch_setup;
15497         }
15498         vsi->alloc_queue_pairs = 1;
15499         err = i40e_config_netdev(vsi);
15500         if (err)
15501                 goto err_switch_setup;
15502         err = register_netdev(vsi->netdev);
15503         if (err)
15504                 goto err_switch_setup;
15505         vsi->netdev_registered = true;
15506         i40e_dbg_pf_init(pf);
15507
15508         err = i40e_setup_misc_vector_for_recovery_mode(pf);
15509         if (err)
15510                 goto err_switch_setup;
15511
15512         /* tell the firmware that we're starting */
15513         i40e_send_version(pf);
15514
15515         /* since everything's happy, start the service_task timer */
15516         mod_timer(&pf->service_timer,
15517                   round_jiffies(jiffies + pf->service_timer_period));
15518
15519         return 0;
15520
15521 err_switch_setup:
15522         i40e_reset_interrupt_capability(pf);
15523         timer_shutdown_sync(&pf->service_timer);
15524         i40e_shutdown_adminq(hw);
15525         iounmap(hw->hw_addr);
15526         pci_release_mem_regions(pf->pdev);
15527         pci_disable_device(pf->pdev);
15528         i40e_free_pf(pf);
15529
15530         return err;
15531 }
15532
15533 /**
15534  * i40e_set_subsystem_device_id - set subsystem device id
15535  * @hw: pointer to the hardware info
15536  *
15537  * Set PCI subsystem device id either from a pci_dev structure or
15538  * a specific FW register.
15539  **/
15540 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15541 {
15542         struct i40e_pf *pf = i40e_hw_to_pf(hw);
15543
15544         hw->subsystem_device_id = pf->pdev->subsystem_device ?
15545                 pf->pdev->subsystem_device :
15546                 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15547 }
15548
15549 /**
15550  * i40e_probe - Device initialization routine
15551  * @pdev: PCI device information struct
15552  * @ent: entry in i40e_pci_tbl
15553  *
15554  * i40e_probe initializes a PF identified by a pci_dev structure.
15555  * The OS initialization, configuring of the PF private structure,
15556  * and a hardware reset occur.
15557  *
15558  * Returns 0 on success, negative on failure
15559  **/
15560 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15561 {
15562         struct i40e_aq_get_phy_abilities_resp abilities;
15563 #ifdef CONFIG_I40E_DCB
15564         enum i40e_get_fw_lldp_status_resp lldp_status;
15565 #endif /* CONFIG_I40E_DCB */
15566         struct i40e_vsi *vsi;
15567         struct i40e_pf *pf;
15568         struct i40e_hw *hw;
15569         u16 wol_nvm_bits;
15570         char nvm_ver[32];
15571         u16 link_status;
15572 #ifdef CONFIG_I40E_DCB
15573         int status;
15574 #endif /* CONFIG_I40E_DCB */
15575         int err;
15576         u32 val;
15577
15578         err = pci_enable_device_mem(pdev);
15579         if (err)
15580                 return err;
15581
15582         /* set up for high or low dma */
15583         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15584         if (err) {
15585                 dev_err(&pdev->dev,
15586                         "DMA configuration failed: 0x%x\n", err);
15587                 goto err_dma;
15588         }
15589
15590         /* set up pci connections */
15591         err = pci_request_mem_regions(pdev, i40e_driver_name);
15592         if (err) {
15593                 dev_info(&pdev->dev,
15594                          "pci_request_selected_regions failed %d\n", err);
15595                 goto err_pci_reg;
15596         }
15597
15598         pci_set_master(pdev);
15599
15600         /* Now that we have a PCI connection, we need to do the
15601          * low level device setup.  This is primarily setting up
15602          * the Admin Queue structures and then querying for the
15603          * device's current profile information.
15604          */
15605         pf = i40e_alloc_pf(&pdev->dev);
15606         if (!pf) {
15607                 err = -ENOMEM;
15608                 goto err_pf_alloc;
15609         }
15610         pf->next_vsi = 0;
15611         pf->pdev = pdev;
15612         set_bit(__I40E_DOWN, pf->state);
15613
15614         hw = &pf->hw;
15615
15616         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15617                                 I40E_MAX_CSR_SPACE);
15618         /* We believe that the highest register to read is
15619          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15620          * is not less than that before mapping to prevent a
15621          * kernel panic.
15622          */
15623         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15624                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15625                         pf->ioremap_len);
15626                 err = -ENOMEM;
15627                 goto err_ioremap;
15628         }
15629         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15630         if (!hw->hw_addr) {
15631                 err = -EIO;
15632                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15633                          (unsigned int)pci_resource_start(pdev, 0),
15634                          pf->ioremap_len, err);
15635                 goto err_ioremap;
15636         }
15637         hw->vendor_id = pdev->vendor;
15638         hw->device_id = pdev->device;
15639         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15640         hw->subsystem_vendor_id = pdev->subsystem_vendor;
15641         i40e_set_subsystem_device_id(hw);
15642         hw->bus.device = PCI_SLOT(pdev->devfn);
15643         hw->bus.func = PCI_FUNC(pdev->devfn);
15644         hw->bus.bus_id = pdev->bus->number;
15645
15646         /* Select something other than the 802.1ad ethertype for the
15647          * switch to use internally and drop on ingress.
15648          */
15649         hw->switch_tag = 0xffff;
15650         hw->first_tag = ETH_P_8021AD;
15651         hw->second_tag = ETH_P_8021Q;
15652
15653         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15654         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15655         INIT_LIST_HEAD(&pf->ddp_old_prof);
15656
15657         /* set up the locks for the AQ, do this only once in probe
15658          * and destroy them only once in remove
15659          */
15660         mutex_init(&hw->aq.asq_mutex);
15661         mutex_init(&hw->aq.arq_mutex);
15662
15663         pf->msg_enable = netif_msg_init(debug,
15664                                         NETIF_MSG_DRV |
15665                                         NETIF_MSG_PROBE |
15666                                         NETIF_MSG_LINK);
15667         if (debug < -1)
15668                 pf->hw.debug_mask = debug;
15669
15670         /* do a special CORER for clearing PXE mode once at init */
15671         if (hw->revision_id == 0 &&
15672             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15673                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15674                 i40e_flush(hw);
15675                 msleep(200);
15676                 pf->corer_count++;
15677
15678                 i40e_clear_pxe_mode(hw);
15679         }
15680
15681         /* Reset here to make sure all is clean and to define PF 'n' */
15682         i40e_clear_hw(hw);
15683
15684         err = i40e_set_mac_type(hw);
15685         if (err) {
15686                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15687                          err);
15688                 goto err_pf_reset;
15689         }
15690
15691         err = i40e_handle_resets(pf);
15692         if (err)
15693                 goto err_pf_reset;
15694
15695         i40e_check_recovery_mode(pf);
15696
15697         if (is_kdump_kernel()) {
15698                 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15699                 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15700         } else {
15701                 hw->aq.num_arq_entries = I40E_AQ_LEN;
15702                 hw->aq.num_asq_entries = I40E_AQ_LEN;
15703         }
15704         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15705         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15706
15707         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15708                  "%s-%s:misc",
15709                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15710
15711         err = i40e_init_shared_code(hw);
15712         if (err) {
15713                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15714                          err);
15715                 goto err_pf_reset;
15716         }
15717
15718         /* set up a default setting for link flow control */
15719         pf->hw.fc.requested_mode = I40E_FC_NONE;
15720
15721         err = i40e_init_adminq(hw);
15722         if (err) {
15723                 if (err == -EIO)
15724                         dev_info(&pdev->dev,
15725                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15726                                  hw->aq.api_maj_ver,
15727                                  hw->aq.api_min_ver,
15728                                  I40E_FW_API_VERSION_MAJOR,
15729                                  I40E_FW_MINOR_VERSION(hw));
15730                 else
15731                         dev_info(&pdev->dev,
15732                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15733
15734                 goto err_pf_reset;
15735         }
15736         i40e_get_oem_version(hw);
15737         i40e_get_pba_string(hw);
15738
15739         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15740         i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver));
15741         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15742                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15743                  hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,
15744                  hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
15745                  hw->subsystem_device_id);
15746
15747         if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
15748                                   I40E_FW_MINOR_VERSION(hw) + 1))
15749                 dev_dbg(&pdev->dev,
15750                         "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15751                          hw->aq.api_maj_ver,
15752                          hw->aq.api_min_ver,
15753                          I40E_FW_API_VERSION_MAJOR,
15754                          I40E_FW_MINOR_VERSION(hw));
15755         else if (i40e_is_aq_api_ver_lt(hw, 1, 4))
15756                 dev_info(&pdev->dev,
15757                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15758                          hw->aq.api_maj_ver,
15759                          hw->aq.api_min_ver,
15760                          I40E_FW_API_VERSION_MAJOR,
15761                          I40E_FW_MINOR_VERSION(hw));
15762
15763         i40e_verify_eeprom(pf);
15764
15765         /* Rev 0 hardware was never productized */
15766         if (hw->revision_id < 1)
15767                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15768
15769         i40e_clear_pxe_mode(hw);
15770
15771         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15772         if (err)
15773                 goto err_adminq_setup;
15774
15775         err = i40e_sw_init(pf);
15776         if (err) {
15777                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15778                 goto err_sw_init;
15779         }
15780
15781         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15782                 return i40e_init_recovery_mode(pf, hw);
15783
15784         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15785                                 hw->func_caps.num_rx_qp, 0, 0);
15786         if (err) {
15787                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15788                 goto err_init_lan_hmc;
15789         }
15790
15791         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15792         if (err) {
15793                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15794                 err = -ENOENT;
15795                 goto err_configure_lan_hmc;
15796         }
15797
15798         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15799          * Ignore error return codes because if it was already disabled via
15800          * hardware settings this will fail
15801          */
15802         if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
15803                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15804                 i40e_aq_stop_lldp(hw, true, false, NULL);
15805         }
15806
15807         /* allow a platform config to override the HW addr */
15808         i40e_get_platform_mac_addr(pdev, pf);
15809
15810         if (!is_valid_ether_addr(hw->mac.addr)) {
15811                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15812                 err = -EIO;
15813                 goto err_mac_addr;
15814         }
15815         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15816         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15817         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15818         if (is_valid_ether_addr(hw->mac.port_addr))
15819                 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);
15820
15821         i40e_ptp_alloc_pins(pf);
15822         pci_set_drvdata(pdev, pf);
15823         pci_save_state(pdev);
15824
15825 #ifdef CONFIG_I40E_DCB
15826         status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15827         (!status &&
15828          lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15829                 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :
15830                 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));
15831         dev_info(&pdev->dev,
15832                  test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
15833                         "FW LLDP is disabled\n" :
15834                         "FW LLDP is enabled\n");
15835
15836         /* Enable FW to write default DCB config on link-up */
15837         i40e_aq_set_dcb_parameters(hw, true, NULL);
15838
15839         err = i40e_init_pf_dcb(pf);
15840         if (err) {
15841                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15842                 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15843                 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15844                 /* Continue without DCB enabled */
15845         }
15846 #endif /* CONFIG_I40E_DCB */
15847
15848         /* set up periodic task facility */
15849         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15850         pf->service_timer_period = HZ;
15851
15852         INIT_WORK(&pf->service_task, i40e_service_task);
15853         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15854
15855         /* NVM bit on means WoL disabled for the port */
15856         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15857         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15858                 pf->wol_en = false;
15859         else
15860                 pf->wol_en = true;
15861         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15862
15863         /* set up the main switch operations */
15864         i40e_determine_queue_usage(pf);
15865         err = i40e_init_interrupt_scheme(pf);
15866         if (err)
15867                 goto err_switch_setup;
15868
15869         /* Reduce Tx and Rx pairs for kdump
15870          * When MSI-X is enabled, it's not allowed to use more TC queue
15871          * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15872          * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15873          */
15874         if (is_kdump_kernel())
15875                 pf->num_lan_msix = 1;
15876
15877         pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15878         pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15879         pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15880         pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15881         pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15882         pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15883                                                     UDP_TUNNEL_TYPE_GENEVE;
15884
15885         /* The number of VSIs reported by the FW is the minimum guaranteed
15886          * to us; HW supports far more and we share the remaining pool with
15887          * the other PFs. We allocate space for more than the guarantee with
15888          * the understanding that we might not get them all later.
15889          */
15890         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15891                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15892         else
15893                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15894         if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15895                 dev_warn(&pf->pdev->dev,
15896                          "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15897                          pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15898                 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15899         }
15900
15901         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15902         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15903                           GFP_KERNEL);
15904         if (!pf->vsi) {
15905                 err = -ENOMEM;
15906                 goto err_switch_setup;
15907         }
15908
15909 #ifdef CONFIG_PCI_IOV
15910         /* prep for VF support */
15911         if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15912             test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15913             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15914                 if (pci_num_vf(pdev))
15915                         set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
15916         }
15917 #endif
15918         err = i40e_setup_pf_switch(pf, false, false);
15919         if (err) {
15920                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15921                 goto err_vsis;
15922         }
15923         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15924
15925         /* if FDIR VSI was set up, start it now */
15926         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
15927         if (vsi)
15928                 i40e_vsi_open(vsi);
15929
15930         /* The driver only wants link up/down and module qualification
15931          * reports from firmware.  Note the negative logic.
15932          */
15933         err = i40e_aq_set_phy_int_mask(&pf->hw,
15934                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15935                                          I40E_AQ_EVENT_MEDIA_NA |
15936                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15937         if (err)
15938                 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
15939                          ERR_PTR(err),
15940                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15941
15942         /* Reconfigure hardware for allowing smaller MSS in the case
15943          * of TSO, so that we avoid the MDD being fired and causing
15944          * a reset in the case of small MSS+TSO.
15945          */
15946         val = rd32(hw, I40E_REG_MSS);
15947         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15948                 val &= ~I40E_REG_MSS_MIN_MASK;
15949                 val |= I40E_64BYTE_MSS;
15950                 wr32(hw, I40E_REG_MSS, val);
15951         }
15952
15953         if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
15954                 msleep(75);
15955                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15956                 if (err)
15957                         dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
15958                                  ERR_PTR(err),
15959                                  i40e_aq_str(&pf->hw,
15960                                              pf->hw.aq.asq_last_status));
15961         }
15962         /* The main driver is (mostly) up and happy. We need to set this state
15963          * before setting up the misc vector or we get a race and the vector
15964          * ends up disabled forever.
15965          */
15966         clear_bit(__I40E_DOWN, pf->state);
15967
15968         /* In case of MSIX we are going to setup the misc vector right here
15969          * to handle admin queue events etc. In case of legacy and MSI
15970          * the misc functionality and queue processing is combined in
15971          * the same vector and that gets setup at open.
15972          */
15973         if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15974                 err = i40e_setup_misc_vector(pf);
15975                 if (err) {
15976                         dev_info(&pdev->dev,
15977                                  "setup of misc vector failed: %d\n", err);
15978                         i40e_cloud_filter_exit(pf);
15979                         i40e_fdir_teardown(pf);
15980                         goto err_vsis;
15981                 }
15982         }
15983
15984 #ifdef CONFIG_PCI_IOV
15985         /* prep for VF support */
15986         if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15987             test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15988             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15989                 /* disable link interrupts for VFs */
15990                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15991                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15992                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15993                 i40e_flush(hw);
15994
15995                 if (pci_num_vf(pdev)) {
15996                         dev_info(&pdev->dev,
15997                                  "Active VFs found, allocating resources.\n");
15998                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15999                         if (err)
16000                                 dev_info(&pdev->dev,
16001                                          "Error %d allocating resources for existing VFs\n",
16002                                          err);
16003                 }
16004         }
16005 #endif /* CONFIG_PCI_IOV */
16006
16007         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16008                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16009                                                       pf->num_iwarp_msix,
16010                                                       I40E_IWARP_IRQ_PILE_ID);
16011                 if (pf->iwarp_base_vector < 0) {
16012                         dev_info(&pdev->dev,
16013                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
16014                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
16015                         clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
16016                 }
16017         }
16018
16019         i40e_dbg_pf_init(pf);
16020
16021         /* tell the firmware that we're starting */
16022         i40e_send_version(pf);
16023
16024         /* since everything's happy, start the service_task timer */
16025         mod_timer(&pf->service_timer,
16026                   round_jiffies(jiffies + pf->service_timer_period));
16027
16028         /* add this PF to client device list and launch a client service task */
16029         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16030                 err = i40e_lan_add_device(pf);
16031                 if (err)
16032                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16033                                  err);
16034         }
16035
16036 #define PCI_SPEED_SIZE 8
16037 #define PCI_WIDTH_SIZE 8
16038         /* Devices on the IOSF bus do not have this information
16039          * and will report PCI Gen 1 x 1 by default so don't bother
16040          * checking them.
16041          */
16042         if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
16043                 char speed[PCI_SPEED_SIZE] = "Unknown";
16044                 char width[PCI_WIDTH_SIZE] = "Unknown";
16045
16046                 /* Get the negotiated link width and speed from PCI config
16047                  * space
16048                  */
16049                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16050                                           &link_status);
16051
16052                 i40e_set_pci_config_data(hw, link_status);
16053
16054                 switch (hw->bus.speed) {
16055                 case i40e_bus_speed_8000:
16056                         strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16057                 case i40e_bus_speed_5000:
16058                         strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16059                 case i40e_bus_speed_2500:
16060                         strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16061                 default:
16062                         break;
16063                 }
16064                 switch (hw->bus.width) {
16065                 case i40e_bus_width_pcie_x8:
16066                         strscpy(width, "8", PCI_WIDTH_SIZE); break;
16067                 case i40e_bus_width_pcie_x4:
16068                         strscpy(width, "4", PCI_WIDTH_SIZE); break;
16069                 case i40e_bus_width_pcie_x2:
16070                         strscpy(width, "2", PCI_WIDTH_SIZE); break;
16071                 case i40e_bus_width_pcie_x1:
16072                         strscpy(width, "1", PCI_WIDTH_SIZE); break;
16073                 default:
16074                         break;
16075                 }
16076
16077                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16078                          speed, width);
16079
16080                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16081                     hw->bus.speed < i40e_bus_speed_8000) {
16082                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16083                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16084                 }
16085         }
16086
16087         /* get the requested speeds from the fw */
16088         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16089         if (err)
16090                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %pe last_status =  %s\n",
16091                         ERR_PTR(err),
16092                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16093         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16094
16095         /* set the FEC config due to the board capabilities */
16096         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);
16097
16098         /* get the supported phy types from the fw */
16099         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16100         if (err)
16101                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %pe last_status =  %s\n",
16102                         ERR_PTR(err),
16103                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16104
16105         /* make sure the MFS hasn't been set lower than the default */
16106 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16107         val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,
16108                         rd32(&pf->hw, I40E_PRTGL_SAH));
16109         if (val < MAX_FRAME_SIZE_DEFAULT)
16110                 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
16111                          pf->hw.port, val);
16112
16113         /* Add a filter to drop all Flow control frames from any VSI from being
16114          * transmitted. By doing so we stop a malicious VF from sending out
16115          * PAUSE or PFC frames and potentially controlling traffic for other
16116          * PF/VF VSIs.
16117          * The FW can still send Flow control frames if enabled.
16118          */
16119         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16120                                                        pf->main_vsi_seid);
16121
16122         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16123             (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16124                 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);
16125         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16126                 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);
16127         /* print a string summarizing features */
16128         i40e_print_features(pf);
16129
16130         i40e_devlink_register(pf);
16131
16132         return 0;
16133
16134         /* Unwind what we've done if something failed in the setup */
16135 err_vsis:
16136         set_bit(__I40E_DOWN, pf->state);
16137         i40e_clear_interrupt_scheme(pf);
16138         kfree(pf->vsi);
16139 err_switch_setup:
16140         i40e_reset_interrupt_capability(pf);
16141         timer_shutdown_sync(&pf->service_timer);
16142 err_mac_addr:
16143 err_configure_lan_hmc:
16144         (void)i40e_shutdown_lan_hmc(hw);
16145 err_init_lan_hmc:
16146         kfree(pf->qp_pile);
16147 err_sw_init:
16148 err_adminq_setup:
16149 err_pf_reset:
16150         iounmap(hw->hw_addr);
16151 err_ioremap:
16152         i40e_free_pf(pf);
16153 err_pf_alloc:
16154         pci_release_mem_regions(pdev);
16155 err_pci_reg:
16156 err_dma:
16157         pci_disable_device(pdev);
16158         return err;
16159 }
16160
16161 /**
16162  * i40e_remove - Device removal routine
16163  * @pdev: PCI device information struct
16164  *
16165  * i40e_remove is called by the PCI subsystem to alert the driver
16166  * that is should release a PCI device.  This could be caused by a
16167  * Hot-Plug event, or because the driver is going to be removed from
16168  * memory.
16169  **/
16170 static void i40e_remove(struct pci_dev *pdev)
16171 {
16172         struct i40e_pf *pf = pci_get_drvdata(pdev);
16173         struct i40e_hw *hw = &pf->hw;
16174         struct i40e_vsi *vsi;
16175         struct i40e_veb *veb;
16176         int ret_code;
16177         int i;
16178
16179         i40e_devlink_unregister(pf);
16180
16181         i40e_dbg_pf_exit(pf);
16182
16183         i40e_ptp_stop(pf);
16184
16185         /* Disable RSS in hw */
16186         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16187         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16188
16189         /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16190          * flags, once they are set, i40e_rebuild should not be called as
16191          * i40e_prep_for_reset always returns early.
16192          */
16193         while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16194                 usleep_range(1000, 2000);
16195         set_bit(__I40E_IN_REMOVE, pf->state);
16196
16197         if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
16198                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16199                 i40e_free_vfs(pf);
16200                 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
16201         }
16202         /* no more scheduling of any task */
16203         set_bit(__I40E_SUSPENDED, pf->state);
16204         set_bit(__I40E_DOWN, pf->state);
16205         if (pf->service_timer.function)
16206                 timer_shutdown_sync(&pf->service_timer);
16207         if (pf->service_task.func)
16208                 cancel_work_sync(&pf->service_task);
16209
16210         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16211                 struct i40e_vsi *vsi = pf->vsi[0];
16212
16213                 /* We know that we have allocated only one vsi for this PF,
16214                  * it was just for registering netdevice, so the interface
16215                  * could be visible in the 'ifconfig' output
16216                  */
16217                 unregister_netdev(vsi->netdev);
16218                 free_netdev(vsi->netdev);
16219
16220                 goto unmap;
16221         }
16222
16223         /* Client close must be called explicitly here because the timer
16224          * has been stopped.
16225          */
16226         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16227
16228         i40e_fdir_teardown(pf);
16229
16230         /* If there is a switch structure or any orphans, remove them.
16231          * This will leave only the PF's VSI remaining.
16232          */
16233         i40e_pf_for_each_veb(pf, i, veb)
16234                 if (veb->uplink_seid == pf->mac_seid ||
16235                     veb->uplink_seid == 0)
16236                         i40e_switch_branch_release(veb);
16237
16238         /* Now we can shutdown the PF's VSIs, just before we kill
16239          * adminq and hmc.
16240          */
16241         i40e_pf_for_each_vsi(pf, i, vsi) {
16242                 i40e_vsi_close(vsi);
16243                 i40e_vsi_release(vsi);
16244                 pf->vsi[i] = NULL;
16245         }
16246
16247         i40e_cloud_filter_exit(pf);
16248
16249         /* remove attached clients */
16250         if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16251                 ret_code = i40e_lan_del_device(pf);
16252                 if (ret_code)
16253                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16254                                  ret_code);
16255         }
16256
16257         /* shutdown and destroy the HMC */
16258         if (hw->hmc.hmc_obj) {
16259                 ret_code = i40e_shutdown_lan_hmc(hw);
16260                 if (ret_code)
16261                         dev_warn(&pdev->dev,
16262                                  "Failed to destroy the HMC resources: %d\n",
16263                                  ret_code);
16264         }
16265
16266 unmap:
16267         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16268         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16269             !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16270                 free_irq(pf->pdev->irq, pf);
16271
16272         /* shutdown the adminq */
16273         i40e_shutdown_adminq(hw);
16274
16275         /* destroy the locks only once, here */
16276         mutex_destroy(&hw->aq.arq_mutex);
16277         mutex_destroy(&hw->aq.asq_mutex);
16278
16279         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16280         rtnl_lock();
16281         i40e_clear_interrupt_scheme(pf);
16282         i40e_pf_for_each_vsi(pf, i, vsi) {
16283                 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16284                         i40e_vsi_clear_rings(vsi);
16285
16286                 i40e_vsi_clear(vsi);
16287                 pf->vsi[i] = NULL;
16288         }
16289         rtnl_unlock();
16290
16291         i40e_pf_for_each_veb(pf, i, veb) {
16292                 kfree(veb);
16293                 pf->veb[i] = NULL;
16294         }
16295
16296         kfree(pf->qp_pile);
16297         kfree(pf->vsi);
16298
16299         iounmap(hw->hw_addr);
16300         i40e_free_pf(pf);
16301         pci_release_mem_regions(pdev);
16302
16303         pci_disable_device(pdev);
16304 }
16305
16306 /**
16307  * i40e_pci_error_detected - warning that something funky happened in PCI land
16308  * @pdev: PCI device information struct
16309  * @error: the type of PCI error
16310  *
16311  * Called to warn that something happened and the error handling steps
16312  * are in progress.  Allows the driver to quiesce things, be ready for
16313  * remediation.
16314  **/
16315 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16316                                                 pci_channel_state_t error)
16317 {
16318         struct i40e_pf *pf = pci_get_drvdata(pdev);
16319
16320         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16321
16322         if (!pf) {
16323                 dev_info(&pdev->dev,
16324                          "Cannot recover - error happened during device probe\n");
16325                 return PCI_ERS_RESULT_DISCONNECT;
16326         }
16327
16328         /* shutdown all operations */
16329         if (!test_bit(__I40E_SUSPENDED, pf->state))
16330                 i40e_prep_for_reset(pf);
16331
16332         /* Request a slot reset */
16333         return PCI_ERS_RESULT_NEED_RESET;
16334 }
16335
16336 /**
16337  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16338  * @pdev: PCI device information struct
16339  *
16340  * Called to find if the driver can work with the device now that
16341  * the pci slot has been reset.  If a basic connection seems good
16342  * (registers are readable and have sane content) then return a
16343  * happy little PCI_ERS_RESULT_xxx.
16344  **/
16345 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16346 {
16347         struct i40e_pf *pf = pci_get_drvdata(pdev);
16348         pci_ers_result_t result;
16349         u32 reg;
16350
16351         dev_dbg(&pdev->dev, "%s\n", __func__);
16352         if (pci_enable_device_mem(pdev)) {
16353                 dev_info(&pdev->dev,
16354                          "Cannot re-enable PCI device after reset.\n");
16355                 result = PCI_ERS_RESULT_DISCONNECT;
16356         } else {
16357                 pci_set_master(pdev);
16358                 pci_restore_state(pdev);
16359                 pci_save_state(pdev);
16360                 pci_wake_from_d3(pdev, false);
16361
16362                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16363                 if (reg == 0)
16364                         result = PCI_ERS_RESULT_RECOVERED;
16365                 else
16366                         result = PCI_ERS_RESULT_DISCONNECT;
16367         }
16368
16369         return result;
16370 }
16371
16372 /**
16373  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16374  * @pdev: PCI device information struct
16375  */
16376 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16377 {
16378         struct i40e_pf *pf = pci_get_drvdata(pdev);
16379
16380         i40e_prep_for_reset(pf);
16381 }
16382
16383 /**
16384  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16385  * @pdev: PCI device information struct
16386  */
16387 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16388 {
16389         struct i40e_pf *pf = pci_get_drvdata(pdev);
16390
16391         if (test_bit(__I40E_IN_REMOVE, pf->state))
16392                 return;
16393
16394         i40e_reset_and_rebuild(pf, false, false);
16395 #ifdef CONFIG_PCI_IOV
16396         i40e_restore_all_vfs_msi_state(pdev);
16397 #endif /* CONFIG_PCI_IOV */
16398 }
16399
16400 /**
16401  * i40e_pci_error_resume - restart operations after PCI error recovery
16402  * @pdev: PCI device information struct
16403  *
16404  * Called to allow the driver to bring things back up after PCI error
16405  * and/or reset recovery has finished.
16406  **/
16407 static void i40e_pci_error_resume(struct pci_dev *pdev)
16408 {
16409         struct i40e_pf *pf = pci_get_drvdata(pdev);
16410
16411         dev_dbg(&pdev->dev, "%s\n", __func__);
16412         if (test_bit(__I40E_SUSPENDED, pf->state))
16413                 return;
16414
16415         i40e_handle_reset_warning(pf, false);
16416 }
16417
16418 /**
16419  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16420  * using the mac_address_write admin q function
16421  * @pf: pointer to i40e_pf struct
16422  **/
16423 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16424 {
16425         struct i40e_hw *hw = &pf->hw;
16426         u8 mac_addr[6];
16427         u16 flags = 0;
16428         int ret;
16429
16430         /* Get current MAC address in case it's an LAA */
16431         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16432                 ether_addr_copy(mac_addr,
16433                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16434         } else {
16435                 dev_err(&pf->pdev->dev,
16436                         "Failed to retrieve MAC address; using default\n");
16437                 ether_addr_copy(mac_addr, hw->mac.addr);
16438         }
16439
16440         /* The FW expects the mac address write cmd to first be called with
16441          * one of these flags before calling it again with the multicast
16442          * enable flags.
16443          */
16444         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16445
16446         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16447                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16448
16449         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16450         if (ret) {
16451                 dev_err(&pf->pdev->dev,
16452                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16453                 return;
16454         }
16455
16456         flags = I40E_AQC_MC_MAG_EN
16457                         | I40E_AQC_WOL_PRESERVE_ON_PFR
16458                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16459         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16460         if (ret)
16461                 dev_err(&pf->pdev->dev,
16462                         "Failed to enable Multicast Magic Packet wake up\n");
16463 }
16464
16465 /**
16466  * i40e_shutdown - PCI callback for shutting down
16467  * @pdev: PCI device information struct
16468  **/
16469 static void i40e_shutdown(struct pci_dev *pdev)
16470 {
16471         struct i40e_pf *pf = pci_get_drvdata(pdev);
16472         struct i40e_hw *hw = &pf->hw;
16473
16474         set_bit(__I40E_SUSPENDED, pf->state);
16475         set_bit(__I40E_DOWN, pf->state);
16476
16477         del_timer_sync(&pf->service_timer);
16478         cancel_work_sync(&pf->service_task);
16479         i40e_cloud_filter_exit(pf);
16480         i40e_fdir_teardown(pf);
16481
16482         /* Client close must be called explicitly here because the timer
16483          * has been stopped.
16484          */
16485         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16486
16487         if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16488             pf->wol_en)
16489                 i40e_enable_mc_magic_wake(pf);
16490
16491         i40e_prep_for_reset(pf);
16492
16493         wr32(hw, I40E_PFPM_APM,
16494              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16495         wr32(hw, I40E_PFPM_WUFC,
16496              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16497
16498         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16499         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16500             !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16501                 free_irq(pf->pdev->irq, pf);
16502
16503         /* Since we're going to destroy queues during the
16504          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16505          * whole section
16506          */
16507         rtnl_lock();
16508         i40e_clear_interrupt_scheme(pf);
16509         rtnl_unlock();
16510
16511         if (system_state == SYSTEM_POWER_OFF) {
16512                 pci_wake_from_d3(pdev, pf->wol_en);
16513                 pci_set_power_state(pdev, PCI_D3hot);
16514         }
16515 }
16516
16517 /**
16518  * i40e_suspend - PM callback for moving to D3
16519  * @dev: generic device information structure
16520  **/
16521 static int __maybe_unused i40e_suspend(struct device *dev)
16522 {
16523         struct i40e_pf *pf = dev_get_drvdata(dev);
16524         struct i40e_hw *hw = &pf->hw;
16525
16526         /* If we're already suspended, then there is nothing to do */
16527         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16528                 return 0;
16529
16530         set_bit(__I40E_DOWN, pf->state);
16531
16532         /* Ensure service task will not be running */
16533         del_timer_sync(&pf->service_timer);
16534         cancel_work_sync(&pf->service_task);
16535
16536         /* Client close must be called explicitly here because the timer
16537          * has been stopped.
16538          */
16539         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16540
16541         if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16542             pf->wol_en)
16543                 i40e_enable_mc_magic_wake(pf);
16544
16545         /* Since we're going to destroy queues during the
16546          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16547          * whole section
16548          */
16549         rtnl_lock();
16550
16551         i40e_prep_for_reset(pf);
16552
16553         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16554         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16555
16556         /* Clear the interrupt scheme and release our IRQs so that the system
16557          * can safely hibernate even when there are a large number of CPUs.
16558          * Otherwise hibernation might fail when mapping all the vectors back
16559          * to CPU0.
16560          */
16561         i40e_clear_interrupt_scheme(pf);
16562
16563         rtnl_unlock();
16564
16565         return 0;
16566 }
16567
16568 /**
16569  * i40e_resume - PM callback for waking up from D3
16570  * @dev: generic device information structure
16571  **/
16572 static int __maybe_unused i40e_resume(struct device *dev)
16573 {
16574         struct i40e_pf *pf = dev_get_drvdata(dev);
16575         int err;
16576
16577         /* If we're not suspended, then there is nothing to do */
16578         if (!test_bit(__I40E_SUSPENDED, pf->state))
16579                 return 0;
16580
16581         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16582          * since we're going to be restoring queues
16583          */
16584         rtnl_lock();
16585
16586         /* We cleared the interrupt scheme when we suspended, so we need to
16587          * restore it now to resume device functionality.
16588          */
16589         err = i40e_restore_interrupt_scheme(pf);
16590         if (err) {
16591                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16592                         err);
16593         }
16594
16595         clear_bit(__I40E_DOWN, pf->state);
16596         i40e_reset_and_rebuild(pf, false, true);
16597
16598         rtnl_unlock();
16599
16600         /* Clear suspended state last after everything is recovered */
16601         clear_bit(__I40E_SUSPENDED, pf->state);
16602
16603         /* Restart the service task */
16604         mod_timer(&pf->service_timer,
16605                   round_jiffies(jiffies + pf->service_timer_period));
16606
16607         return 0;
16608 }
16609
16610 static const struct pci_error_handlers i40e_err_handler = {
16611         .error_detected = i40e_pci_error_detected,
16612         .slot_reset = i40e_pci_error_slot_reset,
16613         .reset_prepare = i40e_pci_error_reset_prepare,
16614         .reset_done = i40e_pci_error_reset_done,
16615         .resume = i40e_pci_error_resume,
16616 };
16617
16618 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16619
16620 static struct pci_driver i40e_driver = {
16621         .name     = i40e_driver_name,
16622         .id_table = i40e_pci_tbl,
16623         .probe    = i40e_probe,
16624         .remove   = i40e_remove,
16625         .driver   = {
16626                 .pm = &i40e_pm_ops,
16627         },
16628         .shutdown = i40e_shutdown,
16629         .err_handler = &i40e_err_handler,
16630         .sriov_configure = i40e_pci_sriov_configure,
16631 };
16632
16633 /**
16634  * i40e_init_module - Driver registration routine
16635  *
16636  * i40e_init_module is the first routine called when the driver is
16637  * loaded. All it does is register with the PCI subsystem.
16638  **/
16639 static int __init i40e_init_module(void)
16640 {
16641         int err;
16642
16643         pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16644         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16645
16646         /* There is no need to throttle the number of active tasks because
16647          * each device limits its own task using a state bit for scheduling
16648          * the service task, and the device tasks do not interfere with each
16649          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16650          * since we need to be able to guarantee forward progress even under
16651          * memory pressure.
16652          */
16653         i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name);
16654         if (!i40e_wq) {
16655                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16656                 return -ENOMEM;
16657         }
16658
16659         i40e_dbg_init();
16660         err = pci_register_driver(&i40e_driver);
16661         if (err) {
16662                 destroy_workqueue(i40e_wq);
16663                 i40e_dbg_exit();
16664                 return err;
16665         }
16666
16667         return 0;
16668 }
16669 module_init(i40e_init_module);
16670
16671 /**
16672  * i40e_exit_module - Driver exit cleanup routine
16673  *
16674  * i40e_exit_module is called just before the driver is removed
16675  * from memory.
16676  **/
16677 static void __exit i40e_exit_module(void)
16678 {
16679         pci_unregister_driver(&i40e_driver);
16680         destroy_workqueue(i40e_wq);
16681         ida_destroy(&i40e_client_ida);
16682         i40e_dbg_exit();
16683 }
16684 module_exit(i40e_exit_module);