drivers/net/ethernet/amd/pds_core/main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
   3
   4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6 #include <linux/pci.h>
   7
   8 #include <linux/pds/pds_common.h>
   9
  10 #include "core.h"
  11
  12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
  13 MODULE_AUTHOR("Advanced Micro Devices, Inc");
  14 MODULE_LICENSE("GPL");
  15
  16 /* Supported devices */
  17 static const struct pci_device_id pdsc_id_table[] = {
  18         { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
  19         { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
  20         { 0, }  /* end of table */
  21 };
  22 MODULE_DEVICE_TABLE(pci, pdsc_id_table);
  23
  24 static void pdsc_wdtimer_cb(struct timer_list *t)
  25 {
  26         struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);
  27
  28         dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
  29         mod_timer(&pdsc->wdtimer,
  30                   round_jiffies(jiffies + pdsc->wdtimer_period));
  31
  32         queue_work(pdsc->wq, &pdsc->health_work);
  33 }
  34
  35 static void pdsc_unmap_bars(struct pdsc *pdsc)
  36 {
  37         struct pdsc_dev_bar *bars = pdsc->bars;
  38         unsigned int i;
  39
  40         pdsc->info_regs = NULL;
  41         pdsc->cmd_regs = NULL;
  42         pdsc->intr_status = NULL;
  43         pdsc->intr_ctrl = NULL;
  44
  45         for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
  46                 if (bars[i].vaddr)
  47                         pci_iounmap(pdsc->pdev, bars[i].vaddr);
  48                 bars[i].vaddr = NULL;
  49         }
  50 }
  51
  52 static int pdsc_map_bars(struct pdsc *pdsc)
  53 {
  54         struct pdsc_dev_bar *bar = pdsc->bars;
  55         struct pci_dev *pdev = pdsc->pdev;
  56         struct device *dev = pdsc->dev;
  57         struct pdsc_dev_bar *bars;
  58         unsigned int i, j;
  59         int num_bars = 0;
  60         int err;
  61         u32 sig;
  62
  63         bars = pdsc->bars;
  64
  65         /* Since the PCI interface in the hardware is configurable,
  66          * we need to poke into all the bars to find the set we're
  67          * expecting.
  68          */
  69         for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
  70                 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
  71                         continue;
  72
  73                 bars[j].len = pci_resource_len(pdev, i);
  74                 bars[j].bus_addr = pci_resource_start(pdev, i);
  75                 bars[j].res_index = i;
  76
  77                 /* only map the whole bar 0 */
  78                 if (j > 0) {
  79                         bars[j].vaddr = NULL;
  80                 } else {
  81                         bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
  82                         if (!bars[j].vaddr) {
  83                                 dev_err(dev, "Cannot map BAR %d, aborting\n", i);
  84                                 return -ENODEV;
  85                         }
  86                 }
  87
  88                 j++;
  89         }
  90         num_bars = j;
  91
  92         /* BAR0: dev_cmd and interrupts */
  93         if (num_bars < 1) {
  94                 dev_err(dev, "No bars found\n");
  95                 err = -EFAULT;
  96                 goto err_out;
  97         }
  98
  99         if (bar->len < PDS_CORE_BAR0_SIZE) {
 100                 dev_err(dev, "Resource bar size %lu too small\n", bar->len);
 101                 err = -EFAULT;
 102                 goto err_out;
 103         }
 104
 105         pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
 106         pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
 107         pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
 108         pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
 109
 110         sig = ioread32(&pdsc->info_regs->signature);
 111         if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
 112                 dev_err(dev, "Incompatible firmware signature %x", sig);
 113                 err = -EFAULT;
 114                 goto err_out;
 115         }
 116
 117         /* BAR1: doorbells */
 118         bar++;
 119         if (num_bars < 2) {
 120                 dev_err(dev, "Doorbell bar missing\n");
 121                 err = -EFAULT;
 122                 goto err_out;
 123         }
 124
 125         pdsc->db_pages = bar->vaddr;
 126         pdsc->phy_db_pages = bar->bus_addr;
 127
 128         return 0;
 129
 130 err_out:
 131         pdsc_unmap_bars(pdsc);
 132         return err;
 133 }
 134
 135 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
 136 {
 137         return pci_iomap_range(pdsc->pdev,
 138                                pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
 139                                (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
 140 }
 141
 142 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 143 {
 144         struct pdsc *pdsc = pci_get_drvdata(pdev);
 145         struct device *dev = pdsc->dev;
 146         int ret = 0;
 147
 148         if (num_vfs > 0) {
 149                 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
 150                                     GFP_KERNEL);
 151                 if (!pdsc->vfs)
 152                         return -ENOMEM;
 153                 pdsc->num_vfs = num_vfs;
 154
 155                 ret = pci_enable_sriov(pdev, num_vfs);
 156                 if (ret) {
 157                         dev_err(dev, "Cannot enable SRIOV: %pe\n",
 158                                 ERR_PTR(ret));
 159                         goto no_vfs;
 160                 }
 161
 162                 return num_vfs;
 163         }
 164
 165 no_vfs:
 166         pci_disable_sriov(pdev);
 167
 168         kfree(pdsc->vfs);
 169         pdsc->vfs = NULL;
 170         pdsc->num_vfs = 0;
 171
 172         return ret;
 173 }
 174
 175 static int pdsc_init_vf(struct pdsc *vf)
 176 {
 177         struct devlink *dl;
 178         struct pdsc *pf;
 179         int err;
 180
 181         pf = pdsc_get_pf_struct(vf->pdev);
 182         if (IS_ERR_OR_NULL(pf))
 183                 return PTR_ERR(pf) ?: -1;
 184
 185         vf->vf_id = pci_iov_vf_id(vf->pdev);
 186
 187         dl = priv_to_devlink(vf);
 188         devl_lock(dl);
 189         devl_register(dl);
 190         devl_unlock(dl);
 191
 192         pf->vfs[vf->vf_id].vf = vf;
 193         err = pdsc_auxbus_dev_add(vf, pf);
 194         if (err) {
 195                 devl_lock(dl);
 196                 devl_unregister(dl);
 197                 devl_unlock(dl);
 198         }
 199
 200         return err;
 201 }
 202
 203 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
 204         .name = "fw",
 205         .diagnose = pdsc_fw_reporter_diagnose,
 206 };
 207
 208 static const struct devlink_param pdsc_dl_params[] = {
 209         DEVLINK_PARAM_GENERIC(ENABLE_VNET,
 210                               BIT(DEVLINK_PARAM_CMODE_RUNTIME),
 211                               pdsc_dl_enable_get,
 212                               pdsc_dl_enable_set,
 213                               pdsc_dl_enable_validate),
 214 };
 215
 216 #define PDSC_WQ_NAME_LEN 24
 217
 218 static int pdsc_init_pf(struct pdsc *pdsc)
 219 {
 220         struct devlink_health_reporter *hr;
 221         char wq_name[PDSC_WQ_NAME_LEN];
 222         struct devlink *dl;
 223         int err;
 224
 225         pcie_print_link_status(pdsc->pdev);
 226
 227         err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
 228         if (err) {
 229                 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
 230                         ERR_PTR(err));
 231                 return err;
 232         }
 233
 234         err = pdsc_map_bars(pdsc);
 235         if (err)
 236                 goto err_out_release_regions;
 237
 238         /* General workqueue and timer, but don't start timer yet */
 239         snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
 240         pdsc->wq = create_singlethread_workqueue(wq_name);
 241         INIT_WORK(&pdsc->health_work, pdsc_health_thread);
 242         INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
 243         timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
 244         pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
 245
 246         mutex_init(&pdsc->devcmd_lock);
 247         mutex_init(&pdsc->config_lock);
 248         spin_lock_init(&pdsc->adminq_lock);
 249
 250         mutex_lock(&pdsc->config_lock);
 251         set_bit(PDSC_S_FW_DEAD, &pdsc->state);
 252
 253         err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
 254         if (err) {
 255                 mutex_unlock(&pdsc->config_lock);
 256                 goto err_out_unmap_bars;
 257         }
 258
 259         err = pdsc_start(pdsc);
 260         if (err) {
 261                 mutex_unlock(&pdsc->config_lock);
 262                 goto err_out_teardown;
 263         }
 264
 265         mutex_unlock(&pdsc->config_lock);
 266
 267         dl = priv_to_devlink(pdsc);
 268         devl_lock(dl);
 269         err = devl_params_register(dl, pdsc_dl_params,
 270                                    ARRAY_SIZE(pdsc_dl_params));
 271         if (err) {
 272                 devl_unlock(dl);
 273                 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
 274                          ERR_PTR(err));
 275                 goto err_out_stop;
 276         }
 277
 278         hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
 279         if (IS_ERR(hr)) {
 280                 devl_unlock(dl);
 281                 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
 282                 err = PTR_ERR(hr);
 283                 goto err_out_unreg_params;
 284         }
 285         pdsc->fw_reporter = hr;
 286
 287         devl_register(dl);
 288         devl_unlock(dl);
 289
 290         /* Lastly, start the health check timer */
 291         mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
 292
 293         return 0;
 294
 295 err_out_unreg_params:
 296         devlink_params_unregister(dl, pdsc_dl_params,
 297                                   ARRAY_SIZE(pdsc_dl_params));
 298 err_out_stop:
 299         pdsc_stop(pdsc);
 300 err_out_teardown:
 301         pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
 302 err_out_unmap_bars:
 303         timer_shutdown_sync(&pdsc->wdtimer);
 304         if (pdsc->wq)
 305                 destroy_workqueue(pdsc->wq);
 306         mutex_destroy(&pdsc->config_lock);
 307         mutex_destroy(&pdsc->devcmd_lock);
 308         pci_free_irq_vectors(pdsc->pdev);
 309         pdsc_unmap_bars(pdsc);
 310 err_out_release_regions:
 311         pci_release_regions(pdsc->pdev);
 312
 313         return err;
 314 }
 315
 316 static const struct devlink_ops pdsc_dl_ops = {
 317         .info_get       = pdsc_dl_info_get,
 318         .flash_update   = pdsc_dl_flash_update,
 319 };
 320
 321 static const struct devlink_ops pdsc_dl_vf_ops = {
 322 };
 323
 324 static DEFINE_IDA(pdsc_ida);
 325
 326 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 327 {
 328         struct device *dev = &pdev->dev;
 329         const struct devlink_ops *ops;
 330         struct devlink *dl;
 331         struct pdsc *pdsc;
 332         bool is_pf;
 333         int err;
 334
 335         is_pf = !pdev->is_virtfn;
 336         ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
 337         dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
 338         if (!dl)
 339                 return -ENOMEM;
 340         pdsc = devlink_priv(dl);
 341
 342         pdsc->pdev = pdev;
 343         pdsc->dev = &pdev->dev;
 344         set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
 345         pci_set_drvdata(pdev, pdsc);
 346         pdsc_debugfs_add_dev(pdsc);
 347
 348         err = ida_alloc(&pdsc_ida, GFP_KERNEL);
 349         if (err < 0) {
 350                 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
 351                         __func__, ERR_PTR(err));
 352                 goto err_out_free_devlink;
 353         }
 354         pdsc->uid = err;
 355
 356         /* Query system for DMA addressing limitation for the device. */
 357         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
 358         if (err) {
 359                 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
 360                         ERR_PTR(err));
 361                 goto err_out_free_ida;
 362         }
 363
 364         err = pci_enable_device(pdev);
 365         if (err) {
 366                 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
 367                 goto err_out_free_ida;
 368         }
 369         pci_set_master(pdev);
 370
 371         if (is_pf)
 372                 err = pdsc_init_pf(pdsc);
 373         else
 374                 err = pdsc_init_vf(pdsc);
 375         if (err) {
 376                 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
 377                 goto err_out_disable_device;
 378         }
 379
 380         clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
 381         return 0;
 382
 383 err_out_disable_device:
 384         pci_disable_device(pdev);
 385 err_out_free_ida:
 386         ida_free(&pdsc_ida, pdsc->uid);
 387 err_out_free_devlink:
 388         pdsc_debugfs_del_dev(pdsc);
 389         devlink_free(dl);
 390
 391         return err;
 392 }
 393
 394 static void pdsc_remove(struct pci_dev *pdev)
 395 {
 396         struct pdsc *pdsc = pci_get_drvdata(pdev);
 397         struct devlink *dl;
 398
 399         /* Unhook the registrations first to be sure there
 400          * are no requests while we're stopping.
 401          */
 402         dl = priv_to_devlink(pdsc);
 403         devl_lock(dl);
 404         devl_unregister(dl);
 405         if (!pdev->is_virtfn) {
 406                 if (pdsc->fw_reporter) {
 407                         devl_health_reporter_destroy(pdsc->fw_reporter);
 408                         pdsc->fw_reporter = NULL;
 409                 }
 410                 devl_params_unregister(dl, pdsc_dl_params,
 411                                        ARRAY_SIZE(pdsc_dl_params));
 412         }
 413         devl_unlock(dl);
 414
 415         if (pdev->is_virtfn) {
 416                 struct pdsc *pf;
 417
 418                 pf = pdsc_get_pf_struct(pdsc->pdev);
 419                 if (!IS_ERR(pf)) {
 420                         pdsc_auxbus_dev_del(pdsc, pf);
 421                         pf->vfs[pdsc->vf_id].vf = NULL;
 422                 }
 423         } else {
 424                 /* Remove the VFs and their aux_bus connections before other
 425                  * cleanup so that the clients can use the AdminQ to cleanly
 426                  * shut themselves down.
 427                  */
 428                 pdsc_sriov_configure(pdev, 0);
 429
 430                 timer_shutdown_sync(&pdsc->wdtimer);
 431                 if (pdsc->wq)
 432                         destroy_workqueue(pdsc->wq);
 433
 434                 mutex_lock(&pdsc->config_lock);
 435                 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
 436
 437                 pdsc_stop(pdsc);
 438                 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
 439                 mutex_unlock(&pdsc->config_lock);
 440                 mutex_destroy(&pdsc->config_lock);
 441                 mutex_destroy(&pdsc->devcmd_lock);
 442
 443                 pdsc_unmap_bars(pdsc);
 444                 pci_release_regions(pdev);
 445         }
 446
 447         pci_disable_device(pdev);
 448
 449         ida_free(&pdsc_ida, pdsc->uid);
 450         pdsc_debugfs_del_dev(pdsc);
 451         devlink_free(dl);
 452 }
 453
 454 static void pdsc_stop_health_thread(struct pdsc *pdsc)
 455 {
 456         if (pdsc->pdev->is_virtfn)
 457                 return;
 458
 459         timer_shutdown_sync(&pdsc->wdtimer);
 460         if (pdsc->health_work.func)
 461                 cancel_work_sync(&pdsc->health_work);
 462 }
 463
 464 static void pdsc_restart_health_thread(struct pdsc *pdsc)
 465 {
 466         if (pdsc->pdev->is_virtfn)
 467                 return;
 468
 469         timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
 470         mod_timer(&pdsc->wdtimer, jiffies + 1);
 471 }
 472
 473 static void pdsc_reset_prepare(struct pci_dev *pdev)
 474 {
 475         struct pdsc *pdsc = pci_get_drvdata(pdev);
 476
 477         pdsc_stop_health_thread(pdsc);
 478         pdsc_fw_down(pdsc);
 479
 480         if (pdev->is_virtfn) {
 481                 struct pdsc *pf;
 482
 483                 pf = pdsc_get_pf_struct(pdsc->pdev);
 484                 if (!IS_ERR(pf))
 485                         pdsc_auxbus_dev_del(pdsc, pf);
 486         }
 487
 488         pdsc_unmap_bars(pdsc);
 489         pci_release_regions(pdev);
 490         if (pci_is_enabled(pdev))
 491                 pci_disable_device(pdev);
 492 }
 493
 494 static void pdsc_reset_done(struct pci_dev *pdev)
 495 {
 496         struct pdsc *pdsc = pci_get_drvdata(pdev);
 497         struct device *dev = pdsc->dev;
 498         int err;
 499
 500         err = pci_enable_device(pdev);
 501         if (err) {
 502                 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
 503                 return;
 504         }
 505         pci_set_master(pdev);
 506
 507         if (!pdev->is_virtfn) {
 508                 pcie_print_link_status(pdsc->pdev);
 509
 510                 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
 511                 if (err) {
 512                         dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
 513                                 ERR_PTR(err));
 514                         return;
 515                 }
 516
 517                 err = pdsc_map_bars(pdsc);
 518                 if (err)
 519                         return;
 520         }
 521
 522         pdsc_fw_up(pdsc);
 523         pdsc_restart_health_thread(pdsc);
 524
 525         if (pdev->is_virtfn) {
 526                 struct pdsc *pf;
 527
 528                 pf = pdsc_get_pf_struct(pdsc->pdev);
 529                 if (!IS_ERR(pf))
 530                         pdsc_auxbus_dev_add(pdsc, pf);
 531         }
 532 }
 533
 534 static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,
 535                                                 pci_channel_state_t error)
 536 {
 537         if (error == pci_channel_io_frozen) {
 538                 pdsc_reset_prepare(pdev);
 539                 return PCI_ERS_RESULT_NEED_RESET;
 540         }
 541
 542         return PCI_ERS_RESULT_NONE;
 543 }
 544
 545 static void pdsc_pci_error_resume(struct pci_dev *pdev)
 546 {
 547         struct pdsc *pdsc = pci_get_drvdata(pdev);
 548
 549         if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
 550                 pci_reset_function_locked(pdev);
 551 }
 552
 553 static const struct pci_error_handlers pdsc_err_handler = {
 554         /* FLR handling */
 555         .reset_prepare      = pdsc_reset_prepare,
 556         .reset_done         = pdsc_reset_done,
 557
 558         /* AER handling */
 559         .error_detected     = pdsc_pci_error_detected,
 560         .resume             = pdsc_pci_error_resume,
 561 };
 562
 563 static struct pci_driver pdsc_driver = {
 564         .name = PDS_CORE_DRV_NAME,
 565         .id_table = pdsc_id_table,
 566         .probe = pdsc_probe,
 567         .remove = pdsc_remove,
 568         .sriov_configure = pdsc_sriov_configure,
 569         .err_handler = &pdsc_err_handler,
 570 };
 571
 572 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
 573 {
 574         return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
 575 }
 576 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
 577
 578 static int __init pdsc_init_module(void)
 579 {
 580         if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
 581                 return -EINVAL;
 582
 583         pdsc_debugfs_create();
 584         return pci_register_driver(&pdsc_driver);
 585 }
 586
 587 static void __exit pdsc_cleanup_module(void)
 588 {
 589         pci_unregister_driver(&pdsc_driver);
 590         pdsc_debugfs_destroy();
 591 }
 592
 593 module_init(pdsc_init_module);
 594 module_exit(pdsc_cleanup_module);