drivers/cxl/core/cdat.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2023 Intel Corporation. All rights reserved. */
   3 #include <linux/acpi.h>
   4 #include <linux/xarray.h>
   5 #include <linux/fw_table.h>
   6 #include <linux/node.h>
   7 #include <linux/overflow.h>
   8 #include "cxlpci.h"
   9 #include "cxlmem.h"
  10 #include "core.h"
  11 #include "cxl.h"
  12 #include "core.h"
  13
  14 struct dsmas_entry {
  15         struct range dpa_range;
  16         u8 handle;
  17         struct access_coordinate coord;
  18
  19         int entries;
  20         int qos_class;
  21 };
  22
  23 static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
  24                               const unsigned long end)
  25 {
  26         struct acpi_cdat_header *hdr = &header->cdat;
  27         struct acpi_cdat_dsmas *dsmas;
  28         int size = sizeof(*hdr) + sizeof(*dsmas);
  29         struct xarray *dsmas_xa = arg;
  30         struct dsmas_entry *dent;
  31         u16 len;
  32         int rc;
  33
  34         len = le16_to_cpu((__force __le16)hdr->length);
  35         if (len != size || (unsigned long)hdr + len > end) {
  36                 pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);
  37                 return -EINVAL;
  38         }
  39
  40         /* Skip common header */
  41         dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);
  42
  43         dent = kzalloc(sizeof(*dent), GFP_KERNEL);
  44         if (!dent)
  45                 return -ENOMEM;
  46
  47         dent->handle = dsmas->dsmad_handle;
  48         dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
  49         dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
  50                               le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;
  51
  52         rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);
  53         if (rc) {
  54                 kfree(dent);
  55                 return rc;
  56         }
  57
  58         return 0;
  59 }
  60
  61 static void cxl_access_coordinate_set(struct access_coordinate *coord,
  62                                       int access, unsigned int val)
  63 {
  64         switch (access) {
  65         case ACPI_HMAT_ACCESS_LATENCY:
  66                 coord->read_latency = val;
  67                 coord->write_latency = val;
  68                 break;
  69         case ACPI_HMAT_READ_LATENCY:
  70                 coord->read_latency = val;
  71                 break;
  72         case ACPI_HMAT_WRITE_LATENCY:
  73                 coord->write_latency = val;
  74                 break;
  75         case ACPI_HMAT_ACCESS_BANDWIDTH:
  76                 coord->read_bandwidth = val;
  77                 coord->write_bandwidth = val;
  78                 break;
  79         case ACPI_HMAT_READ_BANDWIDTH:
  80                 coord->read_bandwidth = val;
  81                 break;
  82         case ACPI_HMAT_WRITE_BANDWIDTH:
  83                 coord->write_bandwidth = val;
  84                 break;
  85         }
  86 }
  87
  88 static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,
  89                                const unsigned long end)
  90 {
  91         struct acpi_cdat_header *hdr = &header->cdat;
  92         struct acpi_cdat_dslbis *dslbis;
  93         int size = sizeof(*hdr) + sizeof(*dslbis);
  94         struct xarray *dsmas_xa = arg;
  95         struct dsmas_entry *dent;
  96         __le64 le_base;
  97         __le16 le_val;
  98         u64 val;
  99         u16 len;
 100         int rc;
 101
 102         len = le16_to_cpu((__force __le16)hdr->length);
 103         if (len != size || (unsigned long)hdr + len > end) {
 104                 pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);
 105                 return -EINVAL;
 106         }
 107
 108         /* Skip common header */
 109         dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);
 110
 111         /* Skip unrecognized data type */
 112         if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
 113                 return 0;
 114
 115         /* Not a memory type, skip */
 116         if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)
 117                 return 0;
 118
 119         dent = xa_load(dsmas_xa, dslbis->handle);
 120         if (!dent) {
 121                 pr_warn("No matching DSMAS entry for DSLBIS entry.\n");
 122                 return 0;
 123         }
 124
 125         le_base = (__force __le64)dslbis->entry_base_unit;
 126         le_val = (__force __le16)dslbis->entry[0];
 127         rc = check_mul_overflow(le64_to_cpu(le_base),
 128                                 le16_to_cpu(le_val), &val);
 129         if (rc)
 130                 pr_warn("DSLBIS value overflowed.\n");
 131
 132         cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val);
 133
 134         return 0;
 135 }
 136
 137 static int cdat_table_parse_output(int rc)
 138 {
 139         if (rc < 0)
 140                 return rc;
 141         if (rc == 0)
 142                 return -ENOENT;
 143
 144         return 0;
 145 }
 146
 147 static int cxl_cdat_endpoint_process(struct cxl_port *port,
 148                                      struct xarray *dsmas_xa)
 149 {
 150         int rc;
 151
 152         rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,
 153                               dsmas_xa, port->cdat.table, port->cdat.length);
 154         rc = cdat_table_parse_output(rc);
 155         if (rc)
 156                 return rc;
 157
 158         rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
 159                               dsmas_xa, port->cdat.table, port->cdat.length);
 160         return cdat_table_parse_output(rc);
 161 }
 162
 163 static int cxl_port_perf_data_calculate(struct cxl_port *port,
 164                                         struct xarray *dsmas_xa)
 165 {
 166         struct access_coordinate ep_c;
 167         struct access_coordinate coord[ACCESS_COORDINATE_MAX];
 168         struct dsmas_entry *dent;
 169         int valid_entries = 0;
 170         unsigned long index;
 171         int rc;
 172
 173         rc = cxl_endpoint_get_perf_coordinates(port, &ep_c);
 174         if (rc) {
 175                 dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n");
 176                 return rc;
 177         }
 178
 179         rc = cxl_hb_get_perf_coordinates(port, coord);
 180         if (rc)  {
 181                 dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n");
 182                 return rc;
 183         }
 184
 185         struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);
 186
 187         if (!cxl_root)
 188                 return -ENODEV;
 189
 190         if (!cxl_root->ops || !cxl_root->ops->qos_class)
 191                 return -EOPNOTSUPP;
 192
 193         xa_for_each(dsmas_xa, index, dent) {
 194                 int qos_class;
 195
 196                 cxl_coordinates_combine(&dent->coord, &dent->coord, &ep_c);
 197                 /*
 198                  * Keeping the host bridge coordinates separate from the dsmas
 199                  * coordinates in order to allow calculation of access class
 200                  * 0 and 1 for region later.
 201                  */
 202                 cxl_coordinates_combine(&coord[ACCESS_COORDINATE_CPU],
 203                                         &coord[ACCESS_COORDINATE_CPU],
 204                                         &dent->coord);
 205                 dent->entries = 1;
 206                 rc = cxl_root->ops->qos_class(cxl_root,
 207                                               &coord[ACCESS_COORDINATE_CPU],
 208                                               1, &qos_class);
 209                 if (rc != 1)
 210                         continue;
 211
 212                 valid_entries++;
 213                 dent->qos_class = qos_class;
 214         }
 215
 216         if (!valid_entries)
 217                 return -ENOENT;
 218
 219         return 0;
 220 }
 221
 222 static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,
 223                               struct cxl_dpa_perf *dpa_perf)
 224 {
 225         dpa_perf->dpa_range = dent->dpa_range;
 226         dpa_perf->coord = dent->coord;
 227         dpa_perf->qos_class = dent->qos_class;
 228         dev_dbg(dev,
 229                 "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",
 230                 dent->dpa_range.start, dpa_perf->qos_class,
 231                 dent->coord.read_bandwidth, dent->coord.write_bandwidth,
 232                 dent->coord.read_latency, dent->coord.write_latency);
 233 }
 234
 235 static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
 236                                      struct xarray *dsmas_xa)
 237 {
 238         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 239         struct device *dev = cxlds->dev;
 240         struct range pmem_range = {
 241                 .start = cxlds->pmem_res.start,
 242                 .end = cxlds->pmem_res.end,
 243         };
 244         struct range ram_range = {
 245                 .start = cxlds->ram_res.start,
 246                 .end = cxlds->ram_res.end,
 247         };
 248         struct dsmas_entry *dent;
 249         unsigned long index;
 250
 251         xa_for_each(dsmas_xa, index, dent) {
 252                 if (resource_size(&cxlds->ram_res) &&
 253                     range_contains(&ram_range, &dent->dpa_range))
 254                         update_perf_entry(dev, dent, &mds->ram_perf);
 255                 else if (resource_size(&cxlds->pmem_res) &&
 256                          range_contains(&pmem_range, &dent->dpa_range))
 257                         update_perf_entry(dev, dent, &mds->pmem_perf);
 258                 else
 259                         dev_dbg(dev, "no partition for dsmas dpa: %#llx\n",
 260                                 dent->dpa_range.start);
 261         }
 262 }
 263
 264 static int match_cxlrd_qos_class(struct device *dev, void *data)
 265 {
 266         int dev_qos_class = *(int *)data;
 267         struct cxl_root_decoder *cxlrd;
 268
 269         if (!is_root_decoder(dev))
 270                 return 0;
 271
 272         cxlrd = to_cxl_root_decoder(dev);
 273         if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)
 274                 return 0;
 275
 276         if (cxlrd->qos_class == dev_qos_class)
 277                 return 1;
 278
 279         return 0;
 280 }
 281
 282 static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf)
 283 {
 284         *dpa_perf = (struct cxl_dpa_perf) {
 285                 .qos_class = CXL_QOS_CLASS_INVALID,
 286         };
 287 }
 288
 289 static bool cxl_qos_match(struct cxl_port *root_port,
 290                           struct cxl_dpa_perf *dpa_perf)
 291 {
 292         if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)
 293                 return false;
 294
 295         if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class,
 296                                    match_cxlrd_qos_class))
 297                 return false;
 298
 299         return true;
 300 }
 301
 302 static int match_cxlrd_hb(struct device *dev, void *data)
 303 {
 304         struct device *host_bridge = data;
 305         struct cxl_switch_decoder *cxlsd;
 306         struct cxl_root_decoder *cxlrd;
 307
 308         if (!is_root_decoder(dev))
 309                 return 0;
 310
 311         cxlrd = to_cxl_root_decoder(dev);
 312         cxlsd = &cxlrd->cxlsd;
 313
 314         guard(rwsem_read)(&cxl_region_rwsem);
 315         for (int i = 0; i < cxlsd->nr_targets; i++) {
 316                 if (host_bridge == cxlsd->target[i]->dport_dev)
 317                         return 1;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int cxl_qos_class_verify(struct cxl_memdev *cxlmd)
 324 {
 325         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 326         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 327         struct cxl_port *root_port;
 328         int rc;
 329
 330         struct cxl_root *cxl_root __free(put_cxl_root) =
 331                 find_cxl_root(cxlmd->endpoint);
 332
 333         if (!cxl_root)
 334                 return -ENODEV;
 335
 336         root_port = &cxl_root->port;
 337
 338         /* Check that the QTG IDs are all sane between end device and root decoders */
 339         if (!cxl_qos_match(root_port, &mds->ram_perf))
 340                 reset_dpa_perf(&mds->ram_perf);
 341         if (!cxl_qos_match(root_port, &mds->pmem_perf))
 342                 reset_dpa_perf(&mds->pmem_perf);
 343
 344         /* Check to make sure that the device's host bridge is under a root decoder */
 345         rc = device_for_each_child(&root_port->dev,
 346                                    cxlmd->endpoint->host_bridge, match_cxlrd_hb);
 347         if (!rc) {
 348                 reset_dpa_perf(&mds->ram_perf);
 349                 reset_dpa_perf(&mds->pmem_perf);
 350         }
 351
 352         return rc;
 353 }
 354
 355 static void discard_dsmas(struct xarray *xa)
 356 {
 357         unsigned long index;
 358         void *ent;
 359
 360         xa_for_each(xa, index, ent) {
 361                 xa_erase(xa, index);
 362                 kfree(ent);
 363         }
 364         xa_destroy(xa);
 365 }
 366 DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))
 367
 368 void cxl_endpoint_parse_cdat(struct cxl_port *port)
 369 {
 370         struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
 371         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 372         struct xarray __dsmas_xa;
 373         struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;
 374         int rc;
 375
 376         xa_init(&__dsmas_xa);
 377         if (!port->cdat.table)
 378                 return;
 379
 380         rc = cxl_cdat_endpoint_process(port, dsmas_xa);
 381         if (rc < 0) {
 382                 dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
 383                 return;
 384         }
 385
 386         rc = cxl_port_perf_data_calculate(port, dsmas_xa);
 387         if (rc) {
 388                 dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
 389                 return;
 390         }
 391
 392         cxl_memdev_set_qos_class(cxlds, dsmas_xa);
 393         cxl_qos_class_verify(cxlmd);
 394         cxl_memdev_update_perf(cxlmd);
 395 }
 396 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL);
 397
 398 static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,
 399                                const unsigned long end)
 400 {
 401         struct acpi_cdat_sslbis_table {
 402                 struct acpi_cdat_header header;
 403                 struct acpi_cdat_sslbis sslbis_header;
 404                 struct acpi_cdat_sslbe entries[];
 405         } *tbl = (struct acpi_cdat_sslbis_table *)header;
 406         int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header);
 407         struct acpi_cdat_sslbis *sslbis;
 408         struct cxl_port *port = arg;
 409         struct device *dev = &port->dev;
 410         int remain, entries, i;
 411         u16 len;
 412
 413         len = le16_to_cpu((__force __le16)header->cdat.length);
 414         remain = len - size;
 415         if (!remain || remain % sizeof(tbl->entries[0]) ||
 416             (unsigned long)header + len > end) {
 417                 dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);
 418                 return -EINVAL;
 419         }
 420
 421         sslbis = &tbl->sslbis_header;
 422         /* Unrecognized data type, we can skip */
 423         if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
 424                 return 0;
 425
 426         entries = remain / sizeof(tbl->entries[0]);
 427         if (struct_size(tbl, entries, entries) != len)
 428                 return -EINVAL;
 429
 430         for (i = 0; i < entries; i++) {
 431                 u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id);
 432                 u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id);
 433                 __le64 le_base;
 434                 __le16 le_val;
 435                 struct cxl_dport *dport;
 436                 unsigned long index;
 437                 u16 dsp_id;
 438                 u64 val;
 439
 440                 switch (x) {
 441                 case ACPI_CDAT_SSLBIS_US_PORT:
 442                         dsp_id = y;
 443                         break;
 444                 case ACPI_CDAT_SSLBIS_ANY_PORT:
 445                         switch (y) {
 446                         case ACPI_CDAT_SSLBIS_US_PORT:
 447                                 dsp_id = x;
 448                                 break;
 449                         case ACPI_CDAT_SSLBIS_ANY_PORT:
 450                                 dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;
 451                                 break;
 452                         default:
 453                                 dsp_id = y;
 454                                 break;
 455                         }
 456                         break;
 457                 default:
 458                         dsp_id = x;
 459                         break;
 460                 }
 461
 462                 le_base = (__force __le64)tbl->sslbis_header.entry_base_unit;
 463                 le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth;
 464
 465                 if (check_mul_overflow(le64_to_cpu(le_base),
 466                                        le16_to_cpu(le_val), &val))
 467                         dev_warn(dev, "SSLBIS value overflowed!\n");
 468
 469                 xa_for_each(&port->dports, index, dport) {
 470                         if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||
 471                             dsp_id == dport->port_id)
 472                                 cxl_access_coordinate_set(&dport->sw_coord,
 473                                                           sslbis->data_type,
 474                                                           val);
 475                 }
 476         }
 477
 478         return 0;
 479 }
 480
 481 void cxl_switch_parse_cdat(struct cxl_port *port)
 482 {
 483         int rc;
 484
 485         if (!port->cdat.table)
 486                 return;
 487
 488         rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,
 489                               port, port->cdat.table, port->cdat.length);
 490         rc = cdat_table_parse_output(rc);
 491         if (rc)
 492                 dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);
 493 }
 494 EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL);
 495
 496 /**
 497  * cxl_coordinates_combine - Combine the two input coordinates
 498  *
 499  * @out: Output coordinate of c1 and c2 combined
 500  * @c1: input coordinates
 501  * @c2: input coordinates
 502  */
 503 void cxl_coordinates_combine(struct access_coordinate *out,
 504                              struct access_coordinate *c1,
 505                              struct access_coordinate *c2)
 506 {
 507                 if (c1->write_bandwidth && c2->write_bandwidth)
 508                         out->write_bandwidth = min(c1->write_bandwidth,
 509                                                    c2->write_bandwidth);
 510                 out->write_latency = c1->write_latency + c2->write_latency;
 511
 512                 if (c1->read_bandwidth && c2->read_bandwidth)
 513                         out->read_bandwidth = min(c1->read_bandwidth,
 514                                                   c2->read_bandwidth);
 515                 out->read_latency = c1->read_latency + c2->read_latency;
 516 }
 517
 518 MODULE_IMPORT_NS(CXL);
 519
 520 void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
 521                                     struct cxl_endpoint_decoder *cxled)
 522 {
 523         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 524         struct cxl_port *port = cxlmd->endpoint;
 525         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 526         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 527         struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX];
 528         struct access_coordinate coord;
 529         struct range dpa = {
 530                         .start = cxled->dpa_res->start,
 531                         .end = cxled->dpa_res->end,
 532         };
 533         struct cxl_dpa_perf *perf;
 534         int rc;
 535
 536         switch (cxlr->mode) {
 537         case CXL_DECODER_RAM:
 538                 perf = &mds->ram_perf;
 539                 break;
 540         case CXL_DECODER_PMEM:
 541                 perf = &mds->pmem_perf;
 542                 break;
 543         default:
 544                 return;
 545         }
 546
 547         lockdep_assert_held(&cxl_dpa_rwsem);
 548
 549         if (!range_contains(&perf->dpa_range, &dpa))
 550                 return;
 551
 552         rc = cxl_hb_get_perf_coordinates(port, hb_coord);
 553         if (rc)  {
 554                 dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n");
 555                 return;
 556         }
 557
 558         for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
 559                 /* Pickup the host bridge coords */
 560                 cxl_coordinates_combine(&coord, &hb_coord[i], &perf->coord);
 561
 562                 /* Get total bandwidth and the worst latency for the cxl region */
 563                 cxlr->coord[i].read_latency = max_t(unsigned int,
 564                                                     cxlr->coord[i].read_latency,
 565                                                     coord.read_latency);
 566                 cxlr->coord[i].write_latency = max_t(unsigned int,
 567                                                      cxlr->coord[i].write_latency,
 568                                                      coord.write_latency);
 569                 cxlr->coord[i].read_bandwidth += coord.read_bandwidth;
 570                 cxlr->coord[i].write_bandwidth += coord.write_bandwidth;
 571
 572                 /*
 573                  * Convert latency to nanosec from picosec to be consistent
 574                  * with the resulting latency coordinates computed by the
 575                  * HMAT_REPORTING code.
 576                  */
 577                 cxlr->coord[i].read_latency =
 578                         DIV_ROUND_UP(cxlr->coord[i].read_latency, 1000);
 579                 cxlr->coord[i].write_latency =
 580                         DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000);
 581         }
 582 }
 583
 584 int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr,
 585                                        enum access_coordinate_class access)
 586 {
 587         return hmat_update_target_coordinates(nid, &cxlr->coord[access], access);
 588 }
 589
 590 bool cxl_need_node_perf_attrs_update(int nid)
 591 {
 592         return !acpi_node_backed_by_real_pxm(nid);
 593 }