tools/testing/cxl/test/cxl.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright(c) 2021 Intel Corporation. All rights reserved.
   3
   4 #include <linux/platform_device.h>
   5 #include <linux/genalloc.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/acpi.h>
   9 #include <linux/pci.h>
  10 #include <linux/mm.h>
  11 #include <cxlmem.h>
  12
  13 #include "../watermark.h"
  14 #include "mock.h"
  15
  16 static int interleave_arithmetic;
  17
  18 #define FAKE_QTG_ID     42
  19
  20 #define NR_CXL_HOST_BRIDGES 2
  21 #define NR_CXL_SINGLE_HOST 1
  22 #define NR_CXL_RCH 1
  23 #define NR_CXL_ROOT_PORTS 2
  24 #define NR_CXL_SWITCH_PORTS 2
  25 #define NR_CXL_PORT_DECODERS 8
  26 #define NR_BRIDGES (NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + NR_CXL_RCH)
  27
  28 static struct platform_device *cxl_acpi;
  29 static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];
  30 #define NR_MULTI_ROOT (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS)
  31 static struct platform_device *cxl_root_port[NR_MULTI_ROOT];
  32 static struct platform_device *cxl_switch_uport[NR_MULTI_ROOT];
  33 #define NR_MEM_MULTI \
  34         (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS * NR_CXL_SWITCH_PORTS)
  35 static struct platform_device *cxl_switch_dport[NR_MEM_MULTI];
  36
  37 static struct platform_device *cxl_hb_single[NR_CXL_SINGLE_HOST];
  38 static struct platform_device *cxl_root_single[NR_CXL_SINGLE_HOST];
  39 static struct platform_device *cxl_swu_single[NR_CXL_SINGLE_HOST];
  40 #define NR_MEM_SINGLE (NR_CXL_SINGLE_HOST * NR_CXL_SWITCH_PORTS)
  41 static struct platform_device *cxl_swd_single[NR_MEM_SINGLE];
  42
  43 struct platform_device *cxl_mem[NR_MEM_MULTI];
  44 struct platform_device *cxl_mem_single[NR_MEM_SINGLE];
  45
  46 static struct platform_device *cxl_rch[NR_CXL_RCH];
  47 static struct platform_device *cxl_rcd[NR_CXL_RCH];
  48
  49 static inline bool is_multi_bridge(struct device *dev)
  50 {
  51         int i;
  52
  53         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++)
  54                 if (&cxl_host_bridge[i]->dev == dev)
  55                         return true;
  56         return false;
  57 }
  58
  59 static inline bool is_single_bridge(struct device *dev)
  60 {
  61         int i;
  62
  63         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++)
  64                 if (&cxl_hb_single[i]->dev == dev)
  65                         return true;
  66         return false;
  67 }
  68
  69 static struct acpi_device acpi0017_mock;
  70 static struct acpi_device host_bridge[NR_BRIDGES] = {
  71         [0] = {
  72                 .handle = &host_bridge[0],
  73                 .pnp.unique_id = "0",
  74         },
  75         [1] = {
  76                 .handle = &host_bridge[1],
  77                 .pnp.unique_id = "1",
  78         },
  79         [2] = {
  80                 .handle = &host_bridge[2],
  81                 .pnp.unique_id = "2",
  82         },
  83         [3] = {
  84                 .handle = &host_bridge[3],
  85                 .pnp.unique_id = "3",
  86         },
  87 };
  88
  89 static bool is_mock_dev(struct device *dev)
  90 {
  91         int i;
  92
  93         for (i = 0; i < ARRAY_SIZE(cxl_mem); i++)
  94                 if (dev == &cxl_mem[i]->dev)
  95                         return true;
  96         for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++)
  97                 if (dev == &cxl_mem_single[i]->dev)
  98                         return true;
  99         for (i = 0; i < ARRAY_SIZE(cxl_rcd); i++)
 100                 if (dev == &cxl_rcd[i]->dev)
 101                         return true;
 102         if (dev == &cxl_acpi->dev)
 103                 return true;
 104         return false;
 105 }
 106
 107 static bool is_mock_adev(struct acpi_device *adev)
 108 {
 109         int i;
 110
 111         if (adev == &acpi0017_mock)
 112                 return true;
 113
 114         for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
 115                 if (adev == &host_bridge[i])
 116                         return true;
 117
 118         return false;
 119 }
 120
 121 static struct {
 122         struct acpi_table_cedt cedt;
 123         struct acpi_cedt_chbs chbs[NR_BRIDGES];
 124         struct {
 125                 struct acpi_cedt_cfmws cfmws;
 126                 u32 target[1];
 127         } cfmws0;
 128         struct {
 129                 struct acpi_cedt_cfmws cfmws;
 130                 u32 target[2];
 131         } cfmws1;
 132         struct {
 133                 struct acpi_cedt_cfmws cfmws;
 134                 u32 target[1];
 135         } cfmws2;
 136         struct {
 137                 struct acpi_cedt_cfmws cfmws;
 138                 u32 target[2];
 139         } cfmws3;
 140         struct {
 141                 struct acpi_cedt_cfmws cfmws;
 142                 u32 target[1];
 143         } cfmws4;
 144         struct {
 145                 struct acpi_cedt_cfmws cfmws;
 146                 u32 target[1];
 147         } cfmws5;
 148         struct {
 149                 struct acpi_cedt_cfmws cfmws;
 150                 u32 target[1];
 151         } cfmws6;
 152         struct {
 153                 struct acpi_cedt_cfmws cfmws;
 154                 u32 target[2];
 155         } cfmws7;
 156         struct {
 157                 struct acpi_cedt_cfmws cfmws;
 158                 u32 target[4];
 159         } cfmws8;
 160         struct {
 161                 struct acpi_cedt_cxims cxims;
 162                 u64 xormap_list[2];
 163         } cxims0;
 164 } __packed mock_cedt = {
 165         .cedt = {
 166                 .header = {
 167                         .signature = "CEDT",
 168                         .length = sizeof(mock_cedt),
 169                         .revision = 1,
 170                 },
 171         },
 172         .chbs[0] = {
 173                 .header = {
 174                         .type = ACPI_CEDT_TYPE_CHBS,
 175                         .length = sizeof(mock_cedt.chbs[0]),
 176                 },
 177                 .uid = 0,
 178                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 179         },
 180         .chbs[1] = {
 181                 .header = {
 182                         .type = ACPI_CEDT_TYPE_CHBS,
 183                         .length = sizeof(mock_cedt.chbs[0]),
 184                 },
 185                 .uid = 1,
 186                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 187         },
 188         .chbs[2] = {
 189                 .header = {
 190                         .type = ACPI_CEDT_TYPE_CHBS,
 191                         .length = sizeof(mock_cedt.chbs[0]),
 192                 },
 193                 .uid = 2,
 194                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 195         },
 196         .chbs[3] = {
 197                 .header = {
 198                         .type = ACPI_CEDT_TYPE_CHBS,
 199                         .length = sizeof(mock_cedt.chbs[0]),
 200                 },
 201                 .uid = 3,
 202                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL11,
 203         },
 204         .cfmws0 = {
 205                 .cfmws = {
 206                         .header = {
 207                                 .type = ACPI_CEDT_TYPE_CFMWS,
 208                                 .length = sizeof(mock_cedt.cfmws0),
 209                         },
 210                         .interleave_ways = 0,
 211                         .granularity = 4,
 212                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 213                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 214                         .qtg_id = FAKE_QTG_ID,
 215                         .window_size = SZ_256M * 4UL,
 216                 },
 217                 .target = { 0 },
 218         },
 219         .cfmws1 = {
 220                 .cfmws = {
 221                         .header = {
 222                                 .type = ACPI_CEDT_TYPE_CFMWS,
 223                                 .length = sizeof(mock_cedt.cfmws1),
 224                         },
 225                         .interleave_ways = 1,
 226                         .granularity = 4,
 227                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 228                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 229                         .qtg_id = FAKE_QTG_ID,
 230                         .window_size = SZ_256M * 8UL,
 231                 },
 232                 .target = { 0, 1, },
 233         },
 234         .cfmws2 = {
 235                 .cfmws = {
 236                         .header = {
 237                                 .type = ACPI_CEDT_TYPE_CFMWS,
 238                                 .length = sizeof(mock_cedt.cfmws2),
 239                         },
 240                         .interleave_ways = 0,
 241                         .granularity = 4,
 242                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 243                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 244                         .qtg_id = FAKE_QTG_ID,
 245                         .window_size = SZ_256M * 4UL,
 246                 },
 247                 .target = { 0 },
 248         },
 249         .cfmws3 = {
 250                 .cfmws = {
 251                         .header = {
 252                                 .type = ACPI_CEDT_TYPE_CFMWS,
 253                                 .length = sizeof(mock_cedt.cfmws3),
 254                         },
 255                         .interleave_ways = 1,
 256                         .granularity = 4,
 257                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 258                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 259                         .qtg_id = FAKE_QTG_ID,
 260                         .window_size = SZ_256M * 8UL,
 261                 },
 262                 .target = { 0, 1, },
 263         },
 264         .cfmws4 = {
 265                 .cfmws = {
 266                         .header = {
 267                                 .type = ACPI_CEDT_TYPE_CFMWS,
 268                                 .length = sizeof(mock_cedt.cfmws4),
 269                         },
 270                         .interleave_ways = 0,
 271                         .granularity = 4,
 272                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 273                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 274                         .qtg_id = FAKE_QTG_ID,
 275                         .window_size = SZ_256M * 4UL,
 276                 },
 277                 .target = { 2 },
 278         },
 279         .cfmws5 = {
 280                 .cfmws = {
 281                         .header = {
 282                                 .type = ACPI_CEDT_TYPE_CFMWS,
 283                                 .length = sizeof(mock_cedt.cfmws5),
 284                         },
 285                         .interleave_ways = 0,
 286                         .granularity = 4,
 287                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 288                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 289                         .qtg_id = FAKE_QTG_ID,
 290                         .window_size = SZ_256M,
 291                 },
 292                 .target = { 3 },
 293         },
 294         /* .cfmws6,7,8 use ACPI_CEDT_CFMWS_ARITHMETIC_XOR */
 295         .cfmws6 = {
 296                 .cfmws = {
 297                         .header = {
 298                                 .type = ACPI_CEDT_TYPE_CFMWS,
 299                                 .length = sizeof(mock_cedt.cfmws6),
 300                         },
 301                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 302                         .interleave_ways = 0,
 303                         .granularity = 4,
 304                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 305                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 306                         .qtg_id = FAKE_QTG_ID,
 307                         .window_size = SZ_256M * 8UL,
 308                 },
 309                 .target = { 0, },
 310         },
 311         .cfmws7 = {
 312                 .cfmws = {
 313                         .header = {
 314                                 .type = ACPI_CEDT_TYPE_CFMWS,
 315                                 .length = sizeof(mock_cedt.cfmws7),
 316                         },
 317                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 318                         .interleave_ways = 1,
 319                         .granularity = 0,
 320                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 321                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 322                         .qtg_id = FAKE_QTG_ID,
 323                         .window_size = SZ_256M * 8UL,
 324                 },
 325                 .target = { 0, 1, },
 326         },
 327         .cfmws8 = {
 328                 .cfmws = {
 329                         .header = {
 330                                 .type = ACPI_CEDT_TYPE_CFMWS,
 331                                 .length = sizeof(mock_cedt.cfmws8),
 332                         },
 333                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 334                         .interleave_ways = 2,
 335                         .granularity = 0,
 336                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 337                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 338                         .qtg_id = FAKE_QTG_ID,
 339                         .window_size = SZ_256M * 16UL,
 340                 },
 341                 .target = { 0, 1, 0, 1, },
 342         },
 343         .cxims0 = {
 344                 .cxims = {
 345                         .header = {
 346                                 .type = ACPI_CEDT_TYPE_CXIMS,
 347                                 .length = sizeof(mock_cedt.cxims0),
 348                         },
 349                         .hbig = 0,
 350                         .nr_xormaps = 2,
 351                 },
 352                 .xormap_list = { 0x404100, 0x808200, },
 353         },
 354 };
 355
 356 struct acpi_cedt_cfmws *mock_cfmws[] = {
 357         [0] = &mock_cedt.cfmws0.cfmws,
 358         [1] = &mock_cedt.cfmws1.cfmws,
 359         [2] = &mock_cedt.cfmws2.cfmws,
 360         [3] = &mock_cedt.cfmws3.cfmws,
 361         [4] = &mock_cedt.cfmws4.cfmws,
 362         [5] = &mock_cedt.cfmws5.cfmws,
 363         /* Modulo Math above, XOR Math below */
 364         [6] = &mock_cedt.cfmws6.cfmws,
 365         [7] = &mock_cedt.cfmws7.cfmws,
 366         [8] = &mock_cedt.cfmws8.cfmws,
 367 };
 368
 369 static int cfmws_start;
 370 static int cfmws_end;
 371 #define CFMWS_MOD_ARRAY_START 0
 372 #define CFMWS_MOD_ARRAY_END   5
 373 #define CFMWS_XOR_ARRAY_START 6
 374 #define CFMWS_XOR_ARRAY_END   8
 375
 376 struct acpi_cedt_cxims *mock_cxims[1] = {
 377         [0] = &mock_cedt.cxims0.cxims,
 378 };
 379
 380 struct cxl_mock_res {
 381         struct list_head list;
 382         struct range range;
 383 };
 384
 385 static LIST_HEAD(mock_res);
 386 static DEFINE_MUTEX(mock_res_lock);
 387 static struct gen_pool *cxl_mock_pool;
 388
 389 static void depopulate_all_mock_resources(void)
 390 {
 391         struct cxl_mock_res *res, *_res;
 392
 393         mutex_lock(&mock_res_lock);
 394         list_for_each_entry_safe(res, _res, &mock_res, list) {
 395                 gen_pool_free(cxl_mock_pool, res->range.start,
 396                               range_len(&res->range));
 397                 list_del(&res->list);
 398                 kfree(res);
 399         }
 400         mutex_unlock(&mock_res_lock);
 401 }
 402
 403 static struct cxl_mock_res *alloc_mock_res(resource_size_t size, int align)
 404 {
 405         struct cxl_mock_res *res = kzalloc(sizeof(*res), GFP_KERNEL);
 406         struct genpool_data_align data = {
 407                 .align = align,
 408         };
 409         unsigned long phys;
 410
 411         INIT_LIST_HEAD(&res->list);
 412         phys = gen_pool_alloc_algo(cxl_mock_pool, size,
 413                                    gen_pool_first_fit_align, &data);
 414         if (!phys)
 415                 return NULL;
 416
 417         res->range = (struct range) {
 418                 .start = phys,
 419                 .end = phys + size - 1,
 420         };
 421         mutex_lock(&mock_res_lock);
 422         list_add(&res->list, &mock_res);
 423         mutex_unlock(&mock_res_lock);
 424
 425         return res;
 426 }
 427
 428 static int populate_cedt(void)
 429 {
 430         struct cxl_mock_res *res;
 431         int i;
 432
 433         for (i = 0; i < ARRAY_SIZE(mock_cedt.chbs); i++) {
 434                 struct acpi_cedt_chbs *chbs = &mock_cedt.chbs[i];
 435                 resource_size_t size;
 436
 437                 if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL20)
 438                         size = ACPI_CEDT_CHBS_LENGTH_CXL20;
 439                 else
 440                         size = ACPI_CEDT_CHBS_LENGTH_CXL11;
 441
 442                 res = alloc_mock_res(size, size);
 443                 if (!res)
 444                         return -ENOMEM;
 445                 chbs->base = res->range.start;
 446                 chbs->length = size;
 447         }
 448
 449         for (i = cfmws_start; i <= cfmws_end; i++) {
 450                 struct acpi_cedt_cfmws *window = mock_cfmws[i];
 451
 452                 res = alloc_mock_res(window->window_size, SZ_256M);
 453                 if (!res)
 454                         return -ENOMEM;
 455                 window->base_hpa = res->range.start;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static bool is_mock_port(struct device *dev);
 462
 463 /*
 464  * WARNING, this hack assumes the format of 'struct cxl_cfmws_context'
 465  * and 'struct cxl_chbs_context' share the property that the first
 466  * struct member is a cxl_test device being probed by the cxl_acpi
 467  * driver.
 468  */
 469 struct cxl_cedt_context {
 470         struct device *dev;
 471 };
 472
 473 static int mock_acpi_table_parse_cedt(enum acpi_cedt_type id,
 474                                       acpi_tbl_entry_handler_arg handler_arg,
 475                                       void *arg)
 476 {
 477         struct cxl_cedt_context *ctx = arg;
 478         struct device *dev = ctx->dev;
 479         union acpi_subtable_headers *h;
 480         unsigned long end;
 481         int i;
 482
 483         if (!is_mock_port(dev) && !is_mock_dev(dev))
 484                 return acpi_table_parse_cedt(id, handler_arg, arg);
 485
 486         if (id == ACPI_CEDT_TYPE_CHBS)
 487                 for (i = 0; i < ARRAY_SIZE(mock_cedt.chbs); i++) {
 488                         h = (union acpi_subtable_headers *)&mock_cedt.chbs[i];
 489                         end = (unsigned long)&mock_cedt.chbs[i + 1];
 490                         handler_arg(h, arg, end);
 491                 }
 492
 493         if (id == ACPI_CEDT_TYPE_CFMWS)
 494                 for (i = cfmws_start; i <= cfmws_end; i++) {
 495                         h = (union acpi_subtable_headers *) mock_cfmws[i];
 496                         end = (unsigned long) h + mock_cfmws[i]->header.length;
 497                         handler_arg(h, arg, end);
 498                 }
 499
 500         if (id == ACPI_CEDT_TYPE_CXIMS)
 501                 for (i = 0; i < ARRAY_SIZE(mock_cxims); i++) {
 502                         h = (union acpi_subtable_headers *)mock_cxims[i];
 503                         end = (unsigned long)h + mock_cxims[i]->header.length;
 504                         handler_arg(h, arg, end);
 505                 }
 506
 507         return 0;
 508 }
 509
 510 static bool is_mock_bridge(struct device *dev)
 511 {
 512         int i;
 513
 514         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++)
 515                 if (dev == &cxl_host_bridge[i]->dev)
 516                         return true;
 517         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++)
 518                 if (dev == &cxl_hb_single[i]->dev)
 519                         return true;
 520         for (i = 0; i < ARRAY_SIZE(cxl_rch); i++)
 521                 if (dev == &cxl_rch[i]->dev)
 522                         return true;
 523
 524         return false;
 525 }
 526
 527 static bool is_mock_port(struct device *dev)
 528 {
 529         int i;
 530
 531         if (is_mock_bridge(dev))
 532                 return true;
 533
 534         for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++)
 535                 if (dev == &cxl_root_port[i]->dev)
 536                         return true;
 537
 538         for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++)
 539                 if (dev == &cxl_switch_uport[i]->dev)
 540                         return true;
 541
 542         for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++)
 543                 if (dev == &cxl_switch_dport[i]->dev)
 544                         return true;
 545
 546         for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++)
 547                 if (dev == &cxl_root_single[i]->dev)
 548                         return true;
 549
 550         for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++)
 551                 if (dev == &cxl_swu_single[i]->dev)
 552                         return true;
 553
 554         for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++)
 555                 if (dev == &cxl_swd_single[i]->dev)
 556                         return true;
 557
 558         if (is_cxl_memdev(dev))
 559                 return is_mock_dev(dev->parent);
 560
 561         return false;
 562 }
 563
 564 static int host_bridge_index(struct acpi_device *adev)
 565 {
 566         return adev - host_bridge;
 567 }
 568
 569 static struct acpi_device *find_host_bridge(acpi_handle handle)
 570 {
 571         int i;
 572
 573         for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
 574                 if (handle == host_bridge[i].handle)
 575                         return &host_bridge[i];
 576         return NULL;
 577 }
 578
 579 static acpi_status
 580 mock_acpi_evaluate_integer(acpi_handle handle, acpi_string pathname,
 581                            struct acpi_object_list *arguments,
 582                            unsigned long long *data)
 583 {
 584         struct acpi_device *adev = find_host_bridge(handle);
 585
 586         if (!adev || strcmp(pathname, METHOD_NAME__UID) != 0)
 587                 return acpi_evaluate_integer(handle, pathname, arguments, data);
 588
 589         *data = host_bridge_index(adev);
 590         return AE_OK;
 591 }
 592
 593 static struct pci_bus mock_pci_bus[NR_BRIDGES];
 594 static struct acpi_pci_root mock_pci_root[ARRAY_SIZE(mock_pci_bus)] = {
 595         [0] = {
 596                 .bus = &mock_pci_bus[0],
 597         },
 598         [1] = {
 599                 .bus = &mock_pci_bus[1],
 600         },
 601         [2] = {
 602                 .bus = &mock_pci_bus[2],
 603         },
 604         [3] = {
 605                 .bus = &mock_pci_bus[3],
 606         },
 607
 608 };
 609
 610 static bool is_mock_bus(struct pci_bus *bus)
 611 {
 612         int i;
 613
 614         for (i = 0; i < ARRAY_SIZE(mock_pci_bus); i++)
 615                 if (bus == &mock_pci_bus[i])
 616                         return true;
 617         return false;
 618 }
 619
 620 static struct acpi_pci_root *mock_acpi_pci_find_root(acpi_handle handle)
 621 {
 622         struct acpi_device *adev = find_host_bridge(handle);
 623
 624         if (!adev)
 625                 return acpi_pci_find_root(handle);
 626         return &mock_pci_root[host_bridge_index(adev)];
 627 }
 628
 629 static struct cxl_hdm *mock_cxl_setup_hdm(struct cxl_port *port,
 630                                           struct cxl_endpoint_dvsec_info *info)
 631 {
 632         struct cxl_hdm *cxlhdm = devm_kzalloc(&port->dev, sizeof(*cxlhdm), GFP_KERNEL);
 633
 634         if (!cxlhdm)
 635                 return ERR_PTR(-ENOMEM);
 636
 637         cxlhdm->port = port;
 638         return cxlhdm;
 639 }
 640
 641 static int mock_cxl_add_passthrough_decoder(struct cxl_port *port)
 642 {
 643         dev_err(&port->dev, "unexpected passthrough decoder for cxl_test\n");
 644         return -EOPNOTSUPP;
 645 }
 646
 647
 648 struct target_map_ctx {
 649         int *target_map;
 650         int index;
 651         int target_count;
 652 };
 653
 654 static int map_targets(struct device *dev, void *data)
 655 {
 656         struct platform_device *pdev = to_platform_device(dev);
 657         struct target_map_ctx *ctx = data;
 658
 659         ctx->target_map[ctx->index++] = pdev->id;
 660
 661         if (ctx->index > ctx->target_count) {
 662                 dev_WARN_ONCE(dev, 1, "too many targets found?\n");
 663                 return -ENXIO;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int mock_decoder_commit(struct cxl_decoder *cxld)
 670 {
 671         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 672         int id = cxld->id;
 673
 674         if (cxld->flags & CXL_DECODER_F_ENABLE)
 675                 return 0;
 676
 677         dev_dbg(&port->dev, "%s commit\n", dev_name(&cxld->dev));
 678         if (cxl_num_decoders_committed(port) != id) {
 679                 dev_dbg(&port->dev,
 680                         "%s: out of order commit, expected decoder%d.%d\n",
 681                         dev_name(&cxld->dev), port->id,
 682                         cxl_num_decoders_committed(port));
 683                 return -EBUSY;
 684         }
 685
 686         port->commit_end++;
 687         cxld->flags |= CXL_DECODER_F_ENABLE;
 688
 689         return 0;
 690 }
 691
 692 static int mock_decoder_reset(struct cxl_decoder *cxld)
 693 {
 694         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 695         int id = cxld->id;
 696
 697         if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
 698                 return 0;
 699
 700         dev_dbg(&port->dev, "%s reset\n", dev_name(&cxld->dev));
 701         if (port->commit_end != id) {
 702                 dev_dbg(&port->dev,
 703                         "%s: out of order reset, expected decoder%d.%d\n",
 704                         dev_name(&cxld->dev), port->id, port->commit_end);
 705                 return -EBUSY;
 706         }
 707
 708         port->commit_end--;
 709         cxld->flags &= ~CXL_DECODER_F_ENABLE;
 710
 711         return 0;
 712 }
 713
 714 static void default_mock_decoder(struct cxl_decoder *cxld)
 715 {
 716         cxld->hpa_range = (struct range){
 717                 .start = 0,
 718                 .end = -1,
 719         };
 720
 721         cxld->interleave_ways = 1;
 722         cxld->interleave_granularity = 256;
 723         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 724         cxld->commit = mock_decoder_commit;
 725         cxld->reset = mock_decoder_reset;
 726 }
 727
 728 static int first_decoder(struct device *dev, void *data)
 729 {
 730         struct cxl_decoder *cxld;
 731
 732         if (!is_switch_decoder(dev))
 733                 return 0;
 734         cxld = to_cxl_decoder(dev);
 735         if (cxld->id == 0)
 736                 return 1;
 737         return 0;
 738 }
 739
 740 static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
 741 {
 742         struct acpi_cedt_cfmws *window = mock_cfmws[0];
 743         struct platform_device *pdev = NULL;
 744         struct cxl_endpoint_decoder *cxled;
 745         struct cxl_switch_decoder *cxlsd;
 746         struct cxl_port *port, *iter;
 747         const int size = SZ_512M;
 748         struct cxl_memdev *cxlmd;
 749         struct cxl_dport *dport;
 750         struct device *dev;
 751         bool hb0 = false;
 752         u64 base;
 753         int i;
 754
 755         if (is_endpoint_decoder(&cxld->dev)) {
 756                 cxled = to_cxl_endpoint_decoder(&cxld->dev);
 757                 cxlmd = cxled_to_memdev(cxled);
 758                 WARN_ON(!dev_is_platform(cxlmd->dev.parent));
 759                 pdev = to_platform_device(cxlmd->dev.parent);
 760
 761                 /* check is endpoint is attach to host-bridge0 */
 762                 port = cxled_to_port(cxled);
 763                 do {
 764                         if (port->uport_dev == &cxl_host_bridge[0]->dev) {
 765                                 hb0 = true;
 766                                 break;
 767                         }
 768                         if (is_cxl_port(port->dev.parent))
 769                                 port = to_cxl_port(port->dev.parent);
 770                         else
 771                                 port = NULL;
 772                 } while (port);
 773                 port = cxled_to_port(cxled);
 774         }
 775
 776         /*
 777          * The first decoder on the first 2 devices on the first switch
 778          * attached to host-bridge0 mock a fake / static RAM region. All
 779          * other decoders are default disabled. Given the round robin
 780          * assignment those devices are named cxl_mem.0, and cxl_mem.4.
 781          *
 782          * See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4'
 783          */
 784         if (!hb0 || pdev->id % 4 || pdev->id > 4 || cxld->id > 0) {
 785                 default_mock_decoder(cxld);
 786                 return;
 787         }
 788
 789         base = window->base_hpa;
 790         cxld->hpa_range = (struct range) {
 791                 .start = base,
 792                 .end = base + size - 1,
 793         };
 794
 795         cxld->interleave_ways = 2;
 796         eig_to_granularity(window->granularity, &cxld->interleave_granularity);
 797         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 798         cxld->flags = CXL_DECODER_F_ENABLE;
 799         cxled->state = CXL_DECODER_STATE_AUTO;
 800         port->commit_end = cxld->id;
 801         devm_cxl_dpa_reserve(cxled, 0, size / cxld->interleave_ways, 0);
 802         cxld->commit = mock_decoder_commit;
 803         cxld->reset = mock_decoder_reset;
 804
 805         /*
 806          * Now that endpoint decoder is set up, walk up the hierarchy
 807          * and setup the switch and root port decoders targeting @cxlmd.
 808          */
 809         iter = port;
 810         for (i = 0; i < 2; i++) {
 811                 dport = iter->parent_dport;
 812                 iter = dport->port;
 813                 dev = device_find_child(&iter->dev, NULL, first_decoder);
 814                 /*
 815                  * Ancestor ports are guaranteed to be enumerated before
 816                  * @port, and all ports have at least one decoder.
 817                  */
 818                 if (WARN_ON(!dev))
 819                         continue;
 820                 cxlsd = to_cxl_switch_decoder(dev);
 821                 if (i == 0) {
 822                         /* put cxl_mem.4 second in the decode order */
 823                         if (pdev->id == 4)
 824                                 cxlsd->target[1] = dport;
 825                         else
 826                                 cxlsd->target[0] = dport;
 827                 } else
 828                         cxlsd->target[0] = dport;
 829                 cxld = &cxlsd->cxld;
 830                 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 831                 cxld->flags = CXL_DECODER_F_ENABLE;
 832                 iter->commit_end = 0;
 833                 /*
 834                  * Switch targets 2 endpoints, while host bridge targets
 835                  * one root port
 836                  */
 837                 if (i == 0)
 838                         cxld->interleave_ways = 2;
 839                 else
 840                         cxld->interleave_ways = 1;
 841                 cxld->interleave_granularity = 4096;
 842                 cxld->hpa_range = (struct range) {
 843                         .start = base,
 844                         .end = base + size - 1,
 845                 };
 846                 put_device(dev);
 847         }
 848 }
 849
 850 static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
 851                                        struct cxl_endpoint_dvsec_info *info)
 852 {
 853         struct cxl_port *port = cxlhdm->port;
 854         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
 855         int target_count, i;
 856
 857         if (is_cxl_endpoint(port))
 858                 target_count = 0;
 859         else if (is_cxl_root(parent_port))
 860                 target_count = NR_CXL_ROOT_PORTS;
 861         else
 862                 target_count = NR_CXL_SWITCH_PORTS;
 863
 864         for (i = 0; i < NR_CXL_PORT_DECODERS; i++) {
 865                 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
 866                 struct target_map_ctx ctx = {
 867                         .target_map = target_map,
 868                         .target_count = target_count,
 869                 };
 870                 struct cxl_decoder *cxld;
 871                 int rc;
 872
 873                 if (target_count) {
 874                         struct cxl_switch_decoder *cxlsd;
 875
 876                         cxlsd = cxl_switch_decoder_alloc(port, target_count);
 877                         if (IS_ERR(cxlsd)) {
 878                                 dev_warn(&port->dev,
 879                                          "Failed to allocate the decoder\n");
 880                                 return PTR_ERR(cxlsd);
 881                         }
 882                         cxld = &cxlsd->cxld;
 883                 } else {
 884                         struct cxl_endpoint_decoder *cxled;
 885
 886                         cxled = cxl_endpoint_decoder_alloc(port);
 887
 888                         if (IS_ERR(cxled)) {
 889                                 dev_warn(&port->dev,
 890                                          "Failed to allocate the decoder\n");
 891                                 return PTR_ERR(cxled);
 892                         }
 893                         cxld = &cxled->cxld;
 894                 }
 895
 896                 mock_init_hdm_decoder(cxld);
 897
 898                 if (target_count) {
 899                         rc = device_for_each_child(port->uport_dev, &ctx,
 900                                                    map_targets);
 901                         if (rc) {
 902                                 put_device(&cxld->dev);
 903                                 return rc;
 904                         }
 905                 }
 906
 907                 rc = cxl_decoder_add_locked(cxld, target_map);
 908                 if (rc) {
 909                         put_device(&cxld->dev);
 910                         dev_err(&port->dev, "Failed to add decoder\n");
 911                         return rc;
 912                 }
 913
 914                 rc = cxl_decoder_autoremove(&port->dev, cxld);
 915                 if (rc)
 916                         return rc;
 917                 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
 918         }
 919
 920         return 0;
 921 }
 922
 923 static int mock_cxl_port_enumerate_dports(struct cxl_port *port)
 924 {
 925         struct platform_device **array;
 926         int i, array_size;
 927
 928         if (port->depth == 1) {
 929                 if (is_multi_bridge(port->uport_dev)) {
 930                         array_size = ARRAY_SIZE(cxl_root_port);
 931                         array = cxl_root_port;
 932                 } else if (is_single_bridge(port->uport_dev)) {
 933                         array_size = ARRAY_SIZE(cxl_root_single);
 934                         array = cxl_root_single;
 935                 } else {
 936                         dev_dbg(&port->dev, "%s: unknown bridge type\n",
 937                                 dev_name(port->uport_dev));
 938                         return -ENXIO;
 939                 }
 940         } else if (port->depth == 2) {
 941                 struct cxl_port *parent = to_cxl_port(port->dev.parent);
 942
 943                 if (is_multi_bridge(parent->uport_dev)) {
 944                         array_size = ARRAY_SIZE(cxl_switch_dport);
 945                         array = cxl_switch_dport;
 946                 } else if (is_single_bridge(parent->uport_dev)) {
 947                         array_size = ARRAY_SIZE(cxl_swd_single);
 948                         array = cxl_swd_single;
 949                 } else {
 950                         dev_dbg(&port->dev, "%s: unknown bridge type\n",
 951                                 dev_name(port->uport_dev));
 952                         return -ENXIO;
 953                 }
 954         } else {
 955                 dev_WARN_ONCE(&port->dev, 1, "unexpected depth %d\n",
 956                               port->depth);
 957                 return -ENXIO;
 958         }
 959
 960         for (i = 0; i < array_size; i++) {
 961                 struct platform_device *pdev = array[i];
 962                 struct cxl_dport *dport;
 963
 964                 if (pdev->dev.parent != port->uport_dev) {
 965                         dev_dbg(&port->dev, "%s: mismatch parent %s\n",
 966                                 dev_name(port->uport_dev),
 967                                 dev_name(pdev->dev.parent));
 968                         continue;
 969                 }
 970
 971                 dport = devm_cxl_add_dport(port, &pdev->dev, pdev->id,
 972                                            CXL_RESOURCE_NONE);
 973
 974                 if (IS_ERR(dport))
 975                         return PTR_ERR(dport);
 976         }
 977
 978         return 0;
 979 }
 980
 981 /*
 982  * Faking the cxl_dpa_perf for the memdev when appropriate.
 983  */
 984 static void dpa_perf_setup(struct cxl_port *endpoint, struct range *range,
 985                            struct cxl_dpa_perf *dpa_perf)
 986 {
 987         dpa_perf->qos_class = FAKE_QTG_ID;
 988         dpa_perf->dpa_range = *range;
 989         for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
 990                 dpa_perf->coord[i].read_latency = 500;
 991                 dpa_perf->coord[i].write_latency = 500;
 992                 dpa_perf->coord[i].read_bandwidth = 1000;
 993                 dpa_perf->coord[i].write_bandwidth = 1000;
 994         }
 995 }
 996
 997 static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port)
 998 {
 999         struct cxl_root *cxl_root __free(put_cxl_root) =
1000                 find_cxl_root(port);
1001         struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
1002         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1003         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1004         struct access_coordinate ep_c[ACCESS_COORDINATE_MAX];
1005         struct range pmem_range = {
1006                 .start = cxlds->pmem_res.start,
1007                 .end = cxlds->pmem_res.end,
1008         };
1009         struct range ram_range = {
1010                 .start = cxlds->ram_res.start,
1011                 .end = cxlds->ram_res.end,
1012         };
1013
1014         if (!cxl_root)
1015                 return;
1016
1017         if (range_len(&ram_range))
1018                 dpa_perf_setup(port, &ram_range, &mds->ram_perf);
1019
1020         if (range_len(&pmem_range))
1021                 dpa_perf_setup(port, &pmem_range, &mds->pmem_perf);
1022
1023         cxl_memdev_update_perf(cxlmd);
1024
1025         /*
1026          * This function is here to only test the topology iterator. It serves
1027          * no other purpose.
1028          */
1029         cxl_endpoint_get_perf_coordinates(port, ep_c);
1030 }
1031
1032 static struct cxl_mock_ops cxl_mock_ops = {
1033         .is_mock_adev = is_mock_adev,
1034         .is_mock_bridge = is_mock_bridge,
1035         .is_mock_bus = is_mock_bus,
1036         .is_mock_port = is_mock_port,
1037         .is_mock_dev = is_mock_dev,
1038         .acpi_table_parse_cedt = mock_acpi_table_parse_cedt,
1039         .acpi_evaluate_integer = mock_acpi_evaluate_integer,
1040         .acpi_pci_find_root = mock_acpi_pci_find_root,
1041         .devm_cxl_port_enumerate_dports = mock_cxl_port_enumerate_dports,
1042         .devm_cxl_setup_hdm = mock_cxl_setup_hdm,
1043         .devm_cxl_add_passthrough_decoder = mock_cxl_add_passthrough_decoder,
1044         .devm_cxl_enumerate_decoders = mock_cxl_enumerate_decoders,
1045         .cxl_endpoint_parse_cdat = mock_cxl_endpoint_parse_cdat,
1046         .list = LIST_HEAD_INIT(cxl_mock_ops.list),
1047 };
1048
1049 static void mock_companion(struct acpi_device *adev, struct device *dev)
1050 {
1051         device_initialize(&adev->dev);
1052         fwnode_init(&adev->fwnode, NULL);
1053         dev->fwnode = &adev->fwnode;
1054         adev->fwnode.dev = dev;
1055 }
1056
1057 #ifndef SZ_64G
1058 #define SZ_64G (SZ_32G * 2)
1059 #endif
1060
1061 static __init int cxl_rch_init(void)
1062 {
1063         int rc, i;
1064
1065         for (i = 0; i < ARRAY_SIZE(cxl_rch); i++) {
1066                 int idx = NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + i;
1067                 struct acpi_device *adev = &host_bridge[idx];
1068                 struct platform_device *pdev;
1069
1070                 pdev = platform_device_alloc("cxl_host_bridge", idx);
1071                 if (!pdev)
1072                         goto err_bridge;
1073
1074                 mock_companion(adev, &pdev->dev);
1075                 rc = platform_device_add(pdev);
1076                 if (rc) {
1077                         platform_device_put(pdev);
1078                         goto err_bridge;
1079                 }
1080
1081                 cxl_rch[i] = pdev;
1082                 mock_pci_bus[idx].bridge = &pdev->dev;
1083                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1084                                        "firmware_node");
1085                 if (rc)
1086                         goto err_bridge;
1087         }
1088
1089         for (i = 0; i < ARRAY_SIZE(cxl_rcd); i++) {
1090                 int idx = NR_MEM_MULTI + NR_MEM_SINGLE + i;
1091                 struct platform_device *rch = cxl_rch[i];
1092                 struct platform_device *pdev;
1093
1094                 pdev = platform_device_alloc("cxl_rcd", idx);
1095                 if (!pdev)
1096                         goto err_mem;
1097                 pdev->dev.parent = &rch->dev;
1098                 set_dev_node(&pdev->dev, i % 2);
1099
1100                 rc = platform_device_add(pdev);
1101                 if (rc) {
1102                         platform_device_put(pdev);
1103                         goto err_mem;
1104                 }
1105                 cxl_rcd[i] = pdev;
1106         }
1107
1108         return 0;
1109
1110 err_mem:
1111         for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--)
1112                 platform_device_unregister(cxl_rcd[i]);
1113 err_bridge:
1114         for (i = ARRAY_SIZE(cxl_rch) - 1; i >= 0; i--) {
1115                 struct platform_device *pdev = cxl_rch[i];
1116
1117                 if (!pdev)
1118                         continue;
1119                 sysfs_remove_link(&pdev->dev.kobj, "firmware_node");
1120                 platform_device_unregister(cxl_rch[i]);
1121         }
1122
1123         return rc;
1124 }
1125
1126 static void cxl_rch_exit(void)
1127 {
1128         int i;
1129
1130         for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--)
1131                 platform_device_unregister(cxl_rcd[i]);
1132         for (i = ARRAY_SIZE(cxl_rch) - 1; i >= 0; i--) {
1133                 struct platform_device *pdev = cxl_rch[i];
1134
1135                 if (!pdev)
1136                         continue;
1137                 sysfs_remove_link(&pdev->dev.kobj, "firmware_node");
1138                 platform_device_unregister(cxl_rch[i]);
1139         }
1140 }
1141
1142 static __init int cxl_single_init(void)
1143 {
1144         int i, rc;
1145
1146         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++) {
1147                 struct acpi_device *adev =
1148                         &host_bridge[NR_CXL_HOST_BRIDGES + i];
1149                 struct platform_device *pdev;
1150
1151                 pdev = platform_device_alloc("cxl_host_bridge",
1152                                              NR_CXL_HOST_BRIDGES + i);
1153                 if (!pdev)
1154                         goto err_bridge;
1155
1156                 mock_companion(adev, &pdev->dev);
1157                 rc = platform_device_add(pdev);
1158                 if (rc) {
1159                         platform_device_put(pdev);
1160                         goto err_bridge;
1161                 }
1162
1163                 cxl_hb_single[i] = pdev;
1164                 mock_pci_bus[i + NR_CXL_HOST_BRIDGES].bridge = &pdev->dev;
1165                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1166                                        "physical_node");
1167                 if (rc)
1168                         goto err_bridge;
1169         }
1170
1171         for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++) {
1172                 struct platform_device *bridge =
1173                         cxl_hb_single[i % ARRAY_SIZE(cxl_hb_single)];
1174                 struct platform_device *pdev;
1175
1176                 pdev = platform_device_alloc("cxl_root_port",
1177                                              NR_MULTI_ROOT + i);
1178                 if (!pdev)
1179                         goto err_port;
1180                 pdev->dev.parent = &bridge->dev;
1181
1182                 rc = platform_device_add(pdev);
1183                 if (rc) {
1184                         platform_device_put(pdev);
1185                         goto err_port;
1186                 }
1187                 cxl_root_single[i] = pdev;
1188         }
1189
1190         for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++) {
1191                 struct platform_device *root_port = cxl_root_single[i];
1192                 struct platform_device *pdev;
1193
1194                 pdev = platform_device_alloc("cxl_switch_uport",
1195                                              NR_MULTI_ROOT + i);
1196                 if (!pdev)
1197                         goto err_uport;
1198                 pdev->dev.parent = &root_port->dev;
1199
1200                 rc = platform_device_add(pdev);
1201                 if (rc) {
1202                         platform_device_put(pdev);
1203                         goto err_uport;
1204                 }
1205                 cxl_swu_single[i] = pdev;
1206         }
1207
1208         for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++) {
1209                 struct platform_device *uport =
1210                         cxl_swu_single[i % ARRAY_SIZE(cxl_swu_single)];
1211                 struct platform_device *pdev;
1212
1213                 pdev = platform_device_alloc("cxl_switch_dport",
1214                                              i + NR_MEM_MULTI);
1215                 if (!pdev)
1216                         goto err_dport;
1217                 pdev->dev.parent = &uport->dev;
1218
1219                 rc = platform_device_add(pdev);
1220                 if (rc) {
1221                         platform_device_put(pdev);
1222                         goto err_dport;
1223                 }
1224                 cxl_swd_single[i] = pdev;
1225         }
1226
1227         for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++) {
1228                 struct platform_device *dport = cxl_swd_single[i];
1229                 struct platform_device *pdev;
1230
1231                 pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i);
1232                 if (!pdev)
1233                         goto err_mem;
1234                 pdev->dev.parent = &dport->dev;
1235                 set_dev_node(&pdev->dev, i % 2);
1236
1237                 rc = platform_device_add(pdev);
1238                 if (rc) {
1239                         platform_device_put(pdev);
1240                         goto err_mem;
1241                 }
1242                 cxl_mem_single[i] = pdev;
1243         }
1244
1245         return 0;
1246
1247 err_mem:
1248         for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--)
1249                 platform_device_unregister(cxl_mem_single[i]);
1250 err_dport:
1251         for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--)
1252                 platform_device_unregister(cxl_swd_single[i]);
1253 err_uport:
1254         for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--)
1255                 platform_device_unregister(cxl_swu_single[i]);
1256 err_port:
1257         for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--)
1258                 platform_device_unregister(cxl_root_single[i]);
1259 err_bridge:
1260         for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) {
1261                 struct platform_device *pdev = cxl_hb_single[i];
1262
1263                 if (!pdev)
1264                         continue;
1265                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1266                 platform_device_unregister(cxl_hb_single[i]);
1267         }
1268
1269         return rc;
1270 }
1271
1272 static void cxl_single_exit(void)
1273 {
1274         int i;
1275
1276         for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--)
1277                 platform_device_unregister(cxl_mem_single[i]);
1278         for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--)
1279                 platform_device_unregister(cxl_swd_single[i]);
1280         for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--)
1281                 platform_device_unregister(cxl_swu_single[i]);
1282         for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--)
1283                 platform_device_unregister(cxl_root_single[i]);
1284         for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) {
1285                 struct platform_device *pdev = cxl_hb_single[i];
1286
1287                 if (!pdev)
1288                         continue;
1289                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1290                 platform_device_unregister(cxl_hb_single[i]);
1291         }
1292 }
1293
1294 static __init int cxl_test_init(void)
1295 {
1296         int rc, i;
1297
1298         cxl_acpi_test();
1299         cxl_core_test();
1300         cxl_mem_test();
1301         cxl_pmem_test();
1302         cxl_port_test();
1303
1304         register_cxl_mock_ops(&cxl_mock_ops);
1305
1306         cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE);
1307         if (!cxl_mock_pool) {
1308                 rc = -ENOMEM;
1309                 goto err_gen_pool_create;
1310         }
1311
1312         rc = gen_pool_add(cxl_mock_pool, iomem_resource.end + 1 - SZ_64G,
1313                           SZ_64G, NUMA_NO_NODE);
1314         if (rc)
1315                 goto err_gen_pool_add;
1316
1317         if (interleave_arithmetic == 1) {
1318                 cfmws_start = CFMWS_XOR_ARRAY_START;
1319                 cfmws_end = CFMWS_XOR_ARRAY_END;
1320         } else {
1321                 cfmws_start = CFMWS_MOD_ARRAY_START;
1322                 cfmws_end = CFMWS_MOD_ARRAY_END;
1323         }
1324
1325         rc = populate_cedt();
1326         if (rc)
1327                 goto err_populate;
1328
1329         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) {
1330                 struct acpi_device *adev = &host_bridge[i];
1331                 struct platform_device *pdev;
1332
1333                 pdev = platform_device_alloc("cxl_host_bridge", i);
1334                 if (!pdev)
1335                         goto err_bridge;
1336
1337                 mock_companion(adev, &pdev->dev);
1338                 rc = platform_device_add(pdev);
1339                 if (rc) {
1340                         platform_device_put(pdev);
1341                         goto err_bridge;
1342                 }
1343
1344                 cxl_host_bridge[i] = pdev;
1345                 mock_pci_bus[i].bridge = &pdev->dev;
1346                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1347                                        "physical_node");
1348                 if (rc)
1349                         goto err_bridge;
1350         }
1351
1352         for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) {
1353                 struct platform_device *bridge =
1354                         cxl_host_bridge[i % ARRAY_SIZE(cxl_host_bridge)];
1355                 struct platform_device *pdev;
1356
1357                 pdev = platform_device_alloc("cxl_root_port", i);
1358                 if (!pdev)
1359                         goto err_port;
1360                 pdev->dev.parent = &bridge->dev;
1361
1362                 rc = platform_device_add(pdev);
1363                 if (rc) {
1364                         platform_device_put(pdev);
1365                         goto err_port;
1366                 }
1367                 cxl_root_port[i] = pdev;
1368         }
1369
1370         BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port));
1371         for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) {
1372                 struct platform_device *root_port = cxl_root_port[i];
1373                 struct platform_device *pdev;
1374
1375                 pdev = platform_device_alloc("cxl_switch_uport", i);
1376                 if (!pdev)
1377                         goto err_uport;
1378                 pdev->dev.parent = &root_port->dev;
1379
1380                 rc = platform_device_add(pdev);
1381                 if (rc) {
1382                         platform_device_put(pdev);
1383                         goto err_uport;
1384                 }
1385                 cxl_switch_uport[i] = pdev;
1386         }
1387
1388         for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) {
1389                 struct platform_device *uport =
1390                         cxl_switch_uport[i % ARRAY_SIZE(cxl_switch_uport)];
1391                 struct platform_device *pdev;
1392
1393                 pdev = platform_device_alloc("cxl_switch_dport", i);
1394                 if (!pdev)
1395                         goto err_dport;
1396                 pdev->dev.parent = &uport->dev;
1397
1398                 rc = platform_device_add(pdev);
1399                 if (rc) {
1400                         platform_device_put(pdev);
1401                         goto err_dport;
1402                 }
1403                 cxl_switch_dport[i] = pdev;
1404         }
1405
1406         for (i = 0; i < ARRAY_SIZE(cxl_mem); i++) {
1407                 struct platform_device *dport = cxl_switch_dport[i];
1408                 struct platform_device *pdev;
1409
1410                 pdev = platform_device_alloc("cxl_mem", i);
1411                 if (!pdev)
1412                         goto err_mem;
1413                 pdev->dev.parent = &dport->dev;
1414                 set_dev_node(&pdev->dev, i % 2);
1415
1416                 rc = platform_device_add(pdev);
1417                 if (rc) {
1418                         platform_device_put(pdev);
1419                         goto err_mem;
1420                 }
1421                 cxl_mem[i] = pdev;
1422         }
1423
1424         rc = cxl_single_init();
1425         if (rc)
1426                 goto err_mem;
1427
1428         rc = cxl_rch_init();
1429         if (rc)
1430                 goto err_single;
1431
1432         cxl_acpi = platform_device_alloc("cxl_acpi", 0);
1433         if (!cxl_acpi)
1434                 goto err_rch;
1435
1436         mock_companion(&acpi0017_mock, &cxl_acpi->dev);
1437         acpi0017_mock.dev.bus = &platform_bus_type;
1438
1439         rc = platform_device_add(cxl_acpi);
1440         if (rc)
1441                 goto err_add;
1442
1443         return 0;
1444
1445 err_add:
1446         platform_device_put(cxl_acpi);
1447 err_rch:
1448         cxl_rch_exit();
1449 err_single:
1450         cxl_single_exit();
1451 err_mem:
1452         for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
1453                 platform_device_unregister(cxl_mem[i]);
1454 err_dport:
1455         for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
1456                 platform_device_unregister(cxl_switch_dport[i]);
1457 err_uport:
1458         for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
1459                 platform_device_unregister(cxl_switch_uport[i]);
1460 err_port:
1461         for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
1462                 platform_device_unregister(cxl_root_port[i]);
1463 err_bridge:
1464         for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
1465                 struct platform_device *pdev = cxl_host_bridge[i];
1466
1467                 if (!pdev)
1468                         continue;
1469                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1470                 platform_device_unregister(cxl_host_bridge[i]);
1471         }
1472 err_populate:
1473         depopulate_all_mock_resources();
1474 err_gen_pool_add:
1475         gen_pool_destroy(cxl_mock_pool);
1476 err_gen_pool_create:
1477         unregister_cxl_mock_ops(&cxl_mock_ops);
1478         return rc;
1479 }
1480
1481 static __exit void cxl_test_exit(void)
1482 {
1483         int i;
1484
1485         platform_device_unregister(cxl_acpi);
1486         cxl_rch_exit();
1487         cxl_single_exit();
1488         for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
1489                 platform_device_unregister(cxl_mem[i]);
1490         for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
1491                 platform_device_unregister(cxl_switch_dport[i]);
1492         for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
1493                 platform_device_unregister(cxl_switch_uport[i]);
1494         for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
1495                 platform_device_unregister(cxl_root_port[i]);
1496         for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
1497                 struct platform_device *pdev = cxl_host_bridge[i];
1498
1499                 if (!pdev)
1500                         continue;
1501                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1502                 platform_device_unregister(cxl_host_bridge[i]);
1503         }
1504         depopulate_all_mock_resources();
1505         gen_pool_destroy(cxl_mock_pool);
1506         unregister_cxl_mock_ops(&cxl_mock_ops);
1507 }
1508
1509 module_param(interleave_arithmetic, int, 0444);
1510 MODULE_PARM_DESC(interleave_arithmetic, "Modulo:0, XOR:1");
1511 module_init(cxl_test_init);
1512 module_exit(cxl_test_exit);
1513 MODULE_LICENSE("GPL v2");
1514 MODULE_IMPORT_NS(ACPI);
1515 MODULE_IMPORT_NS(CXL);