arch/x86/kernel/cpu/topology_common.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/cpu.h>
   3
   4 #include <xen/xen.h>
   5
   6 #include <asm/apic.h>
   7 #include <asm/processor.h>
   8 #include <asm/smp.h>
   9
  10 #include "cpu.h"
  11
  12 struct x86_topology_system x86_topo_system __ro_after_init;
  13
  14 unsigned int __amd_nodes_per_pkg __ro_after_init;
  15 EXPORT_SYMBOL_GPL(__amd_nodes_per_pkg);
  16
  17 void topology_set_dom(struct topo_scan *tscan, enum x86_topology_domains dom,
  18                       unsigned int shift, unsigned int ncpus)
  19 {
  20         topology_update_dom(tscan, dom, shift, ncpus);
  21
  22         /* Propagate to the upper levels */
  23         for (dom++; dom < TOPO_MAX_DOMAIN; dom++) {
  24                 tscan->dom_shifts[dom] = tscan->dom_shifts[dom - 1];
  25                 tscan->dom_ncpus[dom] = tscan->dom_ncpus[dom - 1];
  26         }
  27 }
  28
  29 static unsigned int __maybe_unused parse_num_cores_legacy(struct cpuinfo_x86 *c)
  30 {
  31         struct {
  32                 u32     cache_type      :  5,
  33                         unused          : 21,
  34                         ncores          :  6;
  35         } eax;
  36
  37         if (c->cpuid_level < 4)
  38                 return 1;
  39
  40         cpuid_subleaf_reg(4, 0, CPUID_EAX, &eax);
  41         if (!eax.cache_type)
  42                 return 1;
  43
  44         return eax.ncores + 1;
  45 }
  46
  47 static void parse_legacy(struct topo_scan *tscan)
  48 {
  49         unsigned int cores, core_shift, smt_shift = 0;
  50         struct cpuinfo_x86 *c = tscan->c;
  51
  52         cores = parse_num_cores_legacy(c);
  53         core_shift = get_count_order(cores);
  54
  55         if (cpu_has(c, X86_FEATURE_HT)) {
  56                 if (!WARN_ON_ONCE(tscan->ebx1_nproc_shift < core_shift))
  57                         smt_shift = tscan->ebx1_nproc_shift - core_shift;
  58                 /*
  59                  * The parser expects leaf 0xb/0x1f format, which means
  60                  * the number of logical processors at core level is
  61                  * counting threads.
  62                  */
  63                 core_shift += smt_shift;
  64                 cores <<= smt_shift;
  65         }
  66
  67         topology_set_dom(tscan, TOPO_SMT_DOMAIN, smt_shift, 1U << smt_shift);
  68         topology_set_dom(tscan, TOPO_CORE_DOMAIN, core_shift, cores);
  69 }
  70
  71 static bool fake_topology(struct topo_scan *tscan)
  72 {
  73         /*
  74          * Preset the CORE level shift for CPUID less systems and XEN_PV,
  75          * which has useless CPUID information.
  76          */
  77         topology_set_dom(tscan, TOPO_SMT_DOMAIN, 0, 1);
  78         topology_set_dom(tscan, TOPO_CORE_DOMAIN, 0, 1);
  79
  80         return tscan->c->cpuid_level < 1 || xen_pv_domain();
  81 }
  82
  83 static void parse_topology(struct topo_scan *tscan, bool early)
  84 {
  85         const struct cpuinfo_topology topo_defaults = {
  86                 .cu_id                  = 0xff,
  87                 .llc_id                 = BAD_APICID,
  88                 .l2c_id                 = BAD_APICID,
  89         };
  90         struct cpuinfo_x86 *c = tscan->c;
  91         struct {
  92                 u32     unused0         : 16,
  93                         nproc           :  8,
  94                         apicid          :  8;
  95         } ebx;
  96
  97         c->topo = topo_defaults;
  98
  99         if (fake_topology(tscan))
 100                 return;
 101
 102         /* Preset Initial APIC ID from CPUID leaf 1 */
 103         cpuid_leaf_reg(1, CPUID_EBX, &ebx);
 104         c->topo.initial_apicid = ebx.apicid;
 105
 106         /*
 107          * The initial invocation from early_identify_cpu() happens before
 108          * the APIC is mapped or X2APIC enabled. For establishing the
 109          * topology, that's not required. Use the initial APIC ID.
 110          */
 111         if (early)
 112                 c->topo.apicid = c->topo.initial_apicid;
 113         else
 114                 c->topo.apicid = read_apic_id();
 115
 116         /* The above is sufficient for UP */
 117         if (!IS_ENABLED(CONFIG_SMP))
 118                 return;
 119
 120         tscan->ebx1_nproc_shift = get_count_order(ebx.nproc);
 121
 122         switch (c->x86_vendor) {
 123         case X86_VENDOR_AMD:
 124                 if (IS_ENABLED(CONFIG_CPU_SUP_AMD))
 125                         cpu_parse_topology_amd(tscan);
 126                 break;
 127         case X86_VENDOR_CENTAUR:
 128         case X86_VENDOR_ZHAOXIN:
 129                 parse_legacy(tscan);
 130                 break;
 131         case X86_VENDOR_INTEL:
 132                 if (!IS_ENABLED(CONFIG_CPU_SUP_INTEL) || !cpu_parse_topology_ext(tscan))
 133                         parse_legacy(tscan);
 134                 break;
 135         case X86_VENDOR_HYGON:
 136                 if (IS_ENABLED(CONFIG_CPU_SUP_HYGON))
 137                         cpu_parse_topology_amd(tscan);
 138                 break;
 139         }
 140 }
 141
 142 static void topo_set_ids(struct topo_scan *tscan)
 143 {
 144         struct cpuinfo_x86 *c = tscan->c;
 145         u32 apicid = c->topo.apicid;
 146
 147         c->topo.pkg_id = topo_shift_apicid(apicid, TOPO_PKG_DOMAIN);
 148         c->topo.die_id = topo_shift_apicid(apicid, TOPO_DIE_DOMAIN);
 149
 150         /* Package relative core ID */
 151         c->topo.core_id = (apicid & topo_domain_mask(TOPO_PKG_DOMAIN)) >>
 152                 x86_topo_system.dom_shifts[TOPO_SMT_DOMAIN];
 153
 154         /* Temporary workaround */
 155         if (tscan->amd_nodes_per_pkg)
 156                 c->topo.amd_node_id = c->topo.die_id = tscan->amd_node_id;
 157
 158         if (c->x86_vendor == X86_VENDOR_AMD)
 159                 cpu_topology_fixup_amd(tscan);
 160 }
 161
 162 static void topo_set_max_cores(struct topo_scan *tscan)
 163 {
 164         /*
 165          * Bug compatible for now. This is broken on hybrid systems:
 166          * 8 cores SMT + 8 cores w/o SMT
 167          * tscan.dom_ncpus[TOPO_DIEGRP_DOMAIN] = 24; 24 / 2 = 12 !!
 168          *
 169          * Cannot be fixed without further topology enumeration changes.
 170          */
 171         tscan->c->x86_max_cores = tscan->dom_ncpus[TOPO_DIEGRP_DOMAIN] >>
 172                 x86_topo_system.dom_shifts[TOPO_SMT_DOMAIN];
 173 }
 174
 175 void cpu_parse_topology(struct cpuinfo_x86 *c)
 176 {
 177         unsigned int dom, cpu = smp_processor_id();
 178         struct topo_scan tscan = { .c = c, };
 179
 180         parse_topology(&tscan, false);
 181
 182         for (dom = TOPO_SMT_DOMAIN; dom < TOPO_MAX_DOMAIN; dom++) {
 183                 if (tscan.dom_shifts[dom] == x86_topo_system.dom_shifts[dom])
 184                         continue;
 185                 pr_err(FW_BUG "CPU%d: Topology domain %u shift %u != %u\n", cpu, dom,
 186                        tscan.dom_shifts[dom], x86_topo_system.dom_shifts[dom]);
 187         }
 188
 189         /* Bug compatible with the existing parsers */
 190         if (tscan.dom_ncpus[TOPO_SMT_DOMAIN] > smp_num_siblings) {
 191                 if (system_state == SYSTEM_BOOTING) {
 192                         pr_warn_once("CPU%d: SMT detected and enabled late\n", cpu);
 193                         smp_num_siblings = tscan.dom_ncpus[TOPO_SMT_DOMAIN];
 194                 } else {
 195                         pr_warn_once("CPU%d: SMT detected after init. Too late!\n", cpu);
 196                 }
 197         }
 198
 199         topo_set_ids(&tscan);
 200         topo_set_max_cores(&tscan);
 201 }
 202
 203 void __init cpu_init_topology(struct cpuinfo_x86 *c)
 204 {
 205         struct topo_scan tscan = { .c = c, };
 206         unsigned int dom, sft;
 207
 208         parse_topology(&tscan, true);
 209
 210         /* Copy the shift values and calculate the unit sizes. */
 211         memcpy(x86_topo_system.dom_shifts, tscan.dom_shifts, sizeof(x86_topo_system.dom_shifts));
 212
 213         dom = TOPO_SMT_DOMAIN;
 214         x86_topo_system.dom_size[dom] = 1U << x86_topo_system.dom_shifts[dom];
 215
 216         for (dom++; dom < TOPO_MAX_DOMAIN; dom++) {
 217                 sft = x86_topo_system.dom_shifts[dom] - x86_topo_system.dom_shifts[dom - 1];
 218                 x86_topo_system.dom_size[dom] = 1U << sft;
 219         }
 220
 221         topo_set_ids(&tscan);
 222         topo_set_max_cores(&tscan);
 223
 224         /*
 225          * Bug compatible with the existing code. If the boot CPU does not
 226          * have SMT this ends up with one sibling. This needs way deeper
 227          * changes further down the road to get it right during early boot.
 228          */
 229         smp_num_siblings = tscan.dom_ncpus[TOPO_SMT_DOMAIN];
 230
 231         /*
 232          * Neither it's clear whether there are as many dies as the APIC
 233          * space indicating die level is. But assume that the actual number
 234          * of CPUs gives a proper indication for now to stay bug compatible.
 235          */
 236         __max_die_per_package = tscan.dom_ncpus[TOPO_DIE_DOMAIN] /
 237                 tscan.dom_ncpus[TOPO_DIE_DOMAIN - 1];
 238         /*
 239          * AMD systems have Nodes per package which cannot be mapped to
 240          * APIC ID.
 241          */
 242         if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON)
 243                 __amd_nodes_per_pkg = __max_die_per_package = tscan.amd_nodes_per_pkg;
 244 }