arch/x86/kernel/cpu/common.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* cpu_feature_enabled() cannot be used this early */
   3 #define USE_EARLY_PGTABLE_L5
   4
   5 #include <linux/memblock.h>
   6 #include <linux/linkage.h>
   7 #include <linux/bitops.h>
   8 #include <linux/kernel.h>
   9 #include <linux/export.h>
  10 #include <linux/percpu.h>
  11 #include <linux/string.h>
  12 #include <linux/ctype.h>
  13 #include <linux/delay.h>
  14 #include <linux/sched/mm.h>
  15 #include <linux/sched/clock.h>
  16 #include <linux/sched/task.h>
  17 #include <linux/sched/smt.h>
  18 #include <linux/init.h>
  19 #include <linux/kprobes.h>
  20 #include <linux/kgdb.h>
  21 #include <linux/mem_encrypt.h>
  22 #include <linux/smp.h>
  23 #include <linux/cpu.h>
  24 #include <linux/io.h>
  25 #include <linux/syscore_ops.h>
  26 #include <linux/pgtable.h>
  27 #include <linux/stackprotector.h>
  28 #include <linux/utsname.h>
  29
  30 #include <asm/alternative.h>
  31 #include <asm/cmdline.h>
  32 #include <asm/perf_event.h>
  33 #include <asm/mmu_context.h>
  34 #include <asm/doublefault.h>
  35 #include <asm/archrandom.h>
  36 #include <asm/hypervisor.h>
  37 #include <asm/processor.h>
  38 #include <asm/tlbflush.h>
  39 #include <asm/debugreg.h>
  40 #include <asm/sections.h>
  41 #include <asm/vsyscall.h>
  42 #include <linux/topology.h>
  43 #include <linux/cpumask.h>
  44 #include <linux/atomic.h>
  45 #include <asm/proto.h>
  46 #include <asm/setup.h>
  47 #include <asm/apic.h>
  48 #include <asm/desc.h>
  49 #include <asm/fpu/api.h>
  50 #include <asm/mtrr.h>
  51 #include <asm/hwcap2.h>
  52 #include <linux/numa.h>
  53 #include <asm/numa.h>
  54 #include <asm/asm.h>
  55 #include <asm/bugs.h>
  56 #include <asm/cpu.h>
  57 #include <asm/mce.h>
  58 #include <asm/msr.h>
  59 #include <asm/cacheinfo.h>
  60 #include <asm/memtype.h>
  61 #include <asm/microcode.h>
  62 #include <asm/intel-family.h>
  63 #include <asm/cpu_device_id.h>
  64 #include <asm/fred.h>
  65 #include <asm/uv/uv.h>
  66 #include <asm/ia32.h>
  67 #include <asm/set_memory.h>
  68 #include <asm/traps.h>
  69 #include <asm/sev.h>
  70 #include <asm/tdx.h>
  71
  72 #include "cpu.h"
  73
  74 DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
  75 EXPORT_PER_CPU_SYMBOL(cpu_info);
  76
  77 u32 elf_hwcap2 __read_mostly;
  78
  79 /* Number of siblings per CPU package */
  80 unsigned int __max_threads_per_core __ro_after_init = 1;
  81 EXPORT_SYMBOL(__max_threads_per_core);
  82
  83 unsigned int __max_dies_per_package __ro_after_init = 1;
  84 EXPORT_SYMBOL(__max_dies_per_package);
  85
  86 unsigned int __max_logical_packages __ro_after_init = 1;
  87 EXPORT_SYMBOL(__max_logical_packages);
  88
  89 unsigned int __num_cores_per_package __ro_after_init = 1;
  90 EXPORT_SYMBOL(__num_cores_per_package);
  91
  92 unsigned int __num_threads_per_package __ro_after_init = 1;
  93 EXPORT_SYMBOL(__num_threads_per_package);
  94
  95 static struct ppin_info {
  96         int     feature;
  97         int     msr_ppin_ctl;
  98         int     msr_ppin;
  99 } ppin_info[] = {
 100         [X86_VENDOR_INTEL] = {
 101                 .feature = X86_FEATURE_INTEL_PPIN,
 102                 .msr_ppin_ctl = MSR_PPIN_CTL,
 103                 .msr_ppin = MSR_PPIN
 104         },
 105         [X86_VENDOR_AMD] = {
 106                 .feature = X86_FEATURE_AMD_PPIN,
 107                 .msr_ppin_ctl = MSR_AMD_PPIN_CTL,
 108                 .msr_ppin = MSR_AMD_PPIN
 109         },
 110 };
 111
 112 static const struct x86_cpu_id ppin_cpuids[] = {
 113         X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
 114         X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
 115
 116         /* Legacy models without CPUID enumeration */
 117         X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
 118         X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 119         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
 120         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 121         X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 122         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 123         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
 124         X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 125         X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 126         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
 127         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
 128
 129         {}
 130 };
 131
 132 static void ppin_init(struct cpuinfo_x86 *c)
 133 {
 134         const struct x86_cpu_id *id;
 135         unsigned long long val;
 136         struct ppin_info *info;
 137
 138         id = x86_match_cpu(ppin_cpuids);
 139         if (!id)
 140                 return;
 141
 142         /*
 143          * Testing the presence of the MSR is not enough. Need to check
 144          * that the PPIN_CTL allows reading of the PPIN.
 145          */
 146         info = (struct ppin_info *)id->driver_data;
 147
 148         if (rdmsrl_safe(info->msr_ppin_ctl, &val))
 149                 goto clear_ppin;
 150
 151         if ((val & 3UL) == 1UL) {
 152                 /* PPIN locked in disabled mode */
 153                 goto clear_ppin;
 154         }
 155
 156         /* If PPIN is disabled, try to enable */
 157         if (!(val & 2UL)) {
 158                 wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
 159                 rdmsrl_safe(info->msr_ppin_ctl, &val);
 160         }
 161
 162         /* Is the enable bit set? */
 163         if (val & 2UL) {
 164                 c->ppin = __rdmsr(info->msr_ppin);
 165                 set_cpu_cap(c, info->feature);
 166                 return;
 167         }
 168
 169 clear_ppin:
 170         clear_cpu_cap(c, info->feature);
 171 }
 172
 173 static void default_init(struct cpuinfo_x86 *c)
 174 {
 175 #ifdef CONFIG_X86_64
 176         cpu_detect_cache_sizes(c);
 177 #else
 178         /* Not much we can do here... */
 179         /* Check if at least it has cpuid */
 180         if (c->cpuid_level == -1) {
 181                 /* No cpuid. It must be an ancient CPU */
 182                 if (c->x86 == 4)
 183                         strcpy(c->x86_model_id, "486");
 184                 else if (c->x86 == 3)
 185                         strcpy(c->x86_model_id, "386");
 186         }
 187 #endif
 188 }
 189
 190 static const struct cpu_dev default_cpu = {
 191         .c_init         = default_init,
 192         .c_vendor       = "Unknown",
 193         .c_x86_vendor   = X86_VENDOR_UNKNOWN,
 194 };
 195
 196 static const struct cpu_dev *this_cpu = &default_cpu;
 197
 198 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 199 #ifdef CONFIG_X86_64
 200         /*
 201          * We need valid kernel segments for data and code in long mode too
 202          * IRET will check the segment types  kkeil 2000/10/28
 203          * Also sysret mandates a special GDT layout
 204          *
 205          * TLS descriptors are currently at a different place compared to i386.
 206          * Hopefully nobody expects them at a fixed place (Wine?)
 207          */
 208         [GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 209         [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
 210         [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
 211         [GDT_ENTRY_DEFAULT_USER32_CS]   = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 212         [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
 213         [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
 214 #else
 215         [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 216         [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 217         [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 218         [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
 219         /*
 220          * Segments used for calling PnP BIOS have byte granularity.
 221          * They code segments and data segments have fixed 64k limits,
 222          * the transfer segment sizes are set at run time.
 223          */
 224         [GDT_ENTRY_PNPBIOS_CS32]        = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 225         [GDT_ENTRY_PNPBIOS_CS16]        = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 226         [GDT_ENTRY_PNPBIOS_DS]          = GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
 227         [GDT_ENTRY_PNPBIOS_TS1]         = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 228         [GDT_ENTRY_PNPBIOS_TS2]         = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 229         /*
 230          * The APM segments have byte granularity and their bases
 231          * are set at run time.  All have 64k limits.
 232          */
 233         [GDT_ENTRY_APMBIOS_BASE]        = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 234         [GDT_ENTRY_APMBIOS_BASE+1]      = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 235         [GDT_ENTRY_APMBIOS_BASE+2]      = GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
 236
 237         [GDT_ENTRY_ESPFIX_SS]           = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 238         [GDT_ENTRY_PERCPU]              = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 239 #endif
 240 } };
 241 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 242
 243 #ifdef CONFIG_X86_64
 244 static int __init x86_nopcid_setup(char *s)
 245 {
 246         /* nopcid doesn't accept parameters */
 247         if (s)
 248                 return -EINVAL;
 249
 250         /* do not emit a message if the feature is not present */
 251         if (!boot_cpu_has(X86_FEATURE_PCID))
 252                 return 0;
 253
 254         setup_clear_cpu_cap(X86_FEATURE_PCID);
 255         pr_info("nopcid: PCID feature disabled\n");
 256         return 0;
 257 }
 258 early_param("nopcid", x86_nopcid_setup);
 259 #endif
 260
 261 static int __init x86_noinvpcid_setup(char *s)
 262 {
 263         /* noinvpcid doesn't accept parameters */
 264         if (s)
 265                 return -EINVAL;
 266
 267         /* do not emit a message if the feature is not present */
 268         if (!boot_cpu_has(X86_FEATURE_INVPCID))
 269                 return 0;
 270
 271         setup_clear_cpu_cap(X86_FEATURE_INVPCID);
 272         pr_info("noinvpcid: INVPCID feature disabled\n");
 273         return 0;
 274 }
 275 early_param("noinvpcid", x86_noinvpcid_setup);
 276
 277 #ifdef CONFIG_X86_32
 278 static int cachesize_override = -1;
 279 static int disable_x86_serial_nr = 1;
 280
 281 static int __init cachesize_setup(char *str)
 282 {
 283         get_option(&str, &cachesize_override);
 284         return 1;
 285 }
 286 __setup("cachesize=", cachesize_setup);
 287
 288 /* Standard macro to see if a specific flag is changeable */
 289 static inline int flag_is_changeable_p(u32 flag)
 290 {
 291         u32 f1, f2;
 292
 293         /*
 294          * Cyrix and IDT cpus allow disabling of CPUID
 295          * so the code below may return different results
 296          * when it is executed before and after enabling
 297          * the CPUID. Add "volatile" to not allow gcc to
 298          * optimize the subsequent calls to this function.
 299          */
 300         asm volatile ("pushfl           \n\t"
 301                       "pushfl           \n\t"
 302                       "popl %0          \n\t"
 303                       "movl %0, %1      \n\t"
 304                       "xorl %2, %0      \n\t"
 305                       "pushl %0         \n\t"
 306                       "popfl            \n\t"
 307                       "pushfl           \n\t"
 308                       "popl %0          \n\t"
 309                       "popfl            \n\t"
 310
 311                       : "=&r" (f1), "=&r" (f2)
 312                       : "ir" (flag));
 313
 314         return ((f1^f2) & flag) != 0;
 315 }
 316
 317 /* Probe for the CPUID instruction */
 318 int have_cpuid_p(void)
 319 {
 320         return flag_is_changeable_p(X86_EFLAGS_ID);
 321 }
 322
 323 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 324 {
 325         unsigned long lo, hi;
 326
 327         if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
 328                 return;
 329
 330         /* Disable processor serial number: */
 331
 332         rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 333         lo |= 0x200000;
 334         wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 335
 336         pr_notice("CPU serial number disabled.\n");
 337         clear_cpu_cap(c, X86_FEATURE_PN);
 338
 339         /* Disabling the serial number may affect the cpuid level */
 340         c->cpuid_level = cpuid_eax(0);
 341 }
 342
 343 static int __init x86_serial_nr_setup(char *s)
 344 {
 345         disable_x86_serial_nr = 0;
 346         return 1;
 347 }
 348 __setup("serialnumber", x86_serial_nr_setup);
 349 #else
 350 static inline int flag_is_changeable_p(u32 flag)
 351 {
 352         return 1;
 353 }
 354 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 355 {
 356 }
 357 #endif
 358
 359 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 360 {
 361         if (cpu_has(c, X86_FEATURE_SMEP))
 362                 cr4_set_bits(X86_CR4_SMEP);
 363 }
 364
 365 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 366 {
 367         unsigned long eflags = native_save_fl();
 368
 369         /* This should have been cleared long ago */
 370         BUG_ON(eflags & X86_EFLAGS_AC);
 371
 372         if (cpu_has(c, X86_FEATURE_SMAP))
 373                 cr4_set_bits(X86_CR4_SMAP);
 374 }
 375
 376 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
 377 {
 378         /* Check the boot processor, plus build option for UMIP. */
 379         if (!cpu_feature_enabled(X86_FEATURE_UMIP))
 380                 goto out;
 381
 382         /* Check the current processor's cpuid bits. */
 383         if (!cpu_has(c, X86_FEATURE_UMIP))
 384                 goto out;
 385
 386         cr4_set_bits(X86_CR4_UMIP);
 387
 388         pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
 389
 390         return;
 391
 392 out:
 393         /*
 394          * Make sure UMIP is disabled in case it was enabled in a
 395          * previous boot (e.g., via kexec).
 396          */
 397         cr4_clear_bits(X86_CR4_UMIP);
 398 }
 399
 400 /* These bits should not change their value after CPU init is finished. */
 401 static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
 402                                              X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED;
 403 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
 404 static unsigned long cr4_pinned_bits __ro_after_init;
 405
 406 void native_write_cr0(unsigned long val)
 407 {
 408         unsigned long bits_missing = 0;
 409
 410 set_register:
 411         asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
 412
 413         if (static_branch_likely(&cr_pinning)) {
 414                 if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
 415                         bits_missing = X86_CR0_WP;
 416                         val |= bits_missing;
 417                         goto set_register;
 418                 }
 419                 /* Warn after we've set the missing bits. */
 420                 WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
 421         }
 422 }
 423 EXPORT_SYMBOL(native_write_cr0);
 424
 425 void __no_profile native_write_cr4(unsigned long val)
 426 {
 427         unsigned long bits_changed = 0;
 428
 429 set_register:
 430         asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
 431
 432         if (static_branch_likely(&cr_pinning)) {
 433                 if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
 434                         bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
 435                         val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
 436                         goto set_register;
 437                 }
 438                 /* Warn after we've corrected the changed bits. */
 439                 WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
 440                           bits_changed);
 441         }
 442 }
 443 #if IS_MODULE(CONFIG_LKDTM)
 444 EXPORT_SYMBOL_GPL(native_write_cr4);
 445 #endif
 446
 447 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
 448 {
 449         unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
 450
 451         lockdep_assert_irqs_disabled();
 452
 453         newval = (cr4 & ~clear) | set;
 454         if (newval != cr4) {
 455                 this_cpu_write(cpu_tlbstate.cr4, newval);
 456                 __write_cr4(newval);
 457         }
 458 }
 459 EXPORT_SYMBOL(cr4_update_irqsoff);
 460
 461 /* Read the CR4 shadow. */
 462 unsigned long cr4_read_shadow(void)
 463 {
 464         return this_cpu_read(cpu_tlbstate.cr4);
 465 }
 466 EXPORT_SYMBOL_GPL(cr4_read_shadow);
 467
 468 void cr4_init(void)
 469 {
 470         unsigned long cr4 = __read_cr4();
 471
 472         if (boot_cpu_has(X86_FEATURE_PCID))
 473                 cr4 |= X86_CR4_PCIDE;
 474         if (static_branch_likely(&cr_pinning))
 475                 cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
 476
 477         __write_cr4(cr4);
 478
 479         /* Initialize cr4 shadow for this CPU. */
 480         this_cpu_write(cpu_tlbstate.cr4, cr4);
 481 }
 482
 483 /*
 484  * Once CPU feature detection is finished (and boot params have been
 485  * parsed), record any of the sensitive CR bits that are set, and
 486  * enable CR pinning.
 487  */
 488 static void __init setup_cr_pinning(void)
 489 {
 490         cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
 491         static_key_enable(&cr_pinning.key);
 492 }
 493
 494 static __init int x86_nofsgsbase_setup(char *arg)
 495 {
 496         /* Require an exact match without trailing characters. */
 497         if (strlen(arg))
 498                 return 0;
 499
 500         /* Do not emit a message if the feature is not present. */
 501         if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
 502                 return 1;
 503
 504         setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
 505         pr_info("FSGSBASE disabled via kernel command line\n");
 506         return 1;
 507 }
 508 __setup("nofsgsbase", x86_nofsgsbase_setup);
 509
 510 /*
 511  * Protection Keys are not available in 32-bit mode.
 512  */
 513 static bool pku_disabled;
 514
 515 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
 516 {
 517         if (c == &boot_cpu_data) {
 518                 if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
 519                         return;
 520                 /*
 521                  * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
 522                  * bit to be set.  Enforce it.
 523                  */
 524                 setup_force_cpu_cap(X86_FEATURE_OSPKE);
 525
 526         } else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
 527                 return;
 528         }
 529
 530         cr4_set_bits(X86_CR4_PKE);
 531         /* Load the default PKRU value */
 532         pkru_write_default();
 533 }
 534
 535 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 536 static __init int setup_disable_pku(char *arg)
 537 {
 538         /*
 539          * Do not clear the X86_FEATURE_PKU bit.  All of the
 540          * runtime checks are against OSPKE so clearing the
 541          * bit does nothing.
 542          *
 543          * This way, we will see "pku" in cpuinfo, but not
 544          * "ospke", which is exactly what we want.  It shows
 545          * that the CPU has PKU, but the OS has not enabled it.
 546          * This happens to be exactly how a system would look
 547          * if we disabled the config option.
 548          */
 549         pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
 550         pku_disabled = true;
 551         return 1;
 552 }
 553 __setup("nopku", setup_disable_pku);
 554 #endif
 555
 556 #ifdef CONFIG_X86_KERNEL_IBT
 557
 558 __noendbr u64 ibt_save(bool disable)
 559 {
 560         u64 msr = 0;
 561
 562         if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 563                 rdmsrl(MSR_IA32_S_CET, msr);
 564                 if (disable)
 565                         wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
 566         }
 567
 568         return msr;
 569 }
 570
 571 __noendbr void ibt_restore(u64 save)
 572 {
 573         u64 msr;
 574
 575         if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 576                 rdmsrl(MSR_IA32_S_CET, msr);
 577                 msr &= ~CET_ENDBR_EN;
 578                 msr |= (save & CET_ENDBR_EN);
 579                 wrmsrl(MSR_IA32_S_CET, msr);
 580         }
 581 }
 582
 583 #endif
 584
 585 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
 586 {
 587         bool user_shstk, kernel_ibt;
 588
 589         if (!IS_ENABLED(CONFIG_X86_CET))
 590                 return;
 591
 592         kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
 593         user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
 594                      IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
 595
 596         if (!kernel_ibt && !user_shstk)
 597                 return;
 598
 599         if (user_shstk)
 600                 set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
 601
 602         if (kernel_ibt)
 603                 wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
 604         else
 605                 wrmsrl(MSR_IA32_S_CET, 0);
 606
 607         cr4_set_bits(X86_CR4_CET);
 608
 609         if (kernel_ibt && ibt_selftest()) {
 610                 pr_err("IBT selftest: Failed!\n");
 611                 wrmsrl(MSR_IA32_S_CET, 0);
 612                 setup_clear_cpu_cap(X86_FEATURE_IBT);
 613         }
 614 }
 615
 616 __noendbr void cet_disable(void)
 617 {
 618         if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
 619               cpu_feature_enabled(X86_FEATURE_SHSTK)))
 620                 return;
 621
 622         wrmsrl(MSR_IA32_S_CET, 0);
 623         wrmsrl(MSR_IA32_U_CET, 0);
 624 }
 625
 626 /*
 627  * Some CPU features depend on higher CPUID levels, which may not always
 628  * be available due to CPUID level capping or broken virtualization
 629  * software.  Add those features to this table to auto-disable them.
 630  */
 631 struct cpuid_dependent_feature {
 632         u32 feature;
 633         u32 level;
 634 };
 635
 636 static const struct cpuid_dependent_feature
 637 cpuid_dependent_features[] = {
 638         { X86_FEATURE_MWAIT,            0x00000005 },
 639         { X86_FEATURE_DCA,              0x00000009 },
 640         { X86_FEATURE_XSAVE,            0x0000000d },
 641         { 0, 0 }
 642 };
 643
 644 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
 645 {
 646         const struct cpuid_dependent_feature *df;
 647
 648         for (df = cpuid_dependent_features; df->feature; df++) {
 649
 650                 if (!cpu_has(c, df->feature))
 651                         continue;
 652                 /*
 653                  * Note: cpuid_level is set to -1 if unavailable, but
 654                  * extended_extended_level is set to 0 if unavailable
 655                  * and the legitimate extended levels are all negative
 656                  * when signed; hence the weird messing around with
 657                  * signs here...
 658                  */
 659                 if (!((s32)df->level < 0 ?
 660                      (u32)df->level > (u32)c->extended_cpuid_level :
 661                      (s32)df->level > (s32)c->cpuid_level))
 662                         continue;
 663
 664                 clear_cpu_cap(c, df->feature);
 665                 if (!warn)
 666                         continue;
 667
 668                 pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
 669                         x86_cap_flag(df->feature), df->level);
 670         }
 671 }
 672
 673 /*
 674  * Naming convention should be: <Name> [(<Codename>)]
 675  * This table only is used unless init_<vendor>() below doesn't set it;
 676  * in particular, if CPUID levels 0x80000002..4 are supported, this
 677  * isn't used
 678  */
 679
 680 /* Look up CPU names by table lookup. */
 681 static const char *table_lookup_model(struct cpuinfo_x86 *c)
 682 {
 683 #ifdef CONFIG_X86_32
 684         const struct legacy_cpu_model_info *info;
 685
 686         if (c->x86_model >= 16)
 687                 return NULL;    /* Range check */
 688
 689         if (!this_cpu)
 690                 return NULL;
 691
 692         info = this_cpu->legacy_models;
 693
 694         while (info->family) {
 695                 if (info->family == c->x86)
 696                         return info->model_names[c->x86_model];
 697                 info++;
 698         }
 699 #endif
 700         return NULL;            /* Not found */
 701 }
 702
 703 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
 704 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 705 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 706
 707 #ifdef CONFIG_X86_32
 708 /* The 32-bit entry code needs to find cpu_entry_area. */
 709 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 710 #endif
 711
 712 /* Load the original GDT from the per-cpu structure */
 713 void load_direct_gdt(int cpu)
 714 {
 715         struct desc_ptr gdt_descr;
 716
 717         gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
 718         gdt_descr.size = GDT_SIZE - 1;
 719         load_gdt(&gdt_descr);
 720 }
 721 EXPORT_SYMBOL_GPL(load_direct_gdt);
 722
 723 /* Load a fixmap remapping of the per-cpu GDT */
 724 void load_fixmap_gdt(int cpu)
 725 {
 726         struct desc_ptr gdt_descr;
 727
 728         gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
 729         gdt_descr.size = GDT_SIZE - 1;
 730         load_gdt(&gdt_descr);
 731 }
 732 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
 733
 734 /**
 735  * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
 736  * @cpu:        The CPU number for which this is invoked
 737  *
 738  * Invoked during early boot to switch from early GDT and early per CPU to
 739  * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
 740  * switch is implicit by loading the direct GDT. On 64bit this requires
 741  * to update GSBASE.
 742  */
 743 void __init switch_gdt_and_percpu_base(int cpu)
 744 {
 745         load_direct_gdt(cpu);
 746
 747 #ifdef CONFIG_X86_64
 748         /*
 749          * No need to load %gs. It is already correct.
 750          *
 751          * Writing %gs on 64bit would zero GSBASE which would make any per
 752          * CPU operation up to the point of the wrmsrl() fault.
 753          *
 754          * Set GSBASE to the new offset. Until the wrmsrl() happens the
 755          * early mapping is still valid. That means the GSBASE update will
 756          * lose any prior per CPU data which was not copied over in
 757          * setup_per_cpu_areas().
 758          *
 759          * This works even with stackprotector enabled because the
 760          * per CPU stack canary is 0 in both per CPU areas.
 761          */
 762         wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
 763 #else
 764         /*
 765          * %fs is already set to __KERNEL_PERCPU, but after switching GDT
 766          * it is required to load FS again so that the 'hidden' part is
 767          * updated from the new GDT. Up to this point the early per CPU
 768          * translation is active. Any content of the early per CPU data
 769          * which was not copied over in setup_per_cpu_areas() is lost.
 770          */
 771         loadsegment(fs, __KERNEL_PERCPU);
 772 #endif
 773 }
 774
 775 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
 776
 777 static void get_model_name(struct cpuinfo_x86 *c)
 778 {
 779         unsigned int *v;
 780         char *p, *q, *s;
 781
 782         if (c->extended_cpuid_level < 0x80000004)
 783                 return;
 784
 785         v = (unsigned int *)c->x86_model_id;
 786         cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
 787         cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
 788         cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
 789         c->x86_model_id[48] = 0;
 790
 791         /* Trim whitespace */
 792         p = q = s = &c->x86_model_id[0];
 793
 794         while (*p == ' ')
 795                 p++;
 796
 797         while (*p) {
 798                 /* Note the last non-whitespace index */
 799                 if (!isspace(*p))
 800                         s = q;
 801
 802                 *q++ = *p++;
 803         }
 804
 805         *(s + 1) = '\0';
 806 }
 807
 808 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 809 {
 810         unsigned int n, dummy, ebx, ecx, edx, l2size;
 811
 812         n = c->extended_cpuid_level;
 813
 814         if (n >= 0x80000005) {
 815                 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
 816                 c->x86_cache_size = (ecx>>24) + (edx>>24);
 817 #ifdef CONFIG_X86_64
 818                 /* On K8 L1 TLB is inclusive, so don't count it */
 819                 c->x86_tlbsize = 0;
 820 #endif
 821         }
 822
 823         if (n < 0x80000006)     /* Some chips just has a large L1. */
 824                 return;
 825
 826         cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
 827         l2size = ecx >> 16;
 828
 829 #ifdef CONFIG_X86_64
 830         c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
 831 #else
 832         /* do processor-specific cache resizing */
 833         if (this_cpu->legacy_cache_size)
 834                 l2size = this_cpu->legacy_cache_size(c, l2size);
 835
 836         /* Allow user to override all this if necessary. */
 837         if (cachesize_override != -1)
 838                 l2size = cachesize_override;
 839
 840         if (l2size == 0)
 841                 return;         /* Again, no L2 cache is possible */
 842 #endif
 843
 844         c->x86_cache_size = l2size;
 845 }
 846
 847 u16 __read_mostly tlb_lli_4k[NR_INFO];
 848 u16 __read_mostly tlb_lli_2m[NR_INFO];
 849 u16 __read_mostly tlb_lli_4m[NR_INFO];
 850 u16 __read_mostly tlb_lld_4k[NR_INFO];
 851 u16 __read_mostly tlb_lld_2m[NR_INFO];
 852 u16 __read_mostly tlb_lld_4m[NR_INFO];
 853 u16 __read_mostly tlb_lld_1g[NR_INFO];
 854
 855 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
 856 {
 857         if (this_cpu->c_detect_tlb)
 858                 this_cpu->c_detect_tlb(c);
 859
 860         pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
 861                 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 862                 tlb_lli_4m[ENTRIES]);
 863
 864         pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
 865                 tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
 866                 tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
 867 }
 868
 869 static void get_cpu_vendor(struct cpuinfo_x86 *c)
 870 {
 871         char *v = c->x86_vendor_id;
 872         int i;
 873
 874         for (i = 0; i < X86_VENDOR_NUM; i++) {
 875                 if (!cpu_devs[i])
 876                         break;
 877
 878                 if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
 879                     (cpu_devs[i]->c_ident[1] &&
 880                      !strcmp(v, cpu_devs[i]->c_ident[1]))) {
 881
 882                         this_cpu = cpu_devs[i];
 883                         c->x86_vendor = this_cpu->c_x86_vendor;
 884                         return;
 885                 }
 886         }
 887
 888         pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
 889                     "CPU: Your system may be unstable.\n", v);
 890
 891         c->x86_vendor = X86_VENDOR_UNKNOWN;
 892         this_cpu = &default_cpu;
 893 }
 894
 895 void cpu_detect(struct cpuinfo_x86 *c)
 896 {
 897         /* Get vendor name */
 898         cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
 899               (unsigned int *)&c->x86_vendor_id[0],
 900               (unsigned int *)&c->x86_vendor_id[8],
 901               (unsigned int *)&c->x86_vendor_id[4]);
 902
 903         c->x86 = 4;
 904         /* Intel-defined flags: level 0x00000001 */
 905         if (c->cpuid_level >= 0x00000001) {
 906                 u32 junk, tfms, cap0, misc;
 907
 908                 cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
 909                 c->x86          = x86_family(tfms);
 910                 c->x86_model    = x86_model(tfms);
 911                 c->x86_stepping = x86_stepping(tfms);
 912
 913                 if (cap0 & (1<<19)) {
 914                         c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
 915                         c->x86_cache_alignment = c->x86_clflush_size;
 916                 }
 917         }
 918 }
 919
 920 static void apply_forced_caps(struct cpuinfo_x86 *c)
 921 {
 922         int i;
 923
 924         for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
 925                 c->x86_capability[i] &= ~cpu_caps_cleared[i];
 926                 c->x86_capability[i] |= cpu_caps_set[i];
 927         }
 928 }
 929
 930 static void init_speculation_control(struct cpuinfo_x86 *c)
 931 {
 932         /*
 933          * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
 934          * and they also have a different bit for STIBP support. Also,
 935          * a hypervisor might have set the individual AMD bits even on
 936          * Intel CPUs, for finer-grained selection of what's available.
 937          */
 938         if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
 939                 set_cpu_cap(c, X86_FEATURE_IBRS);
 940                 set_cpu_cap(c, X86_FEATURE_IBPB);
 941                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 942         }
 943
 944         if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
 945                 set_cpu_cap(c, X86_FEATURE_STIBP);
 946
 947         if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
 948             cpu_has(c, X86_FEATURE_VIRT_SSBD))
 949                 set_cpu_cap(c, X86_FEATURE_SSBD);
 950
 951         if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
 952                 set_cpu_cap(c, X86_FEATURE_IBRS);
 953                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 954         }
 955
 956         if (cpu_has(c, X86_FEATURE_AMD_IBPB))
 957                 set_cpu_cap(c, X86_FEATURE_IBPB);
 958
 959         if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
 960                 set_cpu_cap(c, X86_FEATURE_STIBP);
 961                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 962         }
 963
 964         if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
 965                 set_cpu_cap(c, X86_FEATURE_SSBD);
 966                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 967                 clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
 968         }
 969 }
 970
 971 void get_cpu_cap(struct cpuinfo_x86 *c)
 972 {
 973         u32 eax, ebx, ecx, edx;
 974
 975         /* Intel-defined flags: level 0x00000001 */
 976         if (c->cpuid_level >= 0x00000001) {
 977                 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
 978
 979                 c->x86_capability[CPUID_1_ECX] = ecx;
 980                 c->x86_capability[CPUID_1_EDX] = edx;
 981         }
 982
 983         /* Thermal and Power Management Leaf: level 0x00000006 (eax) */
 984         if (c->cpuid_level >= 0x00000006)
 985                 c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
 986
 987         /* Additional Intel-defined flags: level 0x00000007 */
 988         if (c->cpuid_level >= 0x00000007) {
 989                 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
 990                 c->x86_capability[CPUID_7_0_EBX] = ebx;
 991                 c->x86_capability[CPUID_7_ECX] = ecx;
 992                 c->x86_capability[CPUID_7_EDX] = edx;
 993
 994                 /* Check valid sub-leaf index before accessing it */
 995                 if (eax >= 1) {
 996                         cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
 997                         c->x86_capability[CPUID_7_1_EAX] = eax;
 998                 }
 999         }
1000
1001         /* Extended state features: level 0x0000000d */
1002         if (c->cpuid_level >= 0x0000000d) {
1003                 cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1004
1005                 c->x86_capability[CPUID_D_1_EAX] = eax;
1006         }
1007
1008         /* AMD-defined flags: level 0x80000001 */
1009         eax = cpuid_eax(0x80000000);
1010         c->extended_cpuid_level = eax;
1011
1012         if ((eax & 0xffff0000) == 0x80000000) {
1013                 if (eax >= 0x80000001) {
1014                         cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1015
1016                         c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1017                         c->x86_capability[CPUID_8000_0001_EDX] = edx;
1018                 }
1019         }
1020
1021         if (c->extended_cpuid_level >= 0x80000007) {
1022                 cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1023
1024                 c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1025                 c->x86_power = edx;
1026         }
1027
1028         if (c->extended_cpuid_level >= 0x80000008) {
1029                 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1030                 c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1031         }
1032
1033         if (c->extended_cpuid_level >= 0x8000000a)
1034                 c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1035
1036         if (c->extended_cpuid_level >= 0x8000001f)
1037                 c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1038
1039         if (c->extended_cpuid_level >= 0x80000021)
1040                 c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1041
1042         init_scattered_cpuid_features(c);
1043         init_speculation_control(c);
1044
1045         /*
1046          * Clear/Set all flags overridden by options, after probe.
1047          * This needs to happen each time we re-probe, which may happen
1048          * several times during CPU initialization.
1049          */
1050         apply_forced_caps(c);
1051 }
1052
1053 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1054 {
1055         u32 eax, ebx, ecx, edx;
1056         bool vp_bits_from_cpuid = true;
1057
1058         if (!cpu_has(c, X86_FEATURE_CPUID) ||
1059             (c->extended_cpuid_level < 0x80000008))
1060                 vp_bits_from_cpuid = false;
1061
1062         if (vp_bits_from_cpuid) {
1063                 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1064
1065                 c->x86_virt_bits = (eax >> 8) & 0xff;
1066                 c->x86_phys_bits = eax & 0xff;
1067         } else {
1068                 if (IS_ENABLED(CONFIG_X86_64)) {
1069                         c->x86_clflush_size = 64;
1070                         c->x86_phys_bits = 36;
1071                         c->x86_virt_bits = 48;
1072                 } else {
1073                         c->x86_clflush_size = 32;
1074                         c->x86_virt_bits = 32;
1075                         c->x86_phys_bits = 32;
1076
1077                         if (cpu_has(c, X86_FEATURE_PAE) ||
1078                             cpu_has(c, X86_FEATURE_PSE36))
1079                                 c->x86_phys_bits = 36;
1080                 }
1081         }
1082         c->x86_cache_bits = c->x86_phys_bits;
1083         c->x86_cache_alignment = c->x86_clflush_size;
1084 }
1085
1086 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1087 {
1088 #ifdef CONFIG_X86_32
1089         int i;
1090
1091         /*
1092          * First of all, decide if this is a 486 or higher
1093          * It's a 486 if we can modify the AC flag
1094          */
1095         if (flag_is_changeable_p(X86_EFLAGS_AC))
1096                 c->x86 = 4;
1097         else
1098                 c->x86 = 3;
1099
1100         for (i = 0; i < X86_VENDOR_NUM; i++)
1101                 if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1102                         c->x86_vendor_id[0] = 0;
1103                         cpu_devs[i]->c_identify(c);
1104                         if (c->x86_vendor_id[0]) {
1105                                 get_cpu_vendor(c);
1106                                 break;
1107                         }
1108                 }
1109 #endif
1110 }
1111
1112 #define NO_SPECULATION          BIT(0)
1113 #define NO_MELTDOWN             BIT(1)
1114 #define NO_SSB                  BIT(2)
1115 #define NO_L1TF                 BIT(3)
1116 #define NO_MDS                  BIT(4)
1117 #define MSBDS_ONLY              BIT(5)
1118 #define NO_SWAPGS               BIT(6)
1119 #define NO_ITLB_MULTIHIT        BIT(7)
1120 #define NO_SPECTRE_V2           BIT(8)
1121 #define NO_MMIO                 BIT(9)
1122 #define NO_EIBRS_PBRSB          BIT(10)
1123
1124 #define VULNWL(vendor, family, model, whitelist)        \
1125         X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1126
1127 #define VULNWL_INTEL(model, whitelist)          \
1128         VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1129
1130 #define VULNWL_AMD(family, whitelist)           \
1131         VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1132
1133 #define VULNWL_HYGON(family, whitelist)         \
1134         VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1135
1136 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1137         VULNWL(ANY,     4, X86_MODEL_ANY,       NO_SPECULATION),
1138         VULNWL(CENTAUR, 5, X86_MODEL_ANY,       NO_SPECULATION),
1139         VULNWL(INTEL,   5, X86_MODEL_ANY,       NO_SPECULATION),
1140         VULNWL(NSC,     5, X86_MODEL_ANY,       NO_SPECULATION),
1141         VULNWL(VORTEX,  5, X86_MODEL_ANY,       NO_SPECULATION),
1142         VULNWL(VORTEX,  6, X86_MODEL_ANY,       NO_SPECULATION),
1143
1144         /* Intel Family 6 */
1145         VULNWL_INTEL(TIGERLAKE,                 NO_MMIO),
1146         VULNWL_INTEL(TIGERLAKE_L,               NO_MMIO),
1147         VULNWL_INTEL(ALDERLAKE,                 NO_MMIO),
1148         VULNWL_INTEL(ALDERLAKE_L,               NO_MMIO),
1149
1150         VULNWL_INTEL(ATOM_SALTWELL,             NO_SPECULATION | NO_ITLB_MULTIHIT),
1151         VULNWL_INTEL(ATOM_SALTWELL_TABLET,      NO_SPECULATION | NO_ITLB_MULTIHIT),
1152         VULNWL_INTEL(ATOM_SALTWELL_MID,         NO_SPECULATION | NO_ITLB_MULTIHIT),
1153         VULNWL_INTEL(ATOM_BONNELL,              NO_SPECULATION | NO_ITLB_MULTIHIT),
1154         VULNWL_INTEL(ATOM_BONNELL_MID,          NO_SPECULATION | NO_ITLB_MULTIHIT),
1155
1156         VULNWL_INTEL(ATOM_SILVERMONT,           NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1157         VULNWL_INTEL(ATOM_SILVERMONT_D,         NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1158         VULNWL_INTEL(ATOM_SILVERMONT_MID,       NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1159         VULNWL_INTEL(ATOM_AIRMONT,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1160         VULNWL_INTEL(XEON_PHI_KNL,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1161         VULNWL_INTEL(XEON_PHI_KNM,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1162
1163         VULNWL_INTEL(CORE_YONAH,                NO_SSB),
1164
1165         VULNWL_INTEL(ATOM_AIRMONT_MID,          NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1166         VULNWL_INTEL(ATOM_AIRMONT_NP,           NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1167
1168         VULNWL_INTEL(ATOM_GOLDMONT,             NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1169         VULNWL_INTEL(ATOM_GOLDMONT_D,           NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1170         VULNWL_INTEL(ATOM_GOLDMONT_PLUS,        NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1171
1172         /*
1173          * Technically, swapgs isn't serializing on AMD (despite it previously
1174          * being documented as such in the APM).  But according to AMD, %gs is
1175          * updated non-speculatively, and the issuing of %gs-relative memory
1176          * operands will be blocked until the %gs update completes, which is
1177          * good enough for our purposes.
1178          */
1179
1180         VULNWL_INTEL(ATOM_TREMONT,              NO_EIBRS_PBRSB),
1181         VULNWL_INTEL(ATOM_TREMONT_L,            NO_EIBRS_PBRSB),
1182         VULNWL_INTEL(ATOM_TREMONT_D,            NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1183
1184         /* AMD Family 0xf - 0x12 */
1185         VULNWL_AMD(0x0f,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1186         VULNWL_AMD(0x10,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1187         VULNWL_AMD(0x11,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1188         VULNWL_AMD(0x12,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1189
1190         /* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1191         VULNWL_AMD(X86_FAMILY_ANY,      NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1192         VULNWL_HYGON(X86_FAMILY_ANY,    NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1193
1194         /* Zhaoxin Family 7 */
1195         VULNWL(CENTAUR, 7, X86_MODEL_ANY,       NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1196         VULNWL(ZHAOXIN, 7, X86_MODEL_ANY,       NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1197         {}
1198 };
1199
1200 #define VULNBL(vendor, family, model, blacklist)        \
1201         X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1202
1203 #define VULNBL_INTEL_STEPPINGS(model, steppings, issues)                   \
1204         X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6,             \
1205                                             INTEL_FAM6_##model, steppings, \
1206                                             X86_FEATURE_ANY, issues)
1207
1208 #define VULNBL_AMD(family, blacklist)           \
1209         VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1210
1211 #define VULNBL_HYGON(family, blacklist)         \
1212         VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1213
1214 #define SRBDS           BIT(0)
1215 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1216 #define MMIO            BIT(1)
1217 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1218 #define MMIO_SBDS       BIT(2)
1219 /* CPU is affected by RETbleed, speculating where you would not expect it */
1220 #define RETBLEED        BIT(3)
1221 /* CPU is affected by SMT (cross-thread) return predictions */
1222 #define SMT_RSB         BIT(4)
1223 /* CPU is affected by SRSO */
1224 #define SRSO            BIT(5)
1225 /* CPU is affected by GDS */
1226 #define GDS             BIT(6)
1227 /* CPU is affected by Register File Data Sampling */
1228 #define RFDS            BIT(7)
1229
1230 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1231         VULNBL_INTEL_STEPPINGS(IVYBRIDGE,       X86_STEPPING_ANY,               SRBDS),
1232         VULNBL_INTEL_STEPPINGS(HASWELL,         X86_STEPPING_ANY,               SRBDS),
1233         VULNBL_INTEL_STEPPINGS(HASWELL_L,       X86_STEPPING_ANY,               SRBDS),
1234         VULNBL_INTEL_STEPPINGS(HASWELL_G,       X86_STEPPING_ANY,               SRBDS),
1235         VULNBL_INTEL_STEPPINGS(HASWELL_X,       X86_STEPPING_ANY,               MMIO),
1236         VULNBL_INTEL_STEPPINGS(BROADWELL_D,     X86_STEPPING_ANY,               MMIO),
1237         VULNBL_INTEL_STEPPINGS(BROADWELL_G,     X86_STEPPING_ANY,               SRBDS),
1238         VULNBL_INTEL_STEPPINGS(BROADWELL_X,     X86_STEPPING_ANY,               MMIO),
1239         VULNBL_INTEL_STEPPINGS(BROADWELL,       X86_STEPPING_ANY,               SRBDS),
1240         VULNBL_INTEL_STEPPINGS(SKYLAKE_X,       X86_STEPPING_ANY,               MMIO | RETBLEED | GDS),
1241         VULNBL_INTEL_STEPPINGS(SKYLAKE_L,       X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1242         VULNBL_INTEL_STEPPINGS(SKYLAKE,         X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1243         VULNBL_INTEL_STEPPINGS(KABYLAKE_L,      X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1244         VULNBL_INTEL_STEPPINGS(KABYLAKE,        X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1245         VULNBL_INTEL_STEPPINGS(CANNONLAKE_L,    X86_STEPPING_ANY,               RETBLEED),
1246         VULNBL_INTEL_STEPPINGS(ICELAKE_L,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1247         VULNBL_INTEL_STEPPINGS(ICELAKE_D,       X86_STEPPING_ANY,               MMIO | GDS),
1248         VULNBL_INTEL_STEPPINGS(ICELAKE_X,       X86_STEPPING_ANY,               MMIO | GDS),
1249         VULNBL_INTEL_STEPPINGS(COMETLAKE,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1250         VULNBL_INTEL_STEPPINGS(COMETLAKE_L,     X86_STEPPINGS(0x0, 0x0),        MMIO | RETBLEED),
1251         VULNBL_INTEL_STEPPINGS(COMETLAKE_L,     X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1252         VULNBL_INTEL_STEPPINGS(TIGERLAKE_L,     X86_STEPPING_ANY,               GDS),
1253         VULNBL_INTEL_STEPPINGS(TIGERLAKE,       X86_STEPPING_ANY,               GDS),
1254         VULNBL_INTEL_STEPPINGS(LAKEFIELD,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED),
1255         VULNBL_INTEL_STEPPINGS(ROCKETLAKE,      X86_STEPPING_ANY,               MMIO | RETBLEED | GDS),
1256         VULNBL_INTEL_STEPPINGS(ALDERLAKE,       X86_STEPPING_ANY,               RFDS),
1257         VULNBL_INTEL_STEPPINGS(ALDERLAKE_L,     X86_STEPPING_ANY,               RFDS),
1258         VULNBL_INTEL_STEPPINGS(RAPTORLAKE,      X86_STEPPING_ANY,               RFDS),
1259         VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P,    X86_STEPPING_ANY,               RFDS),
1260         VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S,    X86_STEPPING_ANY,               RFDS),
1261         VULNBL_INTEL_STEPPINGS(ATOM_GRACEMONT,  X86_STEPPING_ANY,               RFDS),
1262         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT,    X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RFDS),
1263         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D,  X86_STEPPING_ANY,               MMIO | RFDS),
1264         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L,  X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RFDS),
1265         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT,   X86_STEPPING_ANY,               RFDS),
1266         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_D, X86_STEPPING_ANY,               RFDS),
1267         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_PLUS, X86_STEPPING_ANY,            RFDS),
1268
1269         VULNBL_AMD(0x15, RETBLEED),
1270         VULNBL_AMD(0x16, RETBLEED),
1271         VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1272         VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1273         VULNBL_AMD(0x19, SRSO),
1274         {}
1275 };
1276
1277 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1278 {
1279         const struct x86_cpu_id *m = x86_match_cpu(table);
1280
1281         return m && !!(m->driver_data & which);
1282 }
1283
1284 u64 x86_read_arch_cap_msr(void)
1285 {
1286         u64 ia32_cap = 0;
1287
1288         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1289                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
1290
1291         return ia32_cap;
1292 }
1293
1294 static bool arch_cap_mmio_immune(u64 ia32_cap)
1295 {
1296         return (ia32_cap & ARCH_CAP_FBSDP_NO &&
1297                 ia32_cap & ARCH_CAP_PSDP_NO &&
1298                 ia32_cap & ARCH_CAP_SBDR_SSDP_NO);
1299 }
1300
1301 static bool __init vulnerable_to_rfds(u64 ia32_cap)
1302 {
1303         /* The "immunity" bit trumps everything else: */
1304         if (ia32_cap & ARCH_CAP_RFDS_NO)
1305                 return false;
1306
1307         /*
1308          * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1309          * indicate that mitigation is needed because guest is running on a
1310          * vulnerable hardware or may migrate to such hardware:
1311          */
1312         if (ia32_cap & ARCH_CAP_RFDS_CLEAR)
1313                 return true;
1314
1315         /* Only consult the blacklist when there is no enumeration: */
1316         return cpu_matches(cpu_vuln_blacklist, RFDS);
1317 }
1318
1319 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1320 {
1321         u64 ia32_cap = x86_read_arch_cap_msr();
1322
1323         /* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1324         if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1325             !(ia32_cap & ARCH_CAP_PSCHANGE_MC_NO))
1326                 setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1327
1328         if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1329                 return;
1330
1331         setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1332
1333         if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1334                 setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1335
1336         if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1337             !(ia32_cap & ARCH_CAP_SSB_NO) &&
1338            !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1339                 setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1340
1341         /*
1342          * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1343          * flag and protect from vendor-specific bugs via the whitelist.
1344          *
1345          * Don't use AutoIBRS when SNP is enabled because it degrades host
1346          * userspace indirect branch performance.
1347          */
1348         if ((ia32_cap & ARCH_CAP_IBRS_ALL) ||
1349             (cpu_has(c, X86_FEATURE_AUTOIBRS) &&
1350              !cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
1351                 setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1352                 if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1353                     !(ia32_cap & ARCH_CAP_PBRSB_NO))
1354                         setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1355         }
1356
1357         if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1358             !(ia32_cap & ARCH_CAP_MDS_NO)) {
1359                 setup_force_cpu_bug(X86_BUG_MDS);
1360                 if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1361                         setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1362         }
1363
1364         if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1365                 setup_force_cpu_bug(X86_BUG_SWAPGS);
1366
1367         /*
1368          * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1369          *      - TSX is supported or
1370          *      - TSX_CTRL is present
1371          *
1372          * TSX_CTRL check is needed for cases when TSX could be disabled before
1373          * the kernel boot e.g. kexec.
1374          * TSX_CTRL check alone is not sufficient for cases when the microcode
1375          * update is not present or running as guest that don't get TSX_CTRL.
1376          */
1377         if (!(ia32_cap & ARCH_CAP_TAA_NO) &&
1378             (cpu_has(c, X86_FEATURE_RTM) ||
1379              (ia32_cap & ARCH_CAP_TSX_CTRL_MSR)))
1380                 setup_force_cpu_bug(X86_BUG_TAA);
1381
1382         /*
1383          * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1384          * in the vulnerability blacklist.
1385          *
1386          * Some of the implications and mitigation of Shared Buffers Data
1387          * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1388          * SRBDS.
1389          */
1390         if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1391              cpu_has(c, X86_FEATURE_RDSEED)) &&
1392             cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1393                     setup_force_cpu_bug(X86_BUG_SRBDS);
1394
1395         /*
1396          * Processor MMIO Stale Data bug enumeration
1397          *
1398          * Affected CPU list is generally enough to enumerate the vulnerability,
1399          * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1400          * not want the guest to enumerate the bug.
1401          *
1402          * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1403          * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1404          */
1405         if (!arch_cap_mmio_immune(ia32_cap)) {
1406                 if (cpu_matches(cpu_vuln_blacklist, MMIO))
1407                         setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1408                 else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1409                         setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1410         }
1411
1412         if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1413                 if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA))
1414                         setup_force_cpu_bug(X86_BUG_RETBLEED);
1415         }
1416
1417         if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1418                 setup_force_cpu_bug(X86_BUG_SMT_RSB);
1419
1420         if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1421                 if (cpu_matches(cpu_vuln_blacklist, SRSO))
1422                         setup_force_cpu_bug(X86_BUG_SRSO);
1423         }
1424
1425         /*
1426          * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1427          * an affected processor, the VMM may have disabled the use of GATHER by
1428          * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1429          * which means that AVX will be disabled.
1430          */
1431         if (cpu_matches(cpu_vuln_blacklist, GDS) && !(ia32_cap & ARCH_CAP_GDS_NO) &&
1432             boot_cpu_has(X86_FEATURE_AVX))
1433                 setup_force_cpu_bug(X86_BUG_GDS);
1434
1435         if (vulnerable_to_rfds(ia32_cap))
1436                 setup_force_cpu_bug(X86_BUG_RFDS);
1437
1438         if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1439                 return;
1440
1441         /* Rogue Data Cache Load? No! */
1442         if (ia32_cap & ARCH_CAP_RDCL_NO)
1443                 return;
1444
1445         setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1446
1447         if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1448                 return;
1449
1450         setup_force_cpu_bug(X86_BUG_L1TF);
1451 }
1452
1453 /*
1454  * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1455  * unfortunately, that's not true in practice because of early VIA
1456  * chips and (more importantly) broken virtualizers that are not easy
1457  * to detect. In the latter case it doesn't even *fail* reliably, so
1458  * probing for it doesn't even work. Disable it completely on 32-bit
1459  * unless we can find a reliable way to detect all the broken cases.
1460  * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1461  */
1462 static void detect_nopl(void)
1463 {
1464 #ifdef CONFIG_X86_32
1465         setup_clear_cpu_cap(X86_FEATURE_NOPL);
1466 #else
1467         setup_force_cpu_cap(X86_FEATURE_NOPL);
1468 #endif
1469 }
1470
1471 /*
1472  * We parse cpu parameters early because fpu__init_system() is executed
1473  * before parse_early_param().
1474  */
1475 static void __init cpu_parse_early_param(void)
1476 {
1477         char arg[128];
1478         char *argptr = arg, *opt;
1479         int arglen, taint = 0;
1480
1481 #ifdef CONFIG_X86_32
1482         if (cmdline_find_option_bool(boot_command_line, "no387"))
1483 #ifdef CONFIG_MATH_EMULATION
1484                 setup_clear_cpu_cap(X86_FEATURE_FPU);
1485 #else
1486                 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1487 #endif
1488
1489         if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1490                 setup_clear_cpu_cap(X86_FEATURE_FXSR);
1491 #endif
1492
1493         if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1494                 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1495
1496         if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1497                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1498
1499         if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1500                 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1501
1502         if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1503                 setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1504
1505         arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1506         if (arglen <= 0)
1507                 return;
1508
1509         pr_info("Clearing CPUID bits:");
1510
1511         while (argptr) {
1512                 bool found __maybe_unused = false;
1513                 unsigned int bit;
1514
1515                 opt = strsep(&argptr, ",");
1516
1517                 /*
1518                  * Handle naked numbers first for feature flags which don't
1519                  * have names.
1520                  */
1521                 if (!kstrtouint(opt, 10, &bit)) {
1522                         if (bit < NCAPINTS * 32) {
1523
1524                                 /* empty-string, i.e., ""-defined feature flags */
1525                                 if (!x86_cap_flags[bit])
1526                                         pr_cont(" " X86_CAP_FMT_NUM, x86_cap_flag_num(bit));
1527                                 else
1528                                         pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1529
1530                                 setup_clear_cpu_cap(bit);
1531                                 taint++;
1532                         }
1533                         /*
1534                          * The assumption is that there are no feature names with only
1535                          * numbers in the name thus go to the next argument.
1536                          */
1537                         continue;
1538                 }
1539
1540                 for (bit = 0; bit < 32 * NCAPINTS; bit++) {
1541                         if (!x86_cap_flag(bit))
1542                                 continue;
1543
1544                         if (strcmp(x86_cap_flag(bit), opt))
1545                                 continue;
1546
1547                         pr_cont(" %s", opt);
1548                         setup_clear_cpu_cap(bit);
1549                         taint++;
1550                         found = true;
1551                         break;
1552                 }
1553
1554                 if (!found)
1555                         pr_cont(" (unknown: %s)", opt);
1556         }
1557         pr_cont("\n");
1558
1559         if (taint)
1560                 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1561 }
1562
1563 /*
1564  * Do minimum CPU detection early.
1565  * Fields really needed: vendor, cpuid_level, family, model, mask,
1566  * cache alignment.
1567  * The others are not touched to avoid unwanted side effects.
1568  *
1569  * WARNING: this function is only called on the boot CPU.  Don't add code
1570  * here that is supposed to run on all CPUs.
1571  */
1572 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1573 {
1574         memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1575         c->extended_cpuid_level = 0;
1576
1577         if (!have_cpuid_p())
1578                 identify_cpu_without_cpuid(c);
1579
1580         /* cyrix could have cpuid enabled via c_identify()*/
1581         if (have_cpuid_p()) {
1582                 cpu_detect(c);
1583                 get_cpu_vendor(c);
1584                 get_cpu_cap(c);
1585                 setup_force_cpu_cap(X86_FEATURE_CPUID);
1586                 get_cpu_address_sizes(c);
1587                 cpu_parse_early_param();
1588
1589                 cpu_init_topology(c);
1590
1591                 if (this_cpu->c_early_init)
1592                         this_cpu->c_early_init(c);
1593
1594                 c->cpu_index = 0;
1595                 filter_cpuid_features(c, false);
1596
1597                 if (this_cpu->c_bsp_init)
1598                         this_cpu->c_bsp_init(c);
1599         } else {
1600                 setup_clear_cpu_cap(X86_FEATURE_CPUID);
1601                 get_cpu_address_sizes(c);
1602                 cpu_init_topology(c);
1603         }
1604
1605         setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1606
1607         cpu_set_bug_bits(c);
1608
1609         sld_setup(c);
1610
1611 #ifdef CONFIG_X86_32
1612         /*
1613          * Regardless of whether PCID is enumerated, the SDM says
1614          * that it can't be enabled in 32-bit mode.
1615          */
1616         setup_clear_cpu_cap(X86_FEATURE_PCID);
1617 #endif
1618
1619         /*
1620          * Later in the boot process pgtable_l5_enabled() relies on
1621          * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1622          * enabled by this point we need to clear the feature bit to avoid
1623          * false-positives at the later stage.
1624          *
1625          * pgtable_l5_enabled() can be false here for several reasons:
1626          *  - 5-level paging is disabled compile-time;
1627          *  - it's 32-bit kernel;
1628          *  - machine doesn't support 5-level paging;
1629          *  - user specified 'no5lvl' in kernel command line.
1630          */
1631         if (!pgtable_l5_enabled())
1632                 setup_clear_cpu_cap(X86_FEATURE_LA57);
1633
1634         detect_nopl();
1635 }
1636
1637 void __init early_cpu_init(void)
1638 {
1639         const struct cpu_dev *const *cdev;
1640         int count = 0;
1641
1642 #ifdef CONFIG_PROCESSOR_SELECT
1643         pr_info("KERNEL supported cpus:\n");
1644 #endif
1645
1646         for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1647                 const struct cpu_dev *cpudev = *cdev;
1648
1649                 if (count >= X86_VENDOR_NUM)
1650                         break;
1651                 cpu_devs[count] = cpudev;
1652                 count++;
1653
1654 #ifdef CONFIG_PROCESSOR_SELECT
1655                 {
1656                         unsigned int j;
1657
1658                         for (j = 0; j < 2; j++) {
1659                                 if (!cpudev->c_ident[j])
1660                                         continue;
1661                                 pr_info("  %s %s\n", cpudev->c_vendor,
1662                                         cpudev->c_ident[j]);
1663                         }
1664                 }
1665 #endif
1666         }
1667         early_identify_cpu(&boot_cpu_data);
1668 }
1669
1670 static bool detect_null_seg_behavior(void)
1671 {
1672         /*
1673          * Empirically, writing zero to a segment selector on AMD does
1674          * not clear the base, whereas writing zero to a segment
1675          * selector on Intel does clear the base.  Intel's behavior
1676          * allows slightly faster context switches in the common case
1677          * where GS is unused by the prev and next threads.
1678          *
1679          * Since neither vendor documents this anywhere that I can see,
1680          * detect it directly instead of hard-coding the choice by
1681          * vendor.
1682          *
1683          * I've designated AMD's behavior as the "bug" because it's
1684          * counterintuitive and less friendly.
1685          */
1686
1687         unsigned long old_base, tmp;
1688         rdmsrl(MSR_FS_BASE, old_base);
1689         wrmsrl(MSR_FS_BASE, 1);
1690         loadsegment(fs, 0);
1691         rdmsrl(MSR_FS_BASE, tmp);
1692         wrmsrl(MSR_FS_BASE, old_base);
1693         return tmp == 0;
1694 }
1695
1696 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1697 {
1698         /* BUG_NULL_SEG is only relevant with 64bit userspace */
1699         if (!IS_ENABLED(CONFIG_X86_64))
1700                 return;
1701
1702         if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
1703                 return;
1704
1705         /*
1706          * CPUID bit above wasn't set. If this kernel is still running
1707          * as a HV guest, then the HV has decided not to advertize
1708          * that CPUID bit for whatever reason.  For example, one
1709          * member of the migration pool might be vulnerable.  Which
1710          * means, the bug is present: set the BUG flag and return.
1711          */
1712         if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1713                 set_cpu_bug(c, X86_BUG_NULL_SEG);
1714                 return;
1715         }
1716
1717         /*
1718          * Zen2 CPUs also have this behaviour, but no CPUID bit.
1719          * 0x18 is the respective family for Hygon.
1720          */
1721         if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1722             detect_null_seg_behavior())
1723                 return;
1724
1725         /* All the remaining ones are affected */
1726         set_cpu_bug(c, X86_BUG_NULL_SEG);
1727 }
1728
1729 static void generic_identify(struct cpuinfo_x86 *c)
1730 {
1731         c->extended_cpuid_level = 0;
1732
1733         if (!have_cpuid_p())
1734                 identify_cpu_without_cpuid(c);
1735
1736         /* cyrix could have cpuid enabled via c_identify()*/
1737         if (!have_cpuid_p())
1738                 return;
1739
1740         cpu_detect(c);
1741
1742         get_cpu_vendor(c);
1743
1744         get_cpu_cap(c);
1745
1746         get_cpu_address_sizes(c);
1747
1748         get_model_name(c); /* Default name */
1749
1750         /*
1751          * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1752          * systems that run Linux at CPL > 0 may or may not have the
1753          * issue, but, even if they have the issue, there's absolutely
1754          * nothing we can do about it because we can't use the real IRET
1755          * instruction.
1756          *
1757          * NB: For the time being, only 32-bit kernels support
1758          * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1759          * whether to apply espfix using paravirt hooks.  If any
1760          * non-paravirt system ever shows up that does *not* have the
1761          * ESPFIX issue, we can change this.
1762          */
1763 #ifdef CONFIG_X86_32
1764         set_cpu_bug(c, X86_BUG_ESPFIX);
1765 #endif
1766 }
1767
1768 /*
1769  * This does the hard work of actually picking apart the CPU stuff...
1770  */
1771 static void identify_cpu(struct cpuinfo_x86 *c)
1772 {
1773         int i;
1774
1775         c->loops_per_jiffy = loops_per_jiffy;
1776         c->x86_cache_size = 0;
1777         c->x86_vendor = X86_VENDOR_UNKNOWN;
1778         c->x86_model = c->x86_stepping = 0;     /* So far unknown... */
1779         c->x86_vendor_id[0] = '\0'; /* Unset */
1780         c->x86_model_id[0] = '\0';  /* Unset */
1781 #ifdef CONFIG_X86_64
1782         c->x86_clflush_size = 64;
1783         c->x86_phys_bits = 36;
1784         c->x86_virt_bits = 48;
1785 #else
1786         c->cpuid_level = -1;    /* CPUID not detected */
1787         c->x86_clflush_size = 32;
1788         c->x86_phys_bits = 32;
1789         c->x86_virt_bits = 32;
1790 #endif
1791         c->x86_cache_alignment = c->x86_clflush_size;
1792         memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1793 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1794         memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1795 #endif
1796
1797         generic_identify(c);
1798
1799         cpu_parse_topology(c);
1800
1801         if (this_cpu->c_identify)
1802                 this_cpu->c_identify(c);
1803
1804         /* Clear/Set all flags overridden by options, after probe */
1805         apply_forced_caps(c);
1806
1807         /*
1808          * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1809          * Hygon will clear it in ->c_init() below.
1810          */
1811         set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1812
1813         /*
1814          * Vendor-specific initialization.  In this section we
1815          * canonicalize the feature flags, meaning if there are
1816          * features a certain CPU supports which CPUID doesn't
1817          * tell us, CPUID claiming incorrect flags, or other bugs,
1818          * we handle them here.
1819          *
1820          * At the end of this section, c->x86_capability better
1821          * indicate the features this CPU genuinely supports!
1822          */
1823         if (this_cpu->c_init)
1824                 this_cpu->c_init(c);
1825
1826         /* Disable the PN if appropriate */
1827         squash_the_stupid_serial_number(c);
1828
1829         /* Set up SMEP/SMAP/UMIP */
1830         setup_smep(c);
1831         setup_smap(c);
1832         setup_umip(c);
1833
1834         /* Enable FSGSBASE instructions if available. */
1835         if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1836                 cr4_set_bits(X86_CR4_FSGSBASE);
1837                 elf_hwcap2 |= HWCAP2_FSGSBASE;
1838         }
1839
1840         /*
1841          * The vendor-specific functions might have changed features.
1842          * Now we do "generic changes."
1843          */
1844
1845         /* Filter out anything that depends on CPUID levels we don't have */
1846         filter_cpuid_features(c, true);
1847
1848         /* If the model name is still unset, do table lookup. */
1849         if (!c->x86_model_id[0]) {
1850                 const char *p;
1851                 p = table_lookup_model(c);
1852                 if (p)
1853                         strcpy(c->x86_model_id, p);
1854                 else
1855                         /* Last resort... */
1856                         sprintf(c->x86_model_id, "%02x/%02x",
1857                                 c->x86, c->x86_model);
1858         }
1859
1860         x86_init_rdrand(c);
1861         setup_pku(c);
1862         setup_cet(c);
1863
1864         /*
1865          * Clear/Set all flags overridden by options, need do it
1866          * before following smp all cpus cap AND.
1867          */
1868         apply_forced_caps(c);
1869
1870         /*
1871          * On SMP, boot_cpu_data holds the common feature set between
1872          * all CPUs; so make sure that we indicate which features are
1873          * common between the CPUs.  The first time this routine gets
1874          * executed, c == &boot_cpu_data.
1875          */
1876         if (c != &boot_cpu_data) {
1877                 /* AND the already accumulated flags with these */
1878                 for (i = 0; i < NCAPINTS; i++)
1879                         boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1880
1881                 /* OR, i.e. replicate the bug flags */
1882                 for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1883                         c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1884         }
1885
1886         ppin_init(c);
1887
1888         /* Init Machine Check Exception if available. */
1889         mcheck_cpu_init(c);
1890
1891 #ifdef CONFIG_NUMA
1892         numa_add_cpu(smp_processor_id());
1893 #endif
1894 }
1895
1896 /*
1897  * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1898  * on 32-bit kernels:
1899  */
1900 #ifdef CONFIG_X86_32
1901 void enable_sep_cpu(void)
1902 {
1903         struct tss_struct *tss;
1904         int cpu;
1905
1906         if (!boot_cpu_has(X86_FEATURE_SEP))
1907                 return;
1908
1909         cpu = get_cpu();
1910         tss = &per_cpu(cpu_tss_rw, cpu);
1911
1912         /*
1913          * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1914          * see the big comment in struct x86_hw_tss's definition.
1915          */
1916
1917         tss->x86_tss.ss1 = __KERNEL_CS;
1918         wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1919         wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1920         wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1921
1922         put_cpu();
1923 }
1924 #endif
1925
1926 static __init void identify_boot_cpu(void)
1927 {
1928         identify_cpu(&boot_cpu_data);
1929         if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
1930                 pr_info("CET detected: Indirect Branch Tracking enabled\n");
1931 #ifdef CONFIG_X86_32
1932         enable_sep_cpu();
1933 #endif
1934         cpu_detect_tlb(&boot_cpu_data);
1935         setup_cr_pinning();
1936
1937         tsx_init();
1938         tdx_init();
1939         lkgs_init();
1940 }
1941
1942 void identify_secondary_cpu(struct cpuinfo_x86 *c)
1943 {
1944         BUG_ON(c == &boot_cpu_data);
1945         identify_cpu(c);
1946 #ifdef CONFIG_X86_32
1947         enable_sep_cpu();
1948 #endif
1949         x86_spec_ctrl_setup_ap();
1950         update_srbds_msr();
1951         if (boot_cpu_has_bug(X86_BUG_GDS))
1952                 update_gds_msr();
1953
1954         tsx_ap_init();
1955 }
1956
1957 void print_cpu_info(struct cpuinfo_x86 *c)
1958 {
1959         const char *vendor = NULL;
1960
1961         if (c->x86_vendor < X86_VENDOR_NUM) {
1962                 vendor = this_cpu->c_vendor;
1963         } else {
1964                 if (c->cpuid_level >= 0)
1965                         vendor = c->x86_vendor_id;
1966         }
1967
1968         if (vendor && !strstr(c->x86_model_id, vendor))
1969                 pr_cont("%s ", vendor);
1970
1971         if (c->x86_model_id[0])
1972                 pr_cont("%s", c->x86_model_id);
1973         else
1974                 pr_cont("%d86", c->x86);
1975
1976         pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1977
1978         if (c->x86_stepping || c->cpuid_level >= 0)
1979                 pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
1980         else
1981                 pr_cont(")\n");
1982 }
1983
1984 /*
1985  * clearcpuid= was already parsed in cpu_parse_early_param().  This dummy
1986  * function prevents it from becoming an environment variable for init.
1987  */
1988 static __init int setup_clearcpuid(char *arg)
1989 {
1990         return 1;
1991 }
1992 __setup("clearcpuid=", setup_clearcpuid);
1993
1994 DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
1995         .current_task   = &init_task,
1996         .preempt_count  = INIT_PREEMPT_COUNT,
1997         .top_of_stack   = TOP_OF_INIT_STACK,
1998 };
1999 EXPORT_PER_CPU_SYMBOL(pcpu_hot);
2000 EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
2001
2002 #ifdef CONFIG_X86_64
2003 DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
2004                      fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
2005 EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2006
2007 static void wrmsrl_cstar(unsigned long val)
2008 {
2009         /*
2010          * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2011          * is so far ignored by the CPU, but raises a #VE trap in a TDX
2012          * guest. Avoid the pointless write on all Intel CPUs.
2013          */
2014         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2015                 wrmsrl(MSR_CSTAR, val);
2016 }
2017
2018 static inline void idt_syscall_init(void)
2019 {
2020         wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2021
2022         if (ia32_enabled()) {
2023                 wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2024                 /*
2025                  * This only works on Intel CPUs.
2026                  * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2027                  * This does not cause SYSENTER to jump to the wrong location, because
2028                  * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2029                  */
2030                 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2031                 wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2032                             (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2033                 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2034         } else {
2035                 wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore);
2036                 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2037                 wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2038                 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2039         }
2040
2041         /*
2042          * Flags to clear on syscall; clear as much as possible
2043          * to minimize user space-kernel interference.
2044          */
2045         wrmsrl(MSR_SYSCALL_MASK,
2046                X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2047                X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2048                X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2049                X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2050                X86_EFLAGS_AC|X86_EFLAGS_ID);
2051 }
2052
2053 /* May not be marked __init: used by software suspend */
2054 void syscall_init(void)
2055 {
2056         /* The default user and kernel segments */
2057         wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2058
2059         /*
2060          * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
2061          * SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
2062          * entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
2063          * instruction to return to ring 3 (both sysexit and sysret cause
2064          * #UD when FRED is enabled).
2065          */
2066         if (!cpu_feature_enabled(X86_FEATURE_FRED))
2067                 idt_syscall_init();
2068 }
2069
2070 #else   /* CONFIG_X86_64 */
2071
2072 #ifdef CONFIG_STACKPROTECTOR
2073 DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2074 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2075 #endif
2076
2077 #endif  /* CONFIG_X86_64 */
2078
2079 /*
2080  * Clear all 6 debug registers:
2081  */
2082 static void clear_all_debug_regs(void)
2083 {
2084         int i;
2085
2086         for (i = 0; i < 8; i++) {
2087                 /* Ignore db4, db5 */
2088                 if ((i == 4) || (i == 5))
2089                         continue;
2090
2091                 set_debugreg(0, i);
2092         }
2093 }
2094
2095 #ifdef CONFIG_KGDB
2096 /*
2097  * Restore debug regs if using kgdbwait and you have a kernel debugger
2098  * connection established.
2099  */
2100 static void dbg_restore_debug_regs(void)
2101 {
2102         if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2103                 arch_kgdb_ops.correct_hw_break();
2104 }
2105 #else /* ! CONFIG_KGDB */
2106 #define dbg_restore_debug_regs()
2107 #endif /* ! CONFIG_KGDB */
2108
2109 static inline void setup_getcpu(int cpu)
2110 {
2111         unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2112         struct desc_struct d = { };
2113
2114         if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2115                 wrmsr(MSR_TSC_AUX, cpudata, 0);
2116
2117         /* Store CPU and node number in limit. */
2118         d.limit0 = cpudata;
2119         d.limit1 = cpudata >> 16;
2120
2121         d.type = 5;             /* RO data, expand down, accessed */
2122         d.dpl = 3;              /* Visible to user code */
2123         d.s = 1;                /* Not a system segment */
2124         d.p = 1;                /* Present */
2125         d.d = 1;                /* 32-bit */
2126
2127         write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2128 }
2129
2130 #ifdef CONFIG_X86_64
2131 static inline void tss_setup_ist(struct tss_struct *tss)
2132 {
2133         /* Set up the per-CPU TSS IST stacks */
2134         tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2135         tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2136         tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2137         tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2138         /* Only mapped when SEV-ES is active */
2139         tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2140 }
2141 #else /* CONFIG_X86_64 */
2142 static inline void tss_setup_ist(struct tss_struct *tss) { }
2143 #endif /* !CONFIG_X86_64 */
2144
2145 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2146 {
2147         tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2148
2149 #ifdef CONFIG_X86_IOPL_IOPERM
2150         tss->io_bitmap.prev_max = 0;
2151         tss->io_bitmap.prev_sequence = 0;
2152         memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2153         /*
2154          * Invalidate the extra array entry past the end of the all
2155          * permission bitmap as required by the hardware.
2156          */
2157         tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2158 #endif
2159 }
2160
2161 /*
2162  * Setup everything needed to handle exceptions from the IDT, including the IST
2163  * exceptions which use paranoid_entry().
2164  */
2165 void cpu_init_exception_handling(void)
2166 {
2167         struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2168         int cpu = raw_smp_processor_id();
2169
2170         /* paranoid_entry() gets the CPU number from the GDT */
2171         setup_getcpu(cpu);
2172
2173         /* For IDT mode, IST vectors need to be set in TSS. */
2174         if (!cpu_feature_enabled(X86_FEATURE_FRED))
2175                 tss_setup_ist(tss);
2176         tss_setup_io_bitmap(tss);
2177         set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2178
2179         load_TR_desc();
2180
2181         /* GHCB needs to be setup to handle #VC. */
2182         setup_ghcb();
2183
2184         if (cpu_feature_enabled(X86_FEATURE_FRED))
2185                 cpu_init_fred_exceptions();
2186         else
2187                 load_current_idt();
2188 }
2189
2190 /*
2191  * cpu_init() initializes state that is per-CPU. Some data is already
2192  * initialized (naturally) in the bootstrap process, such as the GDT.  We
2193  * reload it nevertheless, this function acts as a 'CPU state barrier',
2194  * nothing should get across.
2195  */
2196 void cpu_init(void)
2197 {
2198         struct task_struct *cur = current;
2199         int cpu = raw_smp_processor_id();
2200
2201 #ifdef CONFIG_NUMA
2202         if (this_cpu_read(numa_node) == 0 &&
2203             early_cpu_to_node(cpu) != NUMA_NO_NODE)
2204                 set_numa_node(early_cpu_to_node(cpu));
2205 #endif
2206         pr_debug("Initializing CPU#%d\n", cpu);
2207
2208         if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2209             boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2210                 cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2211
2212         if (IS_ENABLED(CONFIG_X86_64)) {
2213                 loadsegment(fs, 0);
2214                 memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2215                 syscall_init();
2216
2217                 wrmsrl(MSR_FS_BASE, 0);
2218                 wrmsrl(MSR_KERNEL_GS_BASE, 0);
2219                 barrier();
2220
2221                 x2apic_setup();
2222         }
2223
2224         mmgrab(&init_mm);
2225         cur->active_mm = &init_mm;
2226         BUG_ON(cur->mm);
2227         initialize_tlbstate_and_flush();
2228         enter_lazy_tlb(&init_mm, cur);
2229
2230         /*
2231          * sp0 points to the entry trampoline stack regardless of what task
2232          * is running.
2233          */
2234         load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2235
2236         load_mm_ldt(&init_mm);
2237
2238         clear_all_debug_regs();
2239         dbg_restore_debug_regs();
2240
2241         doublefault_init_cpu_tss();
2242
2243         if (is_uv_system())
2244                 uv_cpu_init();
2245
2246         load_fixmap_gdt(cpu);
2247 }
2248
2249 #ifdef CONFIG_MICROCODE_LATE_LOADING
2250 /**
2251  * store_cpu_caps() - Store a snapshot of CPU capabilities
2252  * @curr_info: Pointer where to store it
2253  *
2254  * Returns: None
2255  */
2256 void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2257 {
2258         /* Reload CPUID max function as it might've changed. */
2259         curr_info->cpuid_level = cpuid_eax(0);
2260
2261         /* Copy all capability leafs and pick up the synthetic ones. */
2262         memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2263                sizeof(curr_info->x86_capability));
2264
2265         /* Get the hardware CPUID leafs */
2266         get_cpu_cap(curr_info);
2267 }
2268
2269 /**
2270  * microcode_check() - Check if any CPU capabilities changed after an update.
2271  * @prev_info:  CPU capabilities stored before an update.
2272  *
2273  * The microcode loader calls this upon late microcode load to recheck features,
2274  * only when microcode has been updated. Caller holds and CPU hotplug lock.
2275  *
2276  * Return: None
2277  */
2278 void microcode_check(struct cpuinfo_x86 *prev_info)
2279 {
2280         struct cpuinfo_x86 curr_info;
2281
2282         perf_check_microcode();
2283
2284         amd_check_microcode();
2285
2286         store_cpu_caps(&curr_info);
2287
2288         if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2289                     sizeof(prev_info->x86_capability)))
2290                 return;
2291
2292         pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2293         pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2294 }
2295 #endif
2296
2297 /*
2298  * Invoked from core CPU hotplug code after hotplug operations
2299  */
2300 void arch_smt_update(void)
2301 {
2302         /* Handle the speculative execution misfeatures */
2303         cpu_bugs_smt_update();
2304         /* Check whether IPI broadcasting can be enabled */
2305         apic_smt_update();
2306 }
2307
2308 void __init arch_cpu_finalize_init(void)
2309 {
2310         identify_boot_cpu();
2311
2312         select_idle_routine();
2313
2314         /*
2315          * identify_boot_cpu() initialized SMT support information, let the
2316          * core code know.
2317          */
2318         cpu_smt_set_num_threads(__max_threads_per_core, __max_threads_per_core);
2319
2320         if (!IS_ENABLED(CONFIG_SMP)) {
2321                 pr_info("CPU: ");
2322                 print_cpu_info(&boot_cpu_data);
2323         }
2324
2325         cpu_select_mitigations();
2326
2327         arch_smt_update();
2328
2329         if (IS_ENABLED(CONFIG_X86_32)) {
2330                 /*
2331                  * Check whether this is a real i386 which is not longer
2332                  * supported and fixup the utsname.
2333                  */
2334                 if (boot_cpu_data.x86 < 4)
2335                         panic("Kernel requires i486+ for 'invlpg' and other features");
2336
2337                 init_utsname()->machine[1] =
2338                         '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2339         }
2340
2341         /*
2342          * Must be before alternatives because it might set or clear
2343          * feature bits.
2344          */
2345         fpu__init_system();
2346         fpu__init_cpu();
2347
2348         alternative_instructions();
2349
2350         if (IS_ENABLED(CONFIG_X86_64)) {
2351                 /*
2352                  * Make sure the first 2MB area is not mapped by huge pages
2353                  * There are typically fixed size MTRRs in there and overlapping
2354                  * MTRRs into large pages causes slow downs.
2355                  *
2356                  * Right now we don't do that with gbpages because there seems
2357                  * very little benefit for that case.
2358                  */
2359                 if (!direct_gbpages)
2360                         set_memory_4k((unsigned long)__va(0), 1);
2361         } else {
2362                 fpu__init_check_bugs();
2363         }
2364
2365         /*
2366          * This needs to be called before any devices perform DMA
2367          * operations that might use the SWIOTLB bounce buffers. It will
2368          * mark the bounce buffers as decrypted so that their usage will
2369          * not cause "plain-text" data to be decrypted when accessed. It
2370          * must be called after late_time_init() so that Hyper-V x86/x64
2371          * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2372          */
2373         mem_encrypt_init();
2374 }