arch/arm64/kernel/pi/idreg-override.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Early cpufeature override framework
   4  *
   5  * Copyright (C) 2020 Google LLC
   6  * Author: Marc Zyngier <maz@kernel.org>
   7  */
   8
   9 #include <linux/ctype.h>
  10 #include <linux/kernel.h>
  11 #include <linux/libfdt.h>
  12
  13 #include <asm/cacheflush.h>
  14 #include <asm/cpufeature.h>
  15 #include <asm/setup.h>
  16
  17 #include "pi.h"
  18
  19 #define FTR_DESC_NAME_LEN       20
  20 #define FTR_DESC_FIELD_LEN      10
  21 #define FTR_ALIAS_NAME_LEN      30
  22 #define FTR_ALIAS_OPTION_LEN    116
  23
  24 static u64 __boot_status __initdata;
  25
  26 typedef bool filter_t(u64 val);
  27
  28 struct ftr_set_desc {
  29         char                            name[FTR_DESC_NAME_LEN];
  30         PREL64(struct arm64_ftr_override, override);
  31         struct {
  32                 char                    name[FTR_DESC_FIELD_LEN];
  33                 u8                      shift;
  34                 u8                      width;
  35                 PREL64(filter_t,        filter);
  36         }                               fields[];
  37 };
  38
  39 #define FIELD(n, s, f)  { .name = n, .shift = s, .width = 4, .filter = f }
  40
  41 static bool __init mmfr1_vh_filter(u64 val)
  42 {
  43         /*
  44          * If we ever reach this point while running VHE, we're
  45          * guaranteed to be on one of these funky, VHE-stuck CPUs. If
  46          * the user was trying to force nVHE on us, proceed with
  47          * attitude adjustment.
  48          */
  49         return !(__boot_status == (BOOT_CPU_FLAG_E2H | BOOT_CPU_MODE_EL2) &&
  50                  val == 0);
  51 }
  52
  53 static const struct ftr_set_desc mmfr1 __prel64_initconst = {
  54         .name           = "id_aa64mmfr1",
  55         .override       = &id_aa64mmfr1_override,
  56         .fields         = {
  57                 FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter),
  58                 {}
  59         },
  60 };
  61
  62 static bool __init pfr0_sve_filter(u64 val)
  63 {
  64         /*
  65          * Disabling SVE also means disabling all the features that
  66          * are associated with it. The easiest way to do it is just to
  67          * override id_aa64zfr0_el1 to be 0.
  68          */
  69         if (!val) {
  70                 id_aa64zfr0_override.val = 0;
  71                 id_aa64zfr0_override.mask = GENMASK(63, 0);
  72         }
  73
  74         return true;
  75 }
  76
  77 static const struct ftr_set_desc pfr0 __prel64_initconst = {
  78         .name           = "id_aa64pfr0",
  79         .override       = &id_aa64pfr0_override,
  80         .fields         = {
  81                 FIELD("sve", ID_AA64PFR0_EL1_SVE_SHIFT, pfr0_sve_filter),
  82                 {}
  83         },
  84 };
  85
  86 static bool __init pfr1_sme_filter(u64 val)
  87 {
  88         /*
  89          * Similarly to SVE, disabling SME also means disabling all
  90          * the features that are associated with it. Just set
  91          * id_aa64smfr0_el1 to 0 and don't look back.
  92          */
  93         if (!val) {
  94                 id_aa64smfr0_override.val = 0;
  95                 id_aa64smfr0_override.mask = GENMASK(63, 0);
  96         }
  97
  98         return true;
  99 }
 100
 101 static const struct ftr_set_desc pfr1 __prel64_initconst = {
 102         .name           = "id_aa64pfr1",
 103         .override       = &id_aa64pfr1_override,
 104         .fields         = {
 105                 FIELD("bt", ID_AA64PFR1_EL1_BT_SHIFT, NULL ),
 106                 FIELD("mte", ID_AA64PFR1_EL1_MTE_SHIFT, NULL),
 107                 FIELD("sme", ID_AA64PFR1_EL1_SME_SHIFT, pfr1_sme_filter),
 108                 {}
 109         },
 110 };
 111
 112 static const struct ftr_set_desc isar1 __prel64_initconst = {
 113         .name           = "id_aa64isar1",
 114         .override       = &id_aa64isar1_override,
 115         .fields         = {
 116                 FIELD("gpi", ID_AA64ISAR1_EL1_GPI_SHIFT, NULL),
 117                 FIELD("gpa", ID_AA64ISAR1_EL1_GPA_SHIFT, NULL),
 118                 FIELD("api", ID_AA64ISAR1_EL1_API_SHIFT, NULL),
 119                 FIELD("apa", ID_AA64ISAR1_EL1_APA_SHIFT, NULL),
 120                 {}
 121         },
 122 };
 123
 124 static const struct ftr_set_desc isar2 __prel64_initconst = {
 125         .name           = "id_aa64isar2",
 126         .override       = &id_aa64isar2_override,
 127         .fields         = {
 128                 FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL),
 129                 FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL),
 130                 FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL),
 131                 {}
 132         },
 133 };
 134
 135 static const struct ftr_set_desc smfr0 __prel64_initconst = {
 136         .name           = "id_aa64smfr0",
 137         .override       = &id_aa64smfr0_override,
 138         .fields         = {
 139                 FIELD("smever", ID_AA64SMFR0_EL1_SMEver_SHIFT, NULL),
 140                 /* FA64 is a one bit field... :-/ */
 141                 { "fa64", ID_AA64SMFR0_EL1_FA64_SHIFT, 1, },
 142                 {}
 143         },
 144 };
 145
 146 static bool __init hvhe_filter(u64 val)
 147 {
 148         u64 mmfr1 = read_sysreg(id_aa64mmfr1_el1);
 149
 150         return (val == 1 &&
 151                 lower_32_bits(__boot_status) == BOOT_CPU_MODE_EL2 &&
 152                 cpuid_feature_extract_unsigned_field(mmfr1,
 153                                                      ID_AA64MMFR1_EL1_VH_SHIFT));
 154 }
 155
 156 static const struct ftr_set_desc sw_features __prel64_initconst = {
 157         .name           = "arm64_sw",
 158         .override       = &arm64_sw_feature_override,
 159         .fields         = {
 160                 FIELD("nokaslr", ARM64_SW_FEATURE_OVERRIDE_NOKASLR, NULL),
 161                 FIELD("hvhe", ARM64_SW_FEATURE_OVERRIDE_HVHE, hvhe_filter),
 162                 FIELD("rodataoff", ARM64_SW_FEATURE_OVERRIDE_RODATA_OFF, NULL),
 163                 {}
 164         },
 165 };
 166
 167 static const
 168 PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = {
 169         { &mmfr1        },
 170         { &pfr0         },
 171         { &pfr1         },
 172         { &isar1        },
 173         { &isar2        },
 174         { &smfr0        },
 175         { &sw_features  },
 176 };
 177
 178 static const struct {
 179         char    alias[FTR_ALIAS_NAME_LEN];
 180         char    feature[FTR_ALIAS_OPTION_LEN];
 181 } aliases[] __initconst = {
 182         { "kvm_arm.mode=nvhe",          "id_aa64mmfr1.vh=0" },
 183         { "kvm_arm.mode=protected",     "id_aa64mmfr1.vh=0" },
 184         { "arm64.nosve",                "id_aa64pfr0.sve=0" },
 185         { "arm64.nosme",                "id_aa64pfr1.sme=0" },
 186         { "arm64.nobti",                "id_aa64pfr1.bt=0" },
 187         { "arm64.nopauth",
 188           "id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 "
 189           "id_aa64isar1.api=0 id_aa64isar1.apa=0 "
 190           "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0"        },
 191         { "arm64.nomops",               "id_aa64isar2.mops=0" },
 192         { "arm64.nomte",                "id_aa64pfr1.mte=0" },
 193         { "nokaslr",                    "arm64_sw.nokaslr=1" },
 194         { "rodata=off",                 "arm64_sw.rodataoff=1" },
 195 };
 196
 197 static int __init parse_hexdigit(const char *p, u64 *v)
 198 {
 199         // skip "0x" if it comes next
 200         if (p[0] == '0' && tolower(p[1]) == 'x')
 201                 p += 2;
 202
 203         // check whether the RHS is a single hex digit
 204         if (!isxdigit(p[0]) || (p[1] && !isspace(p[1])))
 205                 return -EINVAL;
 206
 207         *v = tolower(*p) - (isdigit(*p) ? '0' : 'a' - 10);
 208         return 0;
 209 }
 210
 211 static int __init find_field(const char *cmdline, char *opt, int len,
 212                              const struct ftr_set_desc *reg, int f, u64 *v)
 213 {
 214         int flen = strlen(reg->fields[f].name);
 215
 216         // append '<fieldname>=' to obtain '<name>.<fieldname>='
 217         memcpy(opt + len, reg->fields[f].name, flen);
 218         len += flen;
 219         opt[len++] = '=';
 220
 221         if (memcmp(cmdline, opt, len))
 222                 return -1;
 223
 224         return parse_hexdigit(cmdline + len, v);
 225 }
 226
 227 static void __init match_options(const char *cmdline)
 228 {
 229         char opt[FTR_DESC_NAME_LEN + FTR_DESC_FIELD_LEN + 2];
 230         int i;
 231
 232         for (i = 0; i < ARRAY_SIZE(regs); i++) {
 233                 const struct ftr_set_desc *reg = prel64_pointer(regs[i].reg);
 234                 struct arm64_ftr_override *override;
 235                 int len = strlen(reg->name);
 236                 int f;
 237
 238                 override = prel64_pointer(reg->override);
 239
 240                 // set opt[] to '<name>.'
 241                 memcpy(opt, reg->name, len);
 242                 opt[len++] = '.';
 243
 244                 for (f = 0; reg->fields[f].name[0] != '\0'; f++) {
 245                         u64 shift = reg->fields[f].shift;
 246                         u64 width = reg->fields[f].width ?: 4;
 247                         u64 mask = GENMASK_ULL(shift + width - 1, shift);
 248                         bool (*filter)(u64 val);
 249                         u64 v;
 250
 251                         if (find_field(cmdline, opt, len, reg, f, &v))
 252                                 continue;
 253
 254                         /*
 255                          * If an override gets filtered out, advertise
 256                          * it by setting the value to the all-ones while
 257                          * clearing the mask... Yes, this is fragile.
 258                          */
 259                         filter = prel64_pointer(reg->fields[f].filter);
 260                         if (filter && !filter(v)) {
 261                                 override->val  |= mask;
 262                                 override->mask &= ~mask;
 263                                 continue;
 264                         }
 265
 266                         override->val  &= ~mask;
 267                         override->val  |= (v << shift) & mask;
 268                         override->mask |= mask;
 269
 270                         return;
 271                 }
 272         }
 273 }
 274
 275 static __init void __parse_cmdline(const char *cmdline, bool parse_aliases)
 276 {
 277         do {
 278                 char buf[256];
 279                 size_t len;
 280                 int i;
 281
 282                 cmdline = skip_spaces(cmdline);
 283
 284                 /* terminate on "--" appearing on the command line by itself */
 285                 if (cmdline[0] == '-' && cmdline[1] == '-' && isspace(cmdline[2]))
 286                         return;
 287
 288                 for (len = 0; cmdline[len] && !isspace(cmdline[len]); len++) {
 289                         if (len >= sizeof(buf) - 1)
 290                                 break;
 291                         if (cmdline[len] == '-')
 292                                 buf[len] = '_';
 293                         else
 294                                 buf[len] = cmdline[len];
 295                 }
 296                 if (!len)
 297                         return;
 298
 299                 buf[len] = 0;
 300
 301                 cmdline += len;
 302
 303                 match_options(buf);
 304
 305                 for (i = 0; parse_aliases && i < ARRAY_SIZE(aliases); i++)
 306                         if (!memcmp(buf, aliases[i].alias, len + 1))
 307                                 __parse_cmdline(aliases[i].feature, false);
 308         } while (1);
 309 }
 310
 311 static __init const u8 *get_bootargs_cmdline(const void *fdt, int node)
 312 {
 313         static char const bootargs[] __initconst = "bootargs";
 314         const u8 *prop;
 315
 316         if (node < 0)
 317                 return NULL;
 318
 319         prop = fdt_getprop(fdt, node, bootargs, NULL);
 320         if (!prop)
 321                 return NULL;
 322
 323         return strlen(prop) ? prop : NULL;
 324 }
 325
 326 static __init void parse_cmdline(const void *fdt, int chosen)
 327 {
 328         static char const cmdline[] __initconst = CONFIG_CMDLINE;
 329         const u8 *prop = get_bootargs_cmdline(fdt, chosen);
 330
 331         if (IS_ENABLED(CONFIG_CMDLINE_FORCE) || !prop)
 332                 __parse_cmdline(cmdline, true);
 333
 334         if (!IS_ENABLED(CONFIG_CMDLINE_FORCE) && prop)
 335                 __parse_cmdline(prop, true);
 336 }
 337
 338 void __init init_feature_override(u64 boot_status, const void *fdt,
 339                                   int chosen)
 340 {
 341         struct arm64_ftr_override *override;
 342         const struct ftr_set_desc *reg;
 343         int i;
 344
 345         for (i = 0; i < ARRAY_SIZE(regs); i++) {
 346                 reg = prel64_pointer(regs[i].reg);
 347                 override = prel64_pointer(reg->override);
 348
 349                 override->val  = 0;
 350                 override->mask = 0;
 351         }
 352
 353         __boot_status = boot_status;
 354
 355         parse_cmdline(fdt, chosen);
 356
 357         for (i = 0; i < ARRAY_SIZE(regs); i++) {
 358                 reg = prel64_pointer(regs[i].reg);
 359                 override = prel64_pointer(reg->override);
 360                 dcache_clean_inval_poc((unsigned long)override,
 361                                        (unsigned long)(override + 1));
 362         }
 363 }
 364
 365 char * __init skip_spaces(const char *str)
 366 {
 367         while (isspace(*str))
 368                 ++str;
 369         return (char *)str;
 370 }