arm64: mm: Add support for folding PUDs at runtime

In order to support LPA2 on 16k pages in a way that permits non-LPA2
systems to run the same kernel image, we have to be able to fall back to
at most 48 bits of virtual addressing.

Falling back to 48 bits would result in a level 0 with only 2 entries,
which is suboptimal in terms of TLB utilization. So instead, let's fall
back to 47 bits in that case. This means we need to be able to fold PUDs
dynamically, similar to how we fold P4Ds for 48 bit virtual addressing
on LPA2 with 4k pages.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-81-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index cae8c648f4628709520cb9797055555d59d11eac..aeba2cf15a253ee1b1ac832b377d027c1a86f333 100644 (file)
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -14,6 +14,7 @@
 #include <asm/tlbflush.h>
 
 #define __HAVE_ARCH_PGD_FREE
+#define __HAVE_ARCH_PUD_FREE
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE       (PTRS_PER_PGD * sizeof(pgd_t))
@@ -43,7 +44,8 @@ static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 
 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
 {
-       set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));
+       if (pgtable_l4_enabled())
+               set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));
 }
 
 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
@@ -53,6 +55,14 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
        p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN;
        __p4d_populate(p4dp, __pa(pudp), p4dval);
 }
+
+static inline void pud_free(struct mm_struct *mm, pud_t *pud)
+{
+       if (!pgtable_l4_enabled())
+               return;
+       BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
+       free_page((unsigned long)pud);
+}
 #else
 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
 {

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 3d7fb3cde83d3fb158b535c0122a39266c89151a..b3c716fa81212d321729a05fcd43f22804786828 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -759,12 +759,27 @@ static inline pmd_t *pud_pgtable(pud_t pud)
 
 #if CONFIG_PGTABLE_LEVELS > 3
 
+static __always_inline bool pgtable_l4_enabled(void)
+{
+       if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
+               return true;
+       if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
+               return vabits_actual == VA_BITS;
+       return alternative_has_cap_unlikely(ARM64_HAS_VA52);
+}
+
+static inline bool mm_pud_folded(const struct mm_struct *mm)
+{
+       return !pgtable_l4_enabled();
+}
+#define mm_pud_folded  mm_pud_folded
+
 #define pud_ERROR(e)   \
        pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
 
-#define p4d_none(p4d)          (!p4d_val(p4d))
-#define p4d_bad(p4d)           (!(p4d_val(p4d) & 2))
-#define p4d_present(p4d)       (p4d_val(p4d))
+#define p4d_none(p4d)          (pgtable_l4_enabled() && !p4d_val(p4d))
+#define p4d_bad(p4d)           (pgtable_l4_enabled() && !(p4d_val(p4d) & 2))
+#define p4d_present(p4d)       (!p4d_none(p4d))
 
 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 {
@@ -780,7 +795,8 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 
 static inline void p4d_clear(p4d_t *p4dp)
 {
-       set_p4d(p4dp, __p4d(0));
+       if (pgtable_l4_enabled())
+               set_p4d(p4dp, __p4d(0));
 }
 
 static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
@@ -788,25 +804,74 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
        return __p4d_to_phys(p4d);
 }
 
+#define pud_index(addr)                (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+
+static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
+{
+       return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
+}
+
 static inline pud_t *p4d_pgtable(p4d_t p4d)
 {
        return (pud_t *)__va(p4d_page_paddr(p4d));
 }
 
-/* Find an entry in the first-level page table. */
-#define pud_offset_phys(dir, addr)     (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
+static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
+{
+       BUG_ON(!pgtable_l4_enabled());
 
-#define pud_set_fixmap(addr)           ((pud_t *)set_fixmap_offset(FIX_PUD, addr))
-#define pud_set_fixmap_offset(p4d, addr)       pud_set_fixmap(pud_offset_phys(p4d, addr))
-#define pud_clear_fixmap()             clear_fixmap(FIX_PUD)
+       return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
+}
 
-#define p4d_page(p4d)          pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
+static inline
+pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr)
+{
+       if (!pgtable_l4_enabled())
+               return p4d_to_folded_pud(p4dp, addr);
+       return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr);
+}
+#define pud_offset_lockless pud_offset_lockless
+
+static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr)
+{
+       return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr);
+}
+#define pud_offset     pud_offset
+
+static inline pud_t *pud_set_fixmap(unsigned long addr)
+{
+       if (!pgtable_l4_enabled())
+               return NULL;
+       return (pud_t *)set_fixmap_offset(FIX_PUD, addr);
+}
+
+static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr)
+{
+       if (!pgtable_l4_enabled())
+               return p4d_to_folded_pud(p4dp, addr);
+       return pud_set_fixmap(pud_offset_phys(p4dp, addr));
+}
+
+static inline void pud_clear_fixmap(void)
+{
+       if (pgtable_l4_enabled())
+               clear_fixmap(FIX_PUD);
+}
 
 /* use ONLY for statically allocated translation tables */
-#define pud_offset_kimg(dir,addr)      ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
+static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr)
+{
+       if (!pgtable_l4_enabled())
+               return p4d_to_folded_pud(p4dp, addr);
+       return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr));
+}
+
+#define p4d_page(p4d)          pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
 
 #else
 
+static inline bool pgtable_l4_enabled(void) { return false; }
+
 #define p4d_page_paddr(p4d)    ({ BUILD_BUG(); 0;})
 
 /* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 0150deb332afd706650e60f571e4578d5a1898db..a947c6e784ed25981f0a2993fec66b6856de7015 100644 (file)
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -103,6 +103,9 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
 {
        struct ptdesc *ptdesc = virt_to_ptdesc(pudp);
 
+       if (!pgtable_l4_enabled())
+               return;
+
        pagetable_pud_dtor(ptdesc);
        tlb_remove_ptdesc(tlb, ptdesc);
 }

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index bc5e4e5698644fb9711c8053172cf0e283691f82..94f035f6c421653ef26ab5ff2a5bf71f9e71b82c 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1767,6 +1767,8 @@ static int __init __kpti_install_ng_mappings(void *__unused)
 
        if (levels == 5 && !pgtable_l5_enabled())
                levels = 4;
+       else if (levels == 4 && !pgtable_l4_enabled())
+               levels = 3;
 
        remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
 

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 8e5b3a7c5afd584ef15dd357e6b5917afc2922cd..b131ed31a6c8420c582b25c9eb844dadb3d2ce70 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1065,7 +1065,7 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
                free_empty_pmd_table(pudp, addr, next, floor, ceiling);
        } while (addr = next, addr < end);
 
-       if (CONFIG_PGTABLE_LEVELS <= 3)
+       if (!pgtable_l4_enabled())
                return;
 
        if (!pgtable_range_aligned(start, end, floor, ceiling, P4D_MASK))

diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 3c4f8a279d2bc76a28837b70f30ade70c3e0774a..0c501cabc23846c4c420c3fc04ac52c578ff3086 100644 (file)
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -21,6 +21,8 @@ static bool pgdir_is_page_size(void)
 {
        if (PGD_SIZE == PAGE_SIZE)
                return true;
+       if (CONFIG_PGTABLE_LEVELS == 4)
+               return !pgtable_l4_enabled();
        if (CONFIG_PGTABLE_LEVELS == 5)
                return !pgtable_l5_enabled();
        return false;