arm64: mm: Add LPA2 support to phys<->pte conversion routines

In preparation for enabling LPA2 support, introduce the mask values for
converting between physical addresses and their representations in a
page table descriptor.

While at it, move the pte_to_phys asm macro into its only user, so that
we can freely modify it to use its input value register as a temp
register.

For LPA2, the PTE_ADDR_MASK contains two non-adjacent sequences of zero
bits, which means it no longer fits into the immediate field of an
ordinary ALU instruction. So let's redefine it to include the bits in
between as well, and only use it when converting from physical address
to PTE representation, where the distinction does not matter. Also
update the name accordingly to emphasize this.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-75-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 68a99b116256ba9dd520891c4908fb46af32a283..7eedcb36ebe01dc6cb3f932e51f2a7157847e4fb 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -612,25 +612,13 @@ alternative_endif
 
        .macro  phys_to_pte, pte, phys
 #ifdef CONFIG_ARM64_PA_BITS_52
-       /*
-        * We assume \phys is 64K aligned and this is guaranteed by only
-        * supporting this configuration with 64K pages.
-        */
-       orr     \pte, \phys, \phys, lsr #36
-       and     \pte, \pte, #PTE_ADDR_MASK
+       orr     \pte, \phys, \phys, lsr #PTE_ADDR_HIGH_SHIFT
+       and     \pte, \pte, #PHYS_TO_PTE_ADDR_MASK
 #else
        mov     \pte, \phys
 #endif
        .endm
 
-       .macro  pte_to_phys, phys, pte
-       and     \phys, \pte, #PTE_ADDR_MASK
-#ifdef CONFIG_ARM64_PA_BITS_52
-       orr     \phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT
-       and     \phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT)
-#endif
-       .endm
-
 /*
  * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
  */

diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index b770f98fc0b5eb156f6c269e1002e5054403a500..4426f48f2ae038dbf33f8ce30a771c574a8a8cea 100644 (file)
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -155,13 +155,17 @@
 #define PTE_PXN                        (_AT(pteval_t, 1) << 53)        /* Privileged XN */
 #define PTE_UXN                        (_AT(pteval_t, 1) << 54)        /* User XN */
 
-#define PTE_ADDR_LOW           (((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
+#define PTE_ADDR_LOW           (((_AT(pteval_t, 1) << (50 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
 #ifdef CONFIG_ARM64_PA_BITS_52
+#ifdef CONFIG_ARM64_64K_PAGES
 #define PTE_ADDR_HIGH          (_AT(pteval_t, 0xf) << 12)
-#define PTE_ADDR_MASK          (PTE_ADDR_LOW | PTE_ADDR_HIGH)
 #define PTE_ADDR_HIGH_SHIFT    36
+#define PHYS_TO_PTE_ADDR_MASK  (PTE_ADDR_LOW | PTE_ADDR_HIGH)
 #else
-#define PTE_ADDR_MASK          PTE_ADDR_LOW
+#define PTE_ADDR_HIGH          (_AT(pteval_t, 0x3) << 8)
+#define PTE_ADDR_HIGH_SHIFT    42
+#define PHYS_TO_PTE_ADDR_MASK  GENMASK_ULL(49, 8)
+#endif
 #endif
 
 /*

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 522c21348ae81a01078e3d4fddfbe750f6d999b0..61de7b1516bc1b8bf3ff1ce7deb94a1b959075ab 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -80,15 +80,16 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 #ifdef CONFIG_ARM64_PA_BITS_52
 static inline phys_addr_t __pte_to_phys(pte_t pte)
 {
+       pte_val(pte) &= ~PTE_MAYBE_SHARED;
        return (pte_val(pte) & PTE_ADDR_LOW) |
                ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
 }
 static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
 {
-       return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK;
+       return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;
 }
 #else
-#define __pte_to_phys(pte)     (pte_val(pte) & PTE_ADDR_MASK)
+#define __pte_to_phys(pte)     (pte_val(pte) & PTE_ADDR_LOW)
 #define __phys_to_pte_val(phys)        (phys)
 #endif
 

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 7c46f8cfd6aef6212d45edbdc7df360e0fa83998..d03434b7bca5b994305dd1d85afaef94dc285321 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -205,6 +205,14 @@ SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1)
 
        .pushsection ".idmap.text", "a"
 
+       .macro  pte_to_phys, phys, pte
+       and     \phys, \pte, #PTE_ADDR_LOW
+#ifdef CONFIG_ARM64_PA_BITS_52
+       and     \pte, \pte, #PTE_ADDR_HIGH
+       orr     \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT
+#endif
+       .endm
+
        .macro  kpti_mk_tbl_ng, type, num_entries
        add     end_\type\()p, cur_\type\()p, #\num_entries * 8
 .Ldo_\type: