x86/decompressor: Call trampoline directly from C code

Instead of returning to the asm calling code to invoke the trampoline,
call it straight from the C code that sets it up. That way, the struct
return type is no longer needed for returning two values, and the call
can be made conditional more cleanly in a subsequent patch.

This means that all callee save 64-bit registers need to be preserved
and restored, as their contents may not survive the legacy mode switch.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Link: https://lore.kernel.org/r/20230807162720.545787-13-ardb@kernel.org

diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 37fd7b7d683d696c7954b2dfaedc1486e95a39dc..cd6e3e175389aa6b77025ea1b0a8f1b31033f971 100644 (file)
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -430,25 +430,14 @@ SYM_CODE_START(startup_64)
 #endif
 
        /*
-        * paging_prepare() sets up the trampoline and checks if we need to
-        * enable 5-level paging.
-        *
-        * paging_prepare() returns a two-quadword structure which lands
-        * into RDX:RAX:
-        *   - Address of the trampoline is returned in RAX.
-        *   - Non zero RDX means trampoline needs to enable 5-level
-        *     paging.
+        * configure_5level_paging() updates the number of paging levels using
+        * a trampoline in 32-bit addressable memory if the current number does
+        * not match the desired number.
         *
         * Pass the boot_params pointer as the first argument.
         */
        movq    %r15, %rdi
-       call    paging_prepare
-
-       /* Pass the trampoline address and boolean flag as args #1 and #2 */
-       movq    %rax, %rdi
-       movq    %rdx, %rsi
-       leaq    TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax
-       call    *%rax
+       call    configure_5level_paging
 
        /*
         * cleanup_trampoline() would restore trampoline memory.
@@ -543,11 +532,14 @@ SYM_FUNC_END(.Lrelocated)
        .section ".rodata", "a", @progbits
 SYM_CODE_START(trampoline_32bit_src)
        /*
-        * Preserve live 64-bit registers on the stack: this is necessary
-        * because the architecture does not guarantee that GPRs will retain
-        * their full 64-bit values across a 32-bit mode switch.
+        * Preserve callee save 64-bit registers on the stack: this is
+        * necessary because the architecture does not guarantee that GPRs will
+        * retain their full 64-bit values across a 32-bit mode switch.
         */
        pushq   %r15
+       pushq   %r14
+       pushq   %r13
+       pushq   %r12
        pushq   %rbp
        pushq   %rbx
 
@@ -574,6 +566,9 @@ SYM_CODE_START(trampoline_32bit_src)
        /* Restore the preserved 64-bit registers */
        popq    %rbx
        popq    %rbp
+       popq    %r12
+       popq    %r13
+       popq    %r14
        popq    %r15
        retq
 

diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 5198a05aefa8d14ae00aea5b281a762abddf6d2f..f9cc86b2ee55ca808b93fbd0a34dfd591ef42bce 100644 (file)
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -16,11 +16,6 @@ unsigned int __section(".data") pgdir_shift = 39;
 unsigned int __section(".data") ptrs_per_p4d = 1;
 #endif
 
-struct paging_config {
-       unsigned long trampoline_start;
-       unsigned long l5_required;
-};
-
 /* Buffer to preserve trampoline memory */
 static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
 
@@ -29,7 +24,7 @@ static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
  * purposes.
  *
  * Avoid putting the pointer into .bss as it will be cleared between
- * paging_prepare() and extract_kernel().
+ * configure_5level_paging() and extract_kernel().
  */
 unsigned long *trampoline_32bit __section(".data");
 
@@ -106,13 +101,13 @@ static unsigned long find_trampoline_placement(void)
        return bios_start - TRAMPOLINE_32BIT_SIZE;
 }
 
-struct paging_config paging_prepare(void *rmode)
+asmlinkage void configure_5level_paging(struct boot_params *bp)
 {
-       struct paging_config paging_config = {};
-       void *tramp_code;
+       void (*toggle_la57)(void *trampoline, bool enable_5lvl);
+       bool l5_required = false;
 
        /* Initialize boot_params. Required for cmdline_find_option_bool(). */
-       boot_params = rmode;
+       boot_params = bp;
 
        /*
         * Check if LA57 is desired and supported.
@@ -130,7 +125,7 @@ struct paging_config paging_prepare(void *rmode)
                        !cmdline_find_option_bool("no5lvl") &&
                        native_cpuid_eax(0) >= 7 &&
                        (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
-               paging_config.l5_required = 1;
+               l5_required = true;
 
                /* Initialize variables for 5-level paging */
                __pgtable_l5_enabled = 1;
@@ -138,9 +133,7 @@ struct paging_config paging_prepare(void *rmode)
                ptrs_per_p4d = 512;
        }
 
-       paging_config.trampoline_start = find_trampoline_placement();
-
-       trampoline_32bit = (unsigned long *)paging_config.trampoline_start;
+       trampoline_32bit = (unsigned long *)find_trampoline_placement();
 
        /* Preserve trampoline memory */
        memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
@@ -149,7 +142,7 @@ struct paging_config paging_prepare(void *rmode)
        memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
 
        /* Copy trampoline code in place */
-       tramp_code = memcpy(trampoline_32bit +
+       toggle_la57 = memcpy(trampoline_32bit +
                        TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
                        &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
 
@@ -159,7 +152,8 @@ struct paging_config paging_prepare(void *rmode)
         * immediate absolute address, which needs to be adjusted based on the
         * placement of the trampoline.
         */
-       *(u32 *)(tramp_code + trampoline_ljmp_imm_offset) += (unsigned long)tramp_code;
+       *(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) +=
+                                               (unsigned long)toggle_la57;
 
        /*
         * The code below prepares page table in trampoline memory.
@@ -175,10 +169,10 @@ struct paging_config paging_prepare(void *rmode)
         * We are not going to use the page table in trampoline memory if we
         * are already in the desired paging mode.
         */
-       if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))
+       if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
                goto out;
 
-       if (paging_config.l5_required) {
+       if (l5_required) {
                /*
                 * For 4- to 5-level paging transition, set up current CR3 as
                 * the first and the only entry in a new top-level page table.
@@ -201,7 +195,7 @@ struct paging_config paging_prepare(void *rmode)
        }
 
 out:
-       return paging_config;
+       toggle_la57(trampoline_32bit, l5_required);
 }
 
 void cleanup_trampoline(void *pgtable)