| /* SPDX-License-Identifier: GPL-2.0 */ |
| |
| #include <linux/linkage.h> |
| #include <asm/segment.h> |
| #include <asm/boot.h> |
| #include <asm/msr.h> |
| #include <asm/processor-flags.h> |
| |
| /* |
| * This is the 32-bit trampoline that will be copied over to low memory. It |
| * will be called using the ordinary 64-bit calling convention from code |
| * running in 64-bit mode. |
| * |
| * Return address is at the top of the stack (might be above 4G). |
| * The first argument (EDI) contains the address of the temporary PGD level |
| * page table in 32-bit addressable memory which will be programmed into |
| * register CR3. |
| */ |
| |
| .section ".rodata", "a", @progbits |
| SYM_CODE_START(trampoline_32bit_src) |
| /* |
| * Preserve callee save 64-bit registers on the stack: this is |
| * necessary because the architecture does not guarantee that GPRs will |
| * retain their full 64-bit values across a 32-bit mode switch. |
| */ |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %rbp |
| pushq %rbx |
| |
| /* Preserve top half of RSP in a legacy mode GPR to avoid truncation */ |
| movq %rsp, %rbx |
| shrq $32, %rbx |
| |
| /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */ |
| pushq $__KERNEL32_CS |
| leaq 0f(%rip), %rax |
| pushq %rax |
| lretq |
| |
| /* |
| * The 32-bit code below will do a far jump back to long mode and end |
| * up here after reconfiguring the number of paging levels. First, the |
| * stack pointer needs to be restored to its full 64-bit value before |
| * the callee save register contents can be popped from the stack. |
| */ |
| .Lret: |
| shlq $32, %rbx |
| orq %rbx, %rsp |
| |
| /* Restore the preserved 64-bit registers */ |
| popq %rbx |
| popq %rbp |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| retq |
| |
| .code32 |
| 0: |
| /* Disable paging */ |
| movl %cr0, %eax |
| btrl $X86_CR0_PG_BIT, %eax |
| movl %eax, %cr0 |
| |
| /* Point CR3 to the trampoline's new top level page table */ |
| movl %edi, %cr3 |
| |
| /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */ |
| movl $MSR_EFER, %ecx |
| rdmsr |
| btsl $_EFER_LME, %eax |
| /* Avoid writing EFER if no change was made (for TDX guest) */ |
| jc 1f |
| wrmsr |
| 1: |
| /* Toggle CR4.LA57 */ |
| movl %cr4, %eax |
| btcl $X86_CR4_LA57_BIT, %eax |
| movl %eax, %cr4 |
| |
| /* Enable paging again. */ |
| movl %cr0, %eax |
| btsl $X86_CR0_PG_BIT, %eax |
| movl %eax, %cr0 |
| |
| /* |
| * Return to the 64-bit calling code using LJMP rather than LRET, to |
| * avoid the need for a 32-bit addressable stack. The destination |
| * address will be adjusted after the template code is copied into a |
| * 32-bit addressable buffer. |
| */ |
| .Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampoline_32bit_src) |
| SYM_CODE_END(trampoline_32bit_src) |
| |
| /* |
| * This symbol is placed right after trampoline_32bit_src() so its address can |
| * be used to infer the size of the trampoline code. |
| */ |
| SYM_DATA(trampoline_ljmp_imm_offset, .word .Ljmp + 1 - trampoline_32bit_src) |
| |
| /* |
| * The trampoline code has a size limit. |
| * Make sure we fail to compile if the trampoline code grows |
| * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes. |
| */ |
| .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE |
