From 7ca57f2f9043000b7373c699c4223d73d1f63704 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Tue, 21 Apr 2020 12:44:49 -0500 Subject: [PATCH 001/113] crypto: ccp - Add support for SEV-ES to the PSP driver commit 97f9ac3db6612f14ac0c509e1a63ce14fd4cc0eb upstream. To provide support for SEV-ES, the hypervisor must provide an area of memory to the PSP. Once this Trusted Memory Region (TMR) is provided to the PSP, the contents of this area of memory are no longer available to the x86. Update the PSP driver to allocate a 1MB region for the TMR that is 1MB aligned and then provide it to the PSP through the SEV INIT command. Hygon-SIG: commit 97f9ac3db661 upstream crypto: ccp - Add support for SEV-ES to the PSP driver Signed-off-by: Tom Lendacky Reviewed-by: Brijesh Singh Reviewed-by: Joerg Roedel Signed-off-by: Herbert Xu [hanliyang: Fix conflict on drivers/crypto/ccp/sev-dev.c, include/linux/psp-sev.h, include/uapi/linux/psp-sev.h] Signed-off-by: hanliyang --- drivers/crypto/ccp/sev-dev.c | 43 ++++++++++++++++++++++++++++++++++++ include/linux/psp-sev.h | 2 ++ include/uapi/linux/psp-sev.h | 2 ++ 3 files changed, 47 insertions(+) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 99eb5d35fb66..ad87742b5136 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -51,6 +52,14 @@ MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ static bool psp_dead; static int psp_timeout; +/* Trusted Memory Region (TMR): + * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator + * to allocate the memory, which will return aligned memory for the specified + * allocation order. + */ +#define SEV_ES_TMR_SIZE (1024 * 1024) +static void *sev_es_tmr; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -236,6 +245,20 @@ static int __sev_platform_init_locked(int *error) if (sev->state == SEV_STATE_INIT) return 0; + if (sev_es_tmr) { + u64 tmr_pa; + + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + tmr_pa = __pa(sev_es_tmr); + + sev->init_cmd_buf.flags |= SEV_INIT_FLAGS_SEV_ES; + sev->init_cmd_buf.tmr_address = tmr_pa; + sev->init_cmd_buf.tmr_len = SEV_ES_TMR_SIZE; + } + rc = __sev_do_cmd_locked(SEV_CMD_INIT, &sev->init_cmd_buf, error); if (rc) return rc; @@ -1069,6 +1092,7 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); void sev_pci_init(void) { struct sev_device *sev = psp_master->sev_data; + struct page *tmr_page; int error, rc; if (!sev) @@ -1098,6 +1122,16 @@ void sev_pci_init(void) sev_update_firmware(sev->dev) == 0) sev_get_api_version(); + /* Obtain the TMR memory area for SEV-ES use */ + tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE)); + if (tmr_page) { + sev_es_tmr = page_address(tmr_page); + } else { + sev_es_tmr = NULL; + dev_warn(sev->dev, + "SEV: TMR allocation failed, SEV-ES support unavailable\n"); + } + /* Initialize the platform */ rc = sev_platform_init(&error); if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) { @@ -1136,4 +1170,13 @@ void sev_pci_exit(void) return; sev_platform_shutdown(NULL); + + if (sev_es_tmr) { + /* The TMR area was encrypted, flush it from the cache */ + wbinvd_on_all_cpus(); + + free_pages((unsigned long)sev_es_tmr, + get_order(SEV_ES_TMR_SIZE)); + sev_es_tmr = NULL; + } } diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index e226ecf702a0..5c73c8c914fb 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -102,6 +102,8 @@ struct sev_data_init { u32 tmr_len; /* In */ } __packed; +#define SEV_INIT_FLAGS_SEV_ES 0x01 + /** * struct sev_data_pek_csr - PEK_CSR command parameters * diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h index 0549a5c622bf..91b4c63d5cbf 100644 --- a/include/uapi/linux/psp-sev.h +++ b/include/uapi/linux/psp-sev.h @@ -83,6 +83,8 @@ struct sev_user_data_status { __u32 guest_count; /* Out */ } __packed; +#define SEV_STATUS_FLAGS_CONFIG_ES 0x0100 + /** * struct sev_user_data_pek_csr - PEK_CSR command parameters * -- Gitee From e2d2ad582ed887d06aadfb016765031dfc8dddad Mon Sep 17 00:00:00 2001 From: hanliyang Date: Tue, 24 May 2022 22:03:04 +0800 Subject: [PATCH 002/113] crypto: ccp: Fix definition of struct sev_data_send_update_vmsa Upstream: no of struct sev_data_send_update_vmsa The current definition of struct sev_data_send_update_vmsa in include/linux/psp-sev.h does not comply with SEV API spec. Fix it here. Hygon-SIG: commit none hygon crypto: ccp: Fix definition of struct sev_data_send_update_vmsa Signed-off-by: hanliyang Signed-off-by: Zhiguang Ni --- include/linux/psp-sev.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 5c73c8c914fb..3dad7eea8df6 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -390,6 +390,7 @@ struct sev_data_send_update_data { */ struct sev_data_send_update_vmsa { u32 handle; /* In */ + u32 reserved1; u64 hdr_address; /* In */ u32 hdr_len; /* In/Out */ u32 reserved2; -- Gitee From 41568b755ecea142d140883283cbacd6e02afb14 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 11 Oct 2019 13:50:45 +0200 Subject: [PATCH 003/113] x86/asm: Annotate local pseudo-functions commit ef77e6880be8fa3033544109e29417c8710fd3f2 upstream. Use the newly added SYM_CODE_START_LOCAL* to annotate beginnings of all pseudo-functions (those ending with END until now) which do not have ".globl" annotation. This is needed to balance END for tools that generate debuginfo. Note that ENDs are switched to SYM_CODE_END too so that everybody can see the pairing. C-like functions (which handle frame ptr etc.) are not annotated here, hence SYM_CODE_* macros are used here, not SYM_FUNC_*. Note that the 32bit version of early_idt_handler_common already had ENDPROC -- switch that to SYM_CODE_END for the same reason as above (and to be the same as 64bit). While early_idt_handler_common is LOCAL, it's name is not prepended with ".L" as it happens to appear in call traces. bad_get_user*, and bad_put_user are now aligned, as they are separate functions. They do not mind to be aligned -- no need to be compact there. early_idt_handler_common is aligned now too, as it is after early_idt_handler_array, so as well no need to be compact there. verify_cpu is self-standing and included in other .S files, so align it too. The others have alignment preserved to what it used to be (using the _NOALIGN variant of macros). Hygon-SIG: commit ef77e6880be8 upstream x86/asm: Annotate local pseudo-functions Signed-off-by: Jiri Slaby Signed-off-by: Borislav Petkov Cc: Alexios Zavras Cc: Allison Randal Cc: Andy Lutomirski Cc: Cao jin Cc: Enrico Weigelt Cc: Greg Kroah-Hartman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Josh Poimboeuf Cc: Juergen Gross Cc: linux-arch@vger.kernel.org Cc: Maran Wilson Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: x86-ml Link: https://lkml.kernel.org/r/20191011115108.12392-6-jslaby@suse.cz [hanliyang: Fix conflict on arch/x86/entry/entry_64.S, arch/x86/kernel/head_32.S, arch/x86/kernel/head_64.S, arch/x86/kernel/verify_cpu.S, arch/x86/lib/getuser.S and 1 more] Signed-off-by: hanliyang --- arch/x86/entry/entry_64.S | 3 ++- arch/x86/kernel/head_32.S | 4 ++-- arch/x86/kernel/head_64.S | 4 ++-- arch/x86/kernel/verify_cpu.S | 4 ++-- arch/x86/lib/getuser.S | 6 ++++-- arch/x86/lib/putuser.S | 3 ++- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 67cdca196aee..1bcb07485319 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1066,7 +1066,7 @@ EXPORT_SYMBOL(native_load_gs_index) _ASM_EXTABLE(.Lgs_change, .Lbad_gs) .section .fixup, "ax" /* running with kernelgs */ -.Lbad_gs: +SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) SWAPGS /* switch back to user gs */ .macro ZAP_GS /* This can't be a string because the preprocessor needs to see it. */ @@ -1077,6 +1077,7 @@ EXPORT_SYMBOL(native_load_gs_index) xorl %eax, %eax movl %eax, %gs jmp 2b +SYM_CODE_END(.Lbad_gs) .previous /* Call softirq on interrupt stack. Interrupts are off. */ diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 11a5d5ade52c..df46214a0b8c 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -410,7 +410,7 @@ ENTRY(early_idt_handler_array) .endr ENDPROC(early_idt_handler_array) -early_idt_handler_common: +SYM_CODE_START_LOCAL(early_idt_handler_common) /* * The stack is the hardware frame, an error code or zero, and the * vector number. @@ -461,7 +461,7 @@ early_idt_handler_common: decl %ss:early_recursion_flag addl $4, %esp /* pop pt_regs->orig_ax */ iret -ENDPROC(early_idt_handler_common) +SYM_CODE_END(early_idt_handler_common) /* This is the default interrupt "handler" :-) */ ENTRY(early_ignore_irq) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index f3d3e9646a99..1a02ef0e2d9f 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -291,7 +291,7 @@ ENTRY(early_idt_handler_array) UNWIND_HINT_IRET_REGS offset=16 END(early_idt_handler_array) -early_idt_handler_common: +SYM_CODE_START_LOCAL(early_idt_handler_common) /* * The stack is the hardware frame, an error code or zero, and the * vector number. @@ -333,7 +333,7 @@ early_idt_handler_common: 20: decl early_recursion_flag(%rip) jmp restore_regs_and_return_to_kernel -END(early_idt_handler_common) +SYM_CODE_END(early_idt_handler_common) __INITDATA diff --git a/arch/x86/kernel/verify_cpu.S b/arch/x86/kernel/verify_cpu.S index a024c4f7ba56..641f0fe1e5b4 100644 --- a/arch/x86/kernel/verify_cpu.S +++ b/arch/x86/kernel/verify_cpu.S @@ -31,7 +31,7 @@ #include #include -ENTRY(verify_cpu) +SYM_FUNC_START_LOCAL(verify_cpu) pushf # Save caller passed flags push $0 # Kill any dangerous flags popf @@ -137,4 +137,4 @@ ENTRY(verify_cpu) popf # Restore caller passed flags xorl %eax, %eax ret -ENDPROC(verify_cpu) +SYM_FUNC_END(verify_cpu) diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S index 9578eb88fc87..f9f59eb85635 100644 --- a/arch/x86/lib/getuser.S +++ b/arch/x86/lib/getuser.S @@ -115,21 +115,23 @@ ENDPROC(__get_user_8) EXPORT_SYMBOL(__get_user_8) -.Lbad_get_user_clac: +SYM_CODE_START_LOCAL(.Lbad_get_user_clac) ASM_CLAC bad_get_user: xor %edx,%edx mov $(-EFAULT),%_ASM_AX ret +SYM_CODE_END(.Lbad_get_user_clac) #ifdef CONFIG_X86_32 -.Lbad_get_user_8_clac: +SYM_CODE_START_LOCAL(.Lbad_get_user_8_clac) ASM_CLAC bad_get_user_8: xor %edx,%edx xor %ecx,%ecx mov $(-EFAULT),%_ASM_AX ret +SYM_CODE_END(.Lbad_get_user_8_clac) #endif _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac) diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S index 126dd6a9ec9b..a9391d772c81 100644 --- a/arch/x86/lib/putuser.S +++ b/arch/x86/lib/putuser.S @@ -91,11 +91,12 @@ ENTRY(__put_user_8) ENDPROC(__put_user_8) EXPORT_SYMBOL(__put_user_8) -.Lbad_put_user_clac: +SYM_CODE_START_LOCAL(.Lbad_put_user_clac) ASM_CLAC .Lbad_put_user: movl $-EFAULT,%eax RET +SYM_CODE_END(.Lbad_put_user_clac) _ASM_EXTABLE_UA(1b, .Lbad_put_user_clac) _ASM_EXTABLE_UA(2b, .Lbad_put_user_clac) -- Gitee From 735566712bc741cd6a168cd929ced51b87d0d0b5 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 11 Oct 2019 13:50:52 +0200 Subject: [PATCH 004/113] x86/boot: Annotate data appropriately commit b8c3f9b554e8bd0edb434a54d0dc48e3d9eb6edd upstream. Use the new SYM_DATA, SYM_DATA_START, and SYM_DATA_END* macros for data, so that the data in the object file look sane: Value Size Type Bind Vis Ndx Name 0000 10 OBJECT GLOBAL DEFAULT 3 efi32_boot_gdt 000a 10 OBJECT LOCAL DEFAULT 3 save_gdt 0014 8 OBJECT LOCAL DEFAULT 3 func_rt_ptr 001c 48 OBJECT GLOBAL DEFAULT 3 efi_gdt64 004c 0 OBJECT LOCAL DEFAULT 3 efi_gdt64_end 0000 48 OBJECT LOCAL DEFAULT 3 gdt 0030 0 OBJECT LOCAL DEFAULT 3 gdt_end 0030 8 OBJECT LOCAL DEFAULT 3 efi_config 0038 49 OBJECT GLOBAL DEFAULT 3 efi32_config 0069 49 OBJECT GLOBAL DEFAULT 3 efi64_config All have correct size and type now. Hygon-SIG: commit b8c3f9b554e8 upstream x86/boot: Annotate data appropriately Signed-off-by: Jiri Slaby Signed-off-by: Borislav Petkov Cc: Allison Randal Cc: Cao jin Cc: Enrico Weigelt Cc: Greg Kroah-Hartman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Kate Stewart Cc: "Kirill A. Shutemov" Cc: linux-arch@vger.kernel.org Cc: Thomas Gleixner Cc: Wei Huang Cc: x86-ml Cc: Xiaoyao Li Link: https://lkml.kernel.org/r/20191011115108.12392-13-jslaby@suse.cz [hanliyang: Fix conflict on arch/x86/boot/compressed/efi_thunk_64.S, arch/x86/boot/compressed/head_64.S, arch/x86/boot/compressed/mem_encrypt.S, arch/x86/realmode/rm/wakeup_asm.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/efi_thunk_64.S | 21 +++++++++------- arch/x86/boot/compressed/head_64.S | 32 ++++++++++++------------- arch/x86/boot/compressed/mem_encrypt.S | 3 +-- arch/x86/realmode/rm/wakeup_asm.S | 4 ++-- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/arch/x86/boot/compressed/efi_thunk_64.S b/arch/x86/boot/compressed/efi_thunk_64.S index bff9ab7c6317..d66000d23921 100644 --- a/arch/x86/boot/compressed/efi_thunk_64.S +++ b/arch/x86/boot/compressed/efi_thunk_64.S @@ -176,16 +176,19 @@ ENDPROC(efi_enter32) .data .balign 8 - .global efi32_boot_gdt -efi32_boot_gdt: .word 0 - .quad 0 +SYM_DATA_START(efi32_boot_gdt) + .word 0 + .quad 0 +SYM_DATA_END(efi32_boot_gdt) + +SYM_DATA_START_LOCAL(save_gdt) + .word 0 + .quad 0 +SYM_DATA_END(save_gdt) -save_gdt: .word 0 - .quad 0 -func_rt_ptr: .quad 0 +SYM_DATA_LOCAL(func_rt_ptr, .quad 0) - .global efi_gdt64 -efi_gdt64: +SYM_DATA_START(efi_gdt64) .word efi_gdt64_end - efi_gdt64 .long 0 /* Filled out by user */ .word 0 @@ -194,4 +197,4 @@ efi_gdt64: .quad 0x00cf92000000ffff /* __KERNEL_DS */ .quad 0x0080890000000000 /* TS descriptor */ .quad 0x0000000000000000 /* TS continued */ -efi_gdt64_end: +SYM_DATA_END_LABEL(efi_gdt64, SYM_L_LOCAL, efi_gdt64_end) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 50c9eeb36f0d..11b955218579 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -662,11 +662,12 @@ ENTRY(trampoline_32bit_src) #include "../../kernel/verify_cpu.S" .data -gdt64: +SYM_DATA_START_LOCAL(gdt64) .word gdt_end - gdt .quad 0 +SYM_DATA_END(gdt64) .balign 8 -gdt: +SYM_DATA_START_LOCAL(gdt) .word gdt_end - gdt .long gdt .word 0 @@ -675,25 +676,24 @@ gdt: .quad 0x00cf92000000ffff /* __KERNEL_DS */ .quad 0x0080890000000000 /* TS descriptor */ .quad 0x0000000000000000 /* TS continued */ -gdt_end: +SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end) #ifdef CONFIG_EFI_STUB -efi_config: - .quad 0 +SYM_DATA_LOCAL(efi_config, .quad 0) #ifdef CONFIG_EFI_MIXED - .global efi32_config -efi32_config: +SYM_DATA_START(efi32_config) .fill 5,8,0 .quad efi64_thunk .byte 0 +SYM_DATA_END(efi32_config) #endif - .global efi64_config -efi64_config: +SYM_DATA_START(efi64_config) .fill 5,8,0 .quad efi_call .byte 1 +SYM_DATA_END(efi64_config) #endif /* CONFIG_EFI_STUB */ /* @@ -701,23 +701,21 @@ efi64_config: */ .bss .balign 4 -boot_heap: - .fill BOOT_HEAP_SIZE, 1, 0 -boot_stack: +SYM_DATA_LOCAL(boot_heap, .fill BOOT_HEAP_SIZE, 1, 0) + +SYM_DATA_START_LOCAL(boot_stack) .fill BOOT_STACK_SIZE, 1, 0 -boot_stack_end: +SYM_DATA_END_LABEL(boot_stack, SYM_L_LOCAL, boot_stack_end) /* * Space for page tables (not in .bss so not zeroed) */ .section ".pgtable","a",@nobits .balign 4096 -pgtable: - .fill BOOT_PGT_SIZE, 1, 0 +SYM_DATA_LOCAL(pgtable, .fill BOOT_PGT_SIZE, 1, 0) /* * The page table is going to be used instead of page table in the trampoline * memory. */ -top_pgtable: - .fill PAGE_SIZE, 1, 0 +SYM_DATA_LOCAL(top_pgtable, .fill PAGE_SIZE, 1, 0) diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S index 6afb7130a387..28d703cad310 100644 --- a/arch/x86/boot/compressed/mem_encrypt.S +++ b/arch/x86/boot/compressed/mem_encrypt.S @@ -96,6 +96,5 @@ ENDPROC(set_sev_encryption_mask) #ifdef CONFIG_AMD_MEM_ENCRYPT .balign 8 -GLOBAL(sme_me_mask) - .quad 0 +SYM_DATA(sme_me_mask, .quad 0) #endif diff --git a/arch/x86/realmode/rm/wakeup_asm.S b/arch/x86/realmode/rm/wakeup_asm.S index 05ac9c17c811..2ddabef839bc 100644 --- a/arch/x86/realmode/rm/wakeup_asm.S +++ b/arch/x86/realmode/rm/wakeup_asm.S @@ -171,8 +171,8 @@ END(wakeup_gdt) /* This is the standard real-mode IDT */ .balign 16 -GLOBAL(wakeup_idt) +SYM_DATA_START(wakeup_idt) .word 0xffff /* limit */ .long 0 /* address */ .word 0 -END(wakeup_idt) +SYM_DATA_END(wakeup_idt) -- Gitee From 0cff6caab7b9beb989e4542905ba141f7f480054 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 11 Oct 2019 13:50:57 +0200 Subject: [PATCH 005/113] x86/asm: Use SYM_INNER_LABEL instead of GLOBAL commit 26ba4e5738a544aa17c462bfbe580e74071c810b upstream. The GLOBAL macro had several meanings and is going away. Convert all the inner function labels marked with GLOBAL to use SYM_INNER_LABEL instead. Hygon-SIG: commit 26ba4e5738a5 upstream x86/asm: Use SYM_INNER_LABEL instead of GLOBAL Signed-off-by: Jiri Slaby Signed-off-by: Borislav Petkov Cc: Andy Lutomirski Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Jiri Kosina Cc: Josh Poimboeuf Cc: linux-arch@vger.kernel.org Cc: Masami Hiramatsu Cc: Peter Zijlstra Cc: "Steven Rostedt (VMware)" Cc: Thomas Gleixner Cc: x86-ml Link: https://lkml.kernel.org/r/20191011115108.12392-18-jslaby@suse.cz [hanliyang: Fix conflict on arch/x86/entry/entry_64.S, arch/x86/entry/entry_64_compat.S, arch/x86/entry/vdso/vdso32/system_call.S, arch/x86/kernel/ftrace_32.S, arch/x86/kernel/ftrace_64.S and 1 more] Signed-off-by: hanliyang --- arch/x86/entry/entry_64.S | 6 +++--- arch/x86/entry/entry_64_compat.S | 4 ++-- arch/x86/entry/vdso/vdso32/system_call.S | 2 +- arch/x86/kernel/ftrace_32.S | 2 +- arch/x86/kernel/ftrace_64.S | 16 ++++++++-------- arch/x86/realmode/rm/reboot.S | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 1bcb07485319..7f27011f5461 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -163,7 +163,7 @@ ENTRY(entry_SYSCALL_64) pushq %r11 /* pt_regs->flags */ pushq $__USER_CS /* pt_regs->cs */ pushq %rcx /* pt_regs->ip */ -GLOBAL(entry_SYSCALL_64_after_hwframe) +SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) pushq %rax /* pt_regs->orig_ax */ PUSH_AND_CLEAR_REGS rax=$-ENOSYS @@ -624,7 +624,7 @@ GLOBAL(retint_user) call prepare_exit_to_usermode TRACE_IRQS_IRETQ -GLOBAL(swapgs_restore_regs_and_return_to_usermode) +SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) IBRS_EXIT #ifdef CONFIG_DEBUG_ENTRY /* Assert that pt_regs indicates user mode. */ @@ -684,7 +684,7 @@ retint_kernel: */ TRACE_IRQS_IRETQ -GLOBAL(restore_regs_and_return_to_kernel) +SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) #ifdef CONFIG_DEBUG_ENTRY /* Assert that pt_regs indicates kernel mode. */ testb $3, CS(%rsp) diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S index c3c4ea4a6711..a74deba50bec 100644 --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -149,7 +149,7 @@ ENTRY(entry_SYSENTER_compat) pushq $X86_EFLAGS_FIXED popfq jmp .Lsysenter_flags_fixed -GLOBAL(__end_entry_SYSENTER_compat) +SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) ENDPROC(entry_SYSENTER_compat) /* @@ -218,7 +218,7 @@ ENTRY(entry_SYSCALL_compat) pushq %r11 /* pt_regs->flags */ pushq $__USER32_CS /* pt_regs->cs */ pushq %rcx /* pt_regs->ip */ -GLOBAL(entry_SYSCALL_compat_after_hwframe) +SYM_INNER_LABEL(entry_SYSCALL_compat_after_hwframe, SYM_L_GLOBAL) movl %eax, %eax /* discard orig_ax high bits */ pushq %rax /* pt_regs->orig_ax */ pushq %rdi /* pt_regs->di */ diff --git a/arch/x86/entry/vdso/vdso32/system_call.S b/arch/x86/entry/vdso/vdso32/system_call.S index 263d7433dea8..de1fff7188aa 100644 --- a/arch/x86/entry/vdso/vdso32/system_call.S +++ b/arch/x86/entry/vdso/vdso32/system_call.S @@ -62,7 +62,7 @@ __kernel_vsyscall: /* Enter using int $0x80 */ int $0x80 -GLOBAL(int80_landing_pad) +SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL) /* * Restore EDX and ECX in case they were clobbered. EBP is not diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S index 2cc0303522c9..083624bd0f6e 100644 --- a/arch/x86/kernel/ftrace_32.S +++ b/arch/x86/kernel/ftrace_32.S @@ -138,7 +138,7 @@ SYM_CODE_START(ftrace_regs_caller) movl function_trace_op, %ecx # 3rd argument: ftrace_pos pushl %esp # 4th argument: pt_regs -GLOBAL(ftrace_regs_call) +SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) call ftrace_stub addl $4, %esp # skip 4th argument diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 809d54397dba..3afaaf555637 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -140,14 +140,14 @@ ENTRY(ftrace_caller) /* save_mcount_regs fills in first two parameters */ save_mcount_regs -GLOBAL(ftrace_caller_op_ptr) +SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx /* regs go into 4th parameter (but make it NULL) */ movq $0, %rcx -GLOBAL(ftrace_call) +SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) call ftrace_stub restore_mcount_regs @@ -157,10 +157,10 @@ GLOBAL(ftrace_call) * think twice before adding any new code or changing the * layout here. */ -GLOBAL(ftrace_epilogue) +SYM_INNER_LABEL(ftrace_epilogue, SYM_L_GLOBAL) #ifdef CONFIG_FUNCTION_GRAPH_TRACER -GLOBAL(ftrace_graph_call) +SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) jmp ftrace_stub #endif @@ -180,7 +180,7 @@ ENTRY(ftrace_regs_caller) save_mcount_regs 8 /* save_mcount_regs fills in first two parameters */ -GLOBAL(ftrace_regs_caller_op_ptr) +SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx @@ -209,7 +209,7 @@ GLOBAL(ftrace_regs_caller_op_ptr) /* regs go into 4th parameter */ leaq (%rsp), %rcx -GLOBAL(ftrace_regs_call) +SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) call ftrace_stub /* Copy flags back to SS, to restore them */ @@ -239,7 +239,7 @@ GLOBAL(ftrace_regs_call) * The trampoline will add the code to jump * to the return. */ -GLOBAL(ftrace_regs_caller_end) +SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) jmp ftrace_epilogue @@ -261,7 +261,7 @@ fgraph_trace: jnz ftrace_graph_caller #endif -GLOBAL(ftrace_stub) +SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) retq trace: diff --git a/arch/x86/realmode/rm/reboot.S b/arch/x86/realmode/rm/reboot.S index cd2f97b9623b..f91425a01f8f 100644 --- a/arch/x86/realmode/rm/reboot.S +++ b/arch/x86/realmode/rm/reboot.S @@ -33,7 +33,7 @@ ENTRY(machine_real_restart_asm) movl %eax, %cr0 ljmpl $__KERNEL32_CS, $pa_machine_real_restart_paging_off -GLOBAL(machine_real_restart_paging_off) +SYM_INNER_LABEL(machine_real_restart_paging_off, SYM_L_GLOBAL) xorl %eax, %eax xorl %edx, %edx movl $MSR_EFER, %ecx -- Gitee From 935f92b8ace271743008df1dca9500c34bae5028 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Tue, 7 Jan 2020 14:44:35 -0500 Subject: [PATCH 006/113] x86/boot/compressed/64: Use LEA to initialize boot stack pointer commit 48bfdb9deffdc6b683feb25e15f4f26aac503501 upstream. It's shorter, and it's what is used in every other place, so make it consistent. Hygon-SIG: commit 48bfdb9deffd upstream x86/boot/compressed/64: Use LEA to initialize boot stack pointer Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200107194436.2166846-2-nivedita@alum.mit.edu Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 11b955218579..178ca6a14097 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -82,9 +82,7 @@ ENTRY(startup_32) subl $1b, %ebp /* setup a stack and make sure cpu supports long mode. */ - movl $boot_stack_end, %eax - addl %ebp, %eax - movl %eax, %esp + leal boot_stack_end(%ebp), %esp call verify_cpu testl %eax, %eax -- Gitee From f8e618c6e29813cdf82186556bc25fc8d0daa10c Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Tue, 11 Feb 2020 12:33:33 -0500 Subject: [PATCH 007/113] x86/boot/compressed/64: Use 32-bit (zero-extended) MOV for z_output_len commit a86255fe5258714e1f7c1bdfe95f08e4d098d450 upstream. z_output_len is the size of the decompressed payload (i.e. vmlinux + vmlinux.relocs) and is generated as an unsigned 32-bit quantity by mkpiggy.c. The current movq $z_output_len, %r9 instruction generates a sign-extended move to %r9. Using movl $z_output_len, %r9d will instead zero-extend into %r9, which is appropriate for an unsigned 32-bit quantity. This is also what is already done for z_input_len, the size of the compressed payload. [ bp: Also, z_output_len cannot be a 64-bit quantity because it participates in: init_size: .long INIT_SIZE # kernel initialization size through INIT_SIZE which is a 32-bit quantity determined by the .long directive (vs .quad for 64-bit). Furthermore, if it really must be a 64-bit quantity, then the insn must be MOVABS which can accommodate a 64-bit immediate and which the toolchain does not generate automatically. ] Hygon-SIG: commit a86255fe5258 upstream x86/boot/compressed/64: Use 32-bit (zero-extended) MOV for z_output_len Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200211173333.1722739-1-nivedita@alum.mit.edu Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 178ca6a14097..c796c012f177 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -536,7 +536,7 @@ ENDPROC(efi64_stub_entry) leaq input_data(%rip), %rdx /* input_data */ movl $z_input_len, %ecx /* input_len */ movq %rbp, %r8 /* output target address */ - movq $z_output_len, %r9 /* decompressed length, end of relocs */ + movl $z_output_len, %r9d /* decompressed length, end of relocs */ call extract_kernel /* returns kernel location in %rax */ popq %rsi -- Gitee From 25a0b1ce043c36ae1e7b391f422b4e71fba98da7 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 2 Feb 2020 12:13:47 -0500 Subject: [PATCH 008/113] x86/boot: Remove KEEP_SEGMENTS support commit 67a6af7ad1d161dbc9c139e868d5549e632923f7 upstream. Commit a24e785111a3 ("i386: paravirt boot sequence") added this flag for use by paravirtualized environments such as Xen. However, Xen never made use of this flag [1], and it was only ever used by lguest [2]. Commit ecda85e70277 ("x86/lguest: Remove lguest support") removed lguest, so KEEP_SEGMENTS has lost its last user. [1] https://lore.kernel.org/lkml/4D4B097C.5050405@goop.org [2] https://www.mail-archive.com/lguest@lists.ozlabs.org/msg00469.html Hygon-SIG: commit 67a6af7ad1d1 upstream x86/boot: Remove KEEP_SEGMENTS support Signed-off-by: Arvind Sankar Link: https://lore.kernel.org/r/20200202171353.3736319-2-nivedita@alum.mit.edu Signed-off-by: Ard Biesheuvel [hanliyang: Fix conflict on Documentation/x86/boot.rst, arch/x86/boot/compressed/head_32.S, arch/x86/boot/compressed/head_64.S, arch/x86/kernel/head_32.S] Signed-off-by: hanliyang --- Documentation/x86/boot.rst | 8 ++------ arch/x86/boot/compressed/head_32.S | 8 -------- arch/x86/boot/compressed/head_64.S | 8 -------- arch/x86/kernel/head_32.S | 6 ------ 4 files changed, 2 insertions(+), 28 deletions(-) diff --git a/Documentation/x86/boot.rst b/Documentation/x86/boot.rst index 08a2f100c0e6..7cde724e9e42 100644 --- a/Documentation/x86/boot.rst +++ b/Documentation/x86/boot.rst @@ -471,15 +471,11 @@ Protocol: 2.00+ kernel) to not write early messages that require accessing the display hardware directly. - Bit 6 (write): KEEP_SEGMENTS + Bit 6 (obsolete): KEEP_SEGMENTS Protocol: 2.07+ - - If 0, reload the segment registers in the 32bit entry point. - - If 1, do not reload the segment registers in the 32bit entry point. - - Assume that %cs %ds %ss %es are all set to flat segments with - a base of 0 (or the equivalent for their environment). + - This flag is obsolete. Bit 7 (write): CAN_USE_HEAP diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index d7c0fcc1dbf9..2fee6bb09282 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -64,13 +64,6 @@ __HEAD ENTRY(startup_32) cld - /* - * Test KEEP_SEGMENTS flag to see if the bootloader is asking - * us to not reload segments - */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 1f - cli movl $__BOOT_DS, %eax movl %eax, %ds @@ -78,7 +71,6 @@ ENTRY(startup_32) movl %eax, %fs movl %eax, %gs movl %eax, %ss -1: /* * Calculate the delta between where we were compiled to run diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index c796c012f177..a39fd3cce52f 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -54,19 +54,11 @@ ENTRY(startup_32) * all need to be under the 4G limit. */ cld - /* - * Test KEEP_SEGMENTS flag to see if the bootloader is asking - * us to not reload segments - */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 1f - cli movl $(__BOOT_DS), %eax movl %eax, %ds movl %eax, %es movl %eax, %ss -1: /* * Calculate the delta between where we were compiled to run diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index df46214a0b8c..417319dabe9b 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -67,11 +67,6 @@ __HEAD SYM_CODE_START(startup_32) movl pa(initial_stack),%ecx - /* test KEEP_SEGMENTS flag to see if the bootloader is asking - us to not reload segments */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 2f - /* * Set segments to known values. */ @@ -82,7 +77,6 @@ SYM_CODE_START(startup_32) movl %eax,%fs movl %eax,%gs movl %eax,%ss -2: leal -__PAGE_OFFSET(%ecx),%esp /* -- Gitee From 73642d27fa5e85e319190f592b8b082e17444fbf Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 2 Feb 2020 12:13:48 -0500 Subject: [PATCH 009/113] efi/x86: Don't depend on firmware GDT layout commit 90ff226281e1083988a42cfc51f89d91734cc55e upstream. When booting in mixed mode, the firmware's GDT is still installed at handover entry in efi32_stub_entry. We save the GDTR for later use in __efi64_thunk but we are assuming that descriptor 2 (__KERNEL_CS) is a valid 32-bit code segment descriptor and that descriptor 3 (__KERNEL_DS/__BOOT_DS) is a valid data segment descriptor. This happens to be true for OVMF (it actually uses descriptor 1 for data segments, but descriptor 3 is also setup as data), but we shouldn't depend on this being the case. Fix this by saving the code and data selectors in addition to the GDTR in efi32_stub_entry, and restoring them in __efi64_thunk before calling the firmware. The UEFI specification guarantees that selectors will be flat, so using the DS selector for all the segment registers should be enough. We also need to install our own GDT before initializing segment registers in startup_32, so move the GDT load up to the beginning of the function. [ardb: mention mixed mode in the commit log] Hygon-SIG: commit 90ff226281e1 upstream efi/x86: Don't depend on firmware GDT layout Signed-off-by: Arvind Sankar Link: https://lore.kernel.org/r/20200202171353.3736319-3-nivedita@alum.mit.edu Signed-off-by: Ard Biesheuvel [hanliyang: Fix conflict on arch/x86/boot/compressed/efi_thunk_64.S, arch/x86/boot/compressed/head_64.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/efi_thunk_64.S | 29 ++++++++++++++++++++----- arch/x86/boot/compressed/head_64.S | 26 ++++++++++++++-------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/arch/x86/boot/compressed/efi_thunk_64.S b/arch/x86/boot/compressed/efi_thunk_64.S index d66000d23921..6679b905074a 100644 --- a/arch/x86/boot/compressed/efi_thunk_64.S +++ b/arch/x86/boot/compressed/efi_thunk_64.S @@ -62,11 +62,16 @@ ENTRY(efi64_thunk) * Switch to gdt with 32-bit segments. This is the firmware GDT * that was installed when the kernel started executing. This * pointer was saved at the EFI stub entry point in head_64.S. + * + * Pass the saved DS selector to the 32-bit code, and use far return to + * restore the saved CS selector. */ leaq efi32_boot_gdt(%rip), %rax lgdt (%rax) - pushq $__KERNEL_CS + movzwl efi32_boot_ds(%rip), %edx + movzwq efi32_boot_cs(%rip), %rax + pushq %rax leaq efi_enter32(%rip), %rax pushq %rax lretq @@ -81,6 +86,10 @@ ENTRY(efi64_thunk) movl %ebx, %es pop %rbx movl %ebx, %ds + /* Clear out 32-bit selector from FS and GS */ + xorl %ebx, %ebx + movl %ebx, %fs + movl %ebx, %gs /* * Convert 32-bit status code into 64-bit. @@ -113,10 +122,12 @@ ENDPROC(efi_exit32) * The stack should represent the 32-bit calling convention. */ ENTRY(efi_enter32) - movl $__KERNEL_DS, %eax - movl %eax, %ds - movl %eax, %es - movl %eax, %ss + /* Load firmware selector into data and stack segment registers */ + movl %edx, %ds + movl %edx, %es + movl %edx, %fs + movl %edx, %gs + movl %edx, %ss /* Reload pgtables */ movl %cr3, %eax @@ -181,6 +192,14 @@ SYM_DATA_START(efi32_boot_gdt) .quad 0 SYM_DATA_END(efi32_boot_gdt) +SYM_DATA_START(efi32_boot_cs) + .word 0 +SYM_DATA_END(efi32_boot_cs) + +SYM_DATA_START(efi32_boot_ds) + .word 0 +SYM_DATA_END(efi32_boot_ds) + SYM_DATA_START_LOCAL(save_gdt) .word 0 .quad 0 diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index a39fd3cce52f..b98a80c77274 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -55,10 +55,6 @@ ENTRY(startup_32) */ cld cli - movl $(__BOOT_DS), %eax - movl %eax, %ds - movl %eax, %es - movl %eax, %ss /* * Calculate the delta between where we were compiled to run @@ -73,6 +69,18 @@ ENTRY(startup_32) 1: popl %ebp subl $1b, %ebp + /* Load new GDT with the 64bit segments using 32bit descriptor */ + addl %ebp, gdt+2(%ebp) + lgdt gdt(%ebp) + + /* Load segment registers with our descriptors */ + movl $__BOOT_DS, %eax + movl %eax, %ds + movl %eax, %es + movl %eax, %fs + movl %eax, %gs + movl %eax, %ss + /* setup a stack and make sure cpu supports long mode. */ leal boot_stack_end(%ebp), %esp @@ -111,10 +119,6 @@ ENTRY(startup_32) * Prepare for entering 64 bit mode */ - /* Load new GDT with the 64bit segments using 32bit descriptor */ - addl %ebp, gdt+2(%ebp) - lgdt gdt(%ebp) - /* Enable PAE mode */ movl %cr4, %eax orl $X86_CR4_PAE, %eax @@ -230,11 +234,15 @@ ENTRY(efi32_stub_entry) movl %ecx, efi32_config(%ebp) movl %edx, efi32_config+8(%ebp) - sgdtl efi32_boot_gdt(%ebp) leal efi32_config(%ebp), %eax movl %eax, efi_config(%ebp) + /* Save firmware code/data selectors */ + sgdtl efi32_boot_gdt(%ebp) + movw %cs, efi32_boot_cs(%ebp) + movw %ds, efi32_boot_ds(%ebp) + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax -- Gitee From f6da08c3d656e6c687f6905f0927b2cb47930552 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 2 Feb 2020 12:13:49 -0500 Subject: [PATCH 010/113] x86/boot: Reload GDTR after copying to the end of the buffer commit 32d009137a5646947d450da6fa641a1f4dc1e42c upstream. The GDT may get overwritten during the copy or during extract_kernel, which will cause problems if any segment register is touched before the GDTR is reloaded by the decompressed kernel. For safety update the GDTR to point to the GDT within the copied kernel. Hygon-SIG: commit 32d009137a56 upstream x86/boot: Reload GDTR after copying to the end of the buffer Signed-off-by: Arvind Sankar Link: https://lore.kernel.org/r/20200202171353.3736319-4-nivedita@alum.mit.edu Signed-off-by: Ard Biesheuvel Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index b98a80c77274..7f0786c36956 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -441,6 +441,16 @@ trampoline_return: cld popq %rsi + /* + * The GDT may get overwritten either during the copy we just did or + * during extract_kernel below. To avoid any issues, repoint the GDTR + * to the new copy of the GDT. + */ + leaq gdt64(%rbx), %rax + subq %rbp, 2(%rax) + addq %rbx, 2(%rax) + lgdt (%rax) + /* * Jump to the relocated address. */ -- Gitee From 3dab6bf4966a623b3f725970224a021b84bb84b1 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 2 Feb 2020 12:13:52 -0500 Subject: [PATCH 011/113] x86/boot: GDT limit value should be size - 1 commit b75e2b076d00751579c73cfbbc8a7eac7d2a0468 upstream. The limit value for the GDTR should be such that adding it to the base address gives the address of the last byte of the GDT, i.e. it should be one less than the size, not the size. Hygon-SIG: commit b75e2b076d00 upstream x86/boot: GDT limit value should be size - 1 Signed-off-by: Arvind Sankar Link: https://lore.kernel.org/r/20200202171353.3736319-7-nivedita@alum.mit.edu Signed-off-by: Ard Biesheuvel Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 7f0786c36956..1c3e49be06c2 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -671,12 +671,12 @@ ENTRY(trampoline_32bit_src) .data SYM_DATA_START_LOCAL(gdt64) - .word gdt_end - gdt + .word gdt_end - gdt - 1 .quad 0 SYM_DATA_END(gdt64) .balign 8 SYM_DATA_START_LOCAL(gdt) - .word gdt_end - gdt + .word gdt_end - gdt - 1 .long gdt .word 0 .quad 0x00cf9a000000ffff /* __KERNEL32_CS */ -- Gitee From 1d460421b38ed013808e74665824de195cadfe87 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 2 Feb 2020 12:13:53 -0500 Subject: [PATCH 012/113] x86/boot: Micro-optimize GDT loading instructions commit 8a3abe30de9fffec8b44adeb78f93ecb0f09b0c5 upstream. Rearrange the instructions a bit to use a 32-bit displacement once instead of 2/3 times. This saves 8 bytes of machine code. Hygon-SIG: commit 8a3abe30de9f upstream x86/boot: Micro-optimize GDT loading instructions Signed-off-by: Arvind Sankar Link: https://lore.kernel.org/r/20200202171353.3736319-8-nivedita@alum.mit.edu Signed-off-by: Ard Biesheuvel Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 1c3e49be06c2..44430ad67df1 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -70,8 +70,9 @@ ENTRY(startup_32) subl $1b, %ebp /* Load new GDT with the 64bit segments using 32bit descriptor */ - addl %ebp, gdt+2(%ebp) - lgdt gdt(%ebp) + leal gdt(%ebp), %eax + movl %eax, 2(%eax) + lgdt (%eax) /* Load segment registers with our descriptors */ movl $__BOOT_DS, %eax @@ -354,9 +355,9 @@ ENTRY(startup_64) */ /* Make sure we have GDT with 32-bit code segment */ - leaq gdt(%rip), %rax - movq %rax, gdt64+2(%rip) - lgdt gdt64(%rip) + leaq gdt64(%rip), %rax + addq %rax, 2(%rax) + lgdt (%rax) /* * paging_prepare() sets up the trampoline and checks if we need to @@ -672,12 +673,12 @@ ENTRY(trampoline_32bit_src) .data SYM_DATA_START_LOCAL(gdt64) .word gdt_end - gdt - 1 - .quad 0 + .quad gdt - gdt64 SYM_DATA_END(gdt64) .balign 8 SYM_DATA_START_LOCAL(gdt) .word gdt_end - gdt - 1 - .long gdt + .long 0 .word 0 .quad 0x00cf9a000000ffff /* __KERNEL32_CS */ .quad 0x00af9a000000ffff /* __KERNEL_CS */ -- Gitee From 1e2e6005cd57107fc88f83caa41ce9bf1dab6e3b Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Wed, 26 Feb 2020 18:00:31 -0500 Subject: [PATCH 013/113] x86/boot/compressed: Fix reloading of GDTR post-relocation commit c98a76eabbb6e7755f3d4a4c33f8fe869dda6383 upstream. The following commit: ef5a7b5eb13e ("efi/x86: Remove GDT setup from efi_main") introduced GDT setup into the 32-bit kernel's startup_32, and reloads the GDTR after relocating the kernel for paranoia's sake. A followup commit: 32d009137a56 ("x86/boot: Reload GDTR after copying to the end of the buffer") introduced a similar GDTR reload in the 64-bit kernel as well. The GDTR is adjusted by (init_size-_end), however this may not be the correct offset to apply if the kernel was loaded at a misaligned address or below LOAD_PHYSICAL_ADDR, as in that case the decompression buffer has an additional offset from the original load address. This should never happen for a conformant bootloader, but we're being paranoid anyway, so just store the new GDT address in there instead of adding any offsets, which is simpler as well. Hygon-SIG: commit c98a76eabbb6 upstream x86/boot/compressed: Fix reloading of GDTR post-relocation Fixes: ef5a7b5eb13e ("efi/x86: Remove GDT setup from efi_main") Fixes: 32d009137a56 ("x86/boot: Reload GDTR after copying to the end of the buffer") Signed-off-by: Arvind Sankar Signed-off-by: Ingo Molnar Cc: Ingo Molnar Cc: Ard Biesheuvel Cc: linux-efi@vger.kernel.org Cc: Thomas Gleixner Cc: x86@kernel.org Link: https://lore.kernel.org/r/20200226230031.3011645-2-nivedita@alum.mit.edu [hanliyang: Fix conflict on arch/x86/boot/compressed/head_64.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 44430ad67df1..5793471a6644 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -448,8 +448,8 @@ trampoline_return: * to the new copy of the GDT. */ leaq gdt64(%rbx), %rax - subq %rbp, 2(%rax) - addq %rbx, 2(%rax) + leaq gdt(%rbx), %rdx + movq %rdx, 2(%rax) lgdt (%rax) /* -- Gitee From 16617cf0032a638963dab61b6dd80942f82752c7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 31 Jul 2020 16:07:45 -0700 Subject: [PATCH 014/113] x86/boot/compressed: Move .got.plt entries out of the .got section commit 262b5cae67a672404da0dcbd009efc1227ad51e4 upstream. The .got.plt section contains the part of the GOT which is used by PLT entries, and which gets updated lazily by the dynamic loader when function calls are dispatched through those PLT entries. On fully linked binaries such as the kernel proper or the decompressor, this never happens, and so in practice, the .got.plt section consists only of the first 3 magic entries that are meant to point at the _DYNAMIC section and at the fixup routine in the loader. However, since we don't use a dynamic loader, those entries are never populated or used. This means that treating those entries like ordinary GOT entries, and updating their values based on the actual placement of the executable in memory is completely pointless, and we can just ignore the .got.plt section entirely, provided that it has no additional entries beyond the first 3 ones. So add an assertion in the linker script to ensure that this assumption holds, and move the contents out of the [_got, _egot) memory range that is modified by the GOT fixup routines. While at it, drop the KEEP(), since it has no effect on the contents of output sections that are created by the linker itself. Hygon-SIG: commit 262b5cae67a6 upstream x86/boot/compressed: Move .got.plt entries out of the .got section Signed-off-by: Ard Biesheuvel Signed-off-by: Arvind Sankar Signed-off-by: Kees Cook Signed-off-by: Ingo Molnar Tested-by: Sedat Dilek Tested-by: Nick Desaulniers Reviewed-by: Kees Cook Acked-by: Arvind Sankar Link: https://lore.kernel.org/r/20200731230820.1742553-2-keescook@chromium.org [hanliyang: Fix conflict on arch/x86/boot/compressed/vmlinux.lds.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/vmlinux.lds.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 508cfa6828c5..0f3fde26a2eb 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -44,10 +44,13 @@ SECTIONS } .got : { _got = .; - KEEP(*(.got.plt)) KEEP(*(.got)) _egot = .; } + .got.plt : { + *(.got.plt) + } + .data : { _data = . ; *(.data) @@ -74,3 +77,9 @@ SECTIONS . = ALIGN(PAGE_SIZE); /* keep ZO size page aligned */ _end = .; } + +#ifdef CONFIG_X86_64 +ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") +#else +ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0xc, "Unexpected GOT/PLT entries detected!") +#endif -- Gitee From 64cb17b495944d5a9b66bfdc57966e9a60ba1c18 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 31 Jul 2020 16:07:46 -0700 Subject: [PATCH 015/113] x86/boot/compressed: Force hidden visibility for all symbol references commit e544ea57ac0734bca752eb2d8635fecbe932c356 upstream. Eliminate all GOT entries in the decompressor binary, by forcing hidden visibility for all symbol references, which informs the compiler that such references will be resolved at link time without the need for allocating GOT entries. To ensure that no GOT entries will creep back in, add an assertion to the decompressor linker script that will fire if the .got section has a non-zero size. Hygon-SIG: commit e544ea57ac07 upstream x86/boot/compressed: Force hidden visibility for all symbol references [Arvind: move hidden.h to include/linux instead of making a copy] Signed-off-by: Ard Biesheuvel Signed-off-by: Arvind Sankar Signed-off-by: Kees Cook Signed-off-by: Ingo Molnar Tested-by: Nick Desaulniers Tested-by: Sedat Dilek Reviewed-by: Kees Cook Acked-by: Arvind Sankar Link: https://lore.kernel.org/r/20200731230820.1742553-3-keescook@chromium.org [hanliyang: Fix conflict on arch/x86/boot/compressed/Makefile, arch/x86/boot/compressed/vmlinux.lds.S, drivers/firmware/efi/libstub/Makefile, include/linux/hidden.h] Signed-off-by: hanliyang --- arch/x86/boot/compressed/Makefile | 1 + arch/x86/boot/compressed/vmlinux.lds.S | 1 + drivers/firmware/efi/libstub/Makefile | 1 + include/linux/hidden.h | 19 +++++++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 include/linux/hidden.h diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index edfb1a718510..16af006316c8 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -40,6 +40,7 @@ KBUILD_CFLAGS += $(call cc-disable-warning, gnu) KBUILD_CFLAGS += -Wno-pointer-sign # Disable relocation relaxation in case the link is not PIE. KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no) +KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 0f3fde26a2eb..9b967c4ea5f4 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -78,6 +78,7 @@ SECTIONS _end = .; } +ASSERT(SIZEOF(.got) == 0, "Unexpected GOT entries detected!") #ifdef CONFIG_X86_64 ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") #else diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 8c5b5529dbc0..bf5a03c9b1ee 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -25,6 +25,7 @@ cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ cflags-$(CONFIG_EFI_ARMSTUB) += -I$(srctree)/scripts/dtc/libfdt KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ + -include $(srctree)/include/linux/hidden.h \ -D__NO_FORTIFY \ $(call cc-option,-ffreestanding) \ $(call cc-option,-fno-stack-protector) \ diff --git a/include/linux/hidden.h b/include/linux/hidden.h new file mode 100644 index 000000000000..49a17b6b5962 --- /dev/null +++ b/include/linux/hidden.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * When building position independent code with GCC using the -fPIC option, + * (or even the -fPIE one on older versions), it will assume that we are + * building a dynamic object (either a shared library or an executable) that + * may have symbol references that can only be resolved at load time. For a + * variety of reasons (ELF symbol preemption, the CoW footprint of the section + * that is modified by the loader), this results in all references to symbols + * with external linkage to go via entries in the Global Offset Table (GOT), + * which carries absolute addresses which need to be fixed up when the + * executable image is loaded at an offset which is different from its link + * time offset. + * + * Fortunately, there is a way to inform the compiler that such symbol + * references will be satisfied at link time rather than at load time, by + * giving them 'hidden' visibility. + */ + +#pragma GCC visibility push(hidden) -- Gitee From e24e4961711afdaff58fb28b30f19bc4f9b91f3d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 31 Jul 2020 16:07:47 -0700 Subject: [PATCH 016/113] x86/boot/compressed: Get rid of GOT fixup code commit 423e4d198a036689de73fd6b073fc4349c4fa1ee upstream. In a previous patch, we have eliminated GOT entries from the decompressor binary and added an assertion that the .got section is empty. This means that the GOT fixup routines that exist in both the 32-bit and 64-bit startup routines have become dead code, and can be removed. While at it, drop the KEEP() from the linker script, as it has no effect on the contents of output sections that are created by the linker itself. Hygon-SIG: commit 423e4d198a03 upstream x86/boot/compressed: Get rid of GOT fixup code Signed-off-by: Ard Biesheuvel Signed-off-by: Arvind Sankar Signed-off-by: Kees Cook Signed-off-by: Ingo Molnar Tested-by: Nick Desaulniers Tested-by: Sedat Dilek Reviewed-by: Kees Cook Acked-by: Arvind Sankar Link: https://lore.kernel.org/r/20200731230820.1742553-4-keescook@chromium.org [hanliyang: Fix conflict on arch/x86/boot/compressed/head_32.S, arch/x86/boot/compressed/head_64.S, arch/x86/boot/compressed/vmlinux.lds.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_32.S | 24 ++--------- arch/x86/boot/compressed/head_64.S | 57 -------------------------- arch/x86/boot/compressed/vmlinux.lds.S | 4 +- 3 files changed, 5 insertions(+), 80 deletions(-) diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index 2fee6bb09282..bd5b6473424d 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -49,16 +49,13 @@ * Position Independent Executable (PIE) so that linker won't optimize * R_386_GOT32X relocation to its fixed symbol address. Older * linkers generate R_386_32 relocations against locally defined symbols, - * _bss, _ebss, _got, _egot and _end, in PIE. It isn't wrong, just less - * optimal than R_386_RELATIVE. But the x86 kernel fails to properly handle - * R_386_32 relocations when relocating the kernel. To generate - * R_386_RELATIVE relocations, we mark _bss, _ebss, _got, _egot and _end as - * hidden: + * _bss, _ebss and _end, in PIE. It isn't wrong, just less optimal than + * R_386_RELATIVE. But the x86 kernel fails to properly handle R_386_32 + * relocations when relocating the kernel. To generate R_386_RELATIVE + * relocations, we mark _bss, _ebss and _end as hidden: */ .hidden _bss .hidden _ebss - .hidden _got - .hidden _egot .hidden _end __HEAD @@ -214,19 +211,6 @@ ENDPROC(efi32_stub_entry) shrl $2, %ecx rep stosl -/* - * Adjust our own GOT - */ - leal _got(%ebx), %edx - leal _egot(%ebx), %ecx -1: - cmpl %ecx, %edx - jae 2f - addl %ebx, (%edx) - addl $4, %edx - jmp 1b -2: - /* * Do the extraction, and jump to the new kernel.. */ diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 5793471a6644..6806b72e5a6e 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -40,8 +40,6 @@ */ .hidden _bss .hidden _ebss - .hidden _got - .hidden _egot .hidden _end __HEAD @@ -310,25 +308,6 @@ ENTRY(startup_64) /* Set up the stack */ leaq boot_stack_end(%rbx), %rsp - /* - * paging_prepare() and cleanup_trampoline() below can have GOT - * references. Adjust the table with address we are running at. - * - * Zero RAX for adjust_got: the GOT was not adjusted before; - * there's no adjustment to undo. - */ - xorq %rax, %rax - - /* - * Calculate the address the binary is loaded at and use it as - * a GOT adjustment. - */ - call 1f -1: popq %rdi - subq $1b, %rdi - - call .Ladjust_got - /* * At this point we are in long mode with 4-level paging enabled, * but we might want to enable 5-level paging or vice versa. @@ -413,21 +392,6 @@ trampoline_return: pushq $0 popfq - /* - * Previously we've adjusted the GOT with address the binary was - * loaded at. Now we need to re-adjust for relocation address. - * - * Calculate the address the binary is loaded at, so that we can - * undo the previous GOT adjustment. - */ - call 1f -1: popq %rax - subq $1b, %rax - - /* The new adjustment is the relocation address */ - movq %rbx, %rdi - call .Ladjust_got - /* * Copy the compressed kernel to the end of our buffer * where decompression in place becomes safe. @@ -556,27 +520,6 @@ ENDPROC(efi64_stub_entry) */ jmp *%rax -/* - * Adjust the global offset table - * - * RAX is the previous adjustment of the table to undo (use 0 if it's the - * first time we touch GOT). - * RDI is the new adjustment to apply. - */ -.Ladjust_got: - /* Walk through the GOT adding the address to the entries */ - leaq _got(%rip), %rdx - leaq _egot(%rip), %rcx -1: - cmpq %rcx, %rdx - jae 2f - subq %rax, (%rdx) /* Undo previous adjustment */ - addq %rdi, (%rdx) /* Apply the new adjustment */ - addq $8, %rdx - jmp 1b -2: - ret - .code32 /* * This is the 32-bit trampoline that will be copied over to low memory. diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 9b967c4ea5f4..3a017206535f 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -43,9 +43,7 @@ SECTIONS _erodata = . ; } .got : { - _got = .; - KEEP(*(.got)) - _egot = .; + *(.got) } .got.plt : { *(.got.plt) -- Gitee From b9246f5c3dc6f49ab2bb5eaf5745363050b047db Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 28 Apr 2020 17:16:22 +0200 Subject: [PATCH 017/113] x86/boot/compressed/64: Switch to __KERNEL_CS after GDT is loaded commit 34bb49229f19399a5b45c323afb5749f31f7876c upstream. When the pre-decompression code loads its first GDT in startup_64(), it is still running on the CS value of the previous GDT. In the case of SEV-ES, this is the EFI GDT but it can be anything depending on what has loaded the kernel (boot loader, container runtime, etc.) To make exception handling work (especially IRET) the CPU needs to switch to a CS value in the current GDT, so jump to __KERNEL_CS after the first GDT is loaded. This is prudent also as a general sanitization of CS to a known good value. [ bp: Massage commit message. ] Hygon-SIG: commit 34bb49229f19 upstream x86/boot/compressed/64: Switch to __KERNEL_CS after GDT is loaded Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200428151725.31091-13-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 6806b72e5a6e..ba30d3da4aba 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -338,6 +338,14 @@ ENTRY(startup_64) addq %rax, 2(%rax) lgdt (%rax) + /* Reload CS so IRET returns to a CS actually in the GDT */ + pushq $__KERNEL_CS + leaq .Lon_kernel_cs(%rip), %rax + pushq %rax + lretq + +.Lon_kernel_cs: + /* * paging_prepare() sets up the trampoline and checks if we need to * enable 5-level paging. -- Gitee From fe173349f4783ae201eff1da970617b199fda4be Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 4 Mar 2020 13:09:50 +0100 Subject: [PATCH 018/113] tracing: Provide lockdep less trace_hardirqs_on/off() variants commit 0995a5dfbe49badff78e78761fb66f46579f2f9a upstream. trace_hardirqs_on/off() is only partially safe vs. RCU idle. The tracer core itself is safe, but the resulting tracepoints can be utilized by e.g. BPF which is unsafe. Provide variants which do not contain the lockdep invocation so the lockdep and tracer invocations can be split at the call site and placed properly. This is required because lockdep needs to be aware of the state before switching away from RCU idle and after switching to RCU idle because these transitions can take locks. As these code pathes are going to be non-instrumentable the tracer can be invoked after RCU is turned on and before the switch to RCU idle. So for these new variants there is no need to invoke the rcuidle aware tracer functions. Name them so they match the lockdep counterparts. Hygon-SIG: commit 0995a5dfbe49 upstream tracing: Provide lockdep less trace_hardirqs_on/off() variants Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134100.270771162@linutronix.de [hanliyang: Fix conflict on include/linux/irqflags.h, kernel/trace/trace_preemptirq.c] Signed-off-by: hanliyang --- include/linux/irqflags.h | 4 ++++ kernel/trace/trace_preemptirq.c | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index cba728fad8ae..b061c3f62655 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -30,6 +30,8 @@ #endif #ifdef CONFIG_TRACE_IRQFLAGS + extern void trace_hardirqs_on_prepare(void); + extern void trace_hardirqs_off_prepare(void); extern void trace_hardirqs_on(void); extern void trace_hardirqs_off(void); # define trace_hardirq_context(p) ((p)->hardirq_context) @@ -53,6 +55,8 @@ do { \ current->softirq_context--; \ } while (0) #else +# define trace_hardirqs_on_prepare() do { } while (0) +# define trace_hardirqs_off_prepare() do { } while (0) # define trace_hardirqs_on() do { } while (0) # define trace_hardirqs_off() do { } while (0) # define trace_hardirq_context(p) 0 diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index e9645f829b94..05f2093d7815 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -19,6 +19,24 @@ /* Per-cpu variable to prevent redundant calls when IRQs already off */ static DEFINE_PER_CPU(int, tracing_irq_cpu); +/* + * Like trace_hardirqs_on() but without the lockdep invocation. This is + * used in the low level entry code where the ordering vs. RCU is important + * and lockdep uses a staged approach which splits the lockdep hardirq + * tracking into a RCU on and a RCU off section. + */ +void trace_hardirqs_on_prepare(void) +{ + if (this_cpu_read(tracing_irq_cpu)) { + if (!in_nmi()) + trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); + tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); + this_cpu_write(tracing_irq_cpu, 0); + } +} +EXPORT_SYMBOL(trace_hardirqs_on_prepare); +NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); + void trace_hardirqs_on(void) { if (this_cpu_read(tracing_irq_cpu)) { @@ -33,6 +51,25 @@ void trace_hardirqs_on(void) EXPORT_SYMBOL(trace_hardirqs_on); NOKPROBE_SYMBOL(trace_hardirqs_on); +/* + * Like trace_hardirqs_off() but without the lockdep invocation. This is + * used in the low level entry code where the ordering vs. RCU is important + * and lockdep uses a staged approach which splits the lockdep hardirq + * tracking into a RCU on and a RCU off section. + */ +void trace_hardirqs_off_prepare(void) +{ + if (!this_cpu_read(tracing_irq_cpu)) { + this_cpu_write(tracing_irq_cpu, 1); + tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); + if (!in_nmi()) + trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); + } + +} +EXPORT_SYMBOL(trace_hardirqs_off_prepare); +NOKPROBE_SYMBOL(trace_hardirqs_off_prepare); + void trace_hardirqs_off(void) { if (!this_cpu_read(tracing_irq_cpu)) { -- Gitee From e97a68e38e3a052a4baa2f50dd6f17069a9bda11 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 18 Mar 2020 14:22:03 +0100 Subject: [PATCH 019/113] lockdep: Prepare for noinstr sections commit c86e9b987cea3dd0209203e714553a47f5d7c6dd upstream. Force inlining and prevent instrumentation of all sorts by marking the functions which are invoked from low level entry code with 'noinstr'. Split the irqflags tracking into two parts. One which does the heavy lifting while RCU is watching and the final one which can be invoked after RCU is turned off. Hygon-SIG: commit c86e9b987cea upstream lockdep: Prepare for noinstr sections Signed-off-by: Peter Zijlstra Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Link: https://lkml.kernel.org/r/20200505134100.484532537@linutronix.de [hanliyang: Fix conflict on include/linux/irqflags.h, include/linux/sched.h, kernel/locking/lockdep.c, kernel/trace/trace_preemptirq.c, lib/debug_locks.c] Signed-off-by: hanliyang --- include/linux/irqflags.h | 2 + include/linux/sched.h | 1 + kernel/locking/lockdep.c | 86 +++++++++++++++++++++++++-------- kernel/trace/trace_preemptirq.c | 2 + lib/debug_locks.c | 2 +- 5 files changed, 71 insertions(+), 22 deletions(-) diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index b061c3f62655..93e62a85fa28 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -20,11 +20,13 @@ #ifdef CONFIG_PROVE_LOCKING extern void trace_softirqs_on(unsigned long ip); extern void trace_softirqs_off(unsigned long ip); + extern void lockdep_hardirqs_on_prepare(unsigned long ip); extern void lockdep_hardirqs_on(unsigned long ip); extern void lockdep_hardirqs_off(unsigned long ip); #else static inline void trace_softirqs_on(unsigned long ip) { } static inline void trace_softirqs_off(unsigned long ip) { } + static inline void lockdep_hardirqs_on_prepare(unsigned long ip) { } static inline void lockdep_hardirqs_on(unsigned long ip) { } static inline void lockdep_hardirqs_off(unsigned long ip) { } #endif diff --git a/include/linux/sched.h b/include/linux/sched.h index 2e7048f23f89..bc6fb6e391fc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1115,6 +1115,7 @@ struct task_struct { unsigned int hardirq_disable_event; int hardirqs_enabled; int hardirq_context; + u64 hardirq_chain_key; unsigned long softirq_disable_ip; unsigned long softirq_enable_ip; unsigned int softirq_disable_event; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 6f66ee8d0aeb..96c036f83463 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -3489,13 +3489,10 @@ mark_held_locks(struct task_struct *curr, enum lock_usage_bit base_bit) /* * Hardirqs will be enabled: */ -static void __trace_hardirqs_on_caller(unsigned long ip) +static void __trace_hardirqs_on_caller(void) { struct task_struct *curr = current; - /* we'll do an OFF -> ON transition: */ - curr->hardirqs_enabled = 1; - /* * We are going to turn hardirqs on, so set the * usage bit for all held locks: @@ -3508,15 +3505,19 @@ static void __trace_hardirqs_on_caller(unsigned long ip) * this bit from being set before) */ if (curr->softirqs_enabled) - if (!mark_held_locks(curr, LOCK_ENABLED_SOFTIRQ)) - return; - - curr->hardirq_enable_ip = ip; - curr->hardirq_enable_event = ++curr->irq_events; - debug_atomic_inc(hardirqs_on_events); + mark_held_locks(curr, LOCK_ENABLED_SOFTIRQ); } -void lockdep_hardirqs_on(unsigned long ip) +/** + * lockdep_hardirqs_on_prepare - Prepare for enabling interrupts + * @ip: Caller address + * + * Invoked before a possible transition to RCU idle from exit to user or + * guest mode. This ensures that all RCU operations are done before RCU + * stops watching. After the RCU transition lockdep_hardirqs_on() has to be + * invoked to set the final state. + */ +void lockdep_hardirqs_on_prepare(unsigned long ip) { if (unlikely(!debug_locks || current->lockdep_recursion)) return; @@ -3552,20 +3553,62 @@ void lockdep_hardirqs_on(unsigned long ip) if (DEBUG_LOCKS_WARN_ON(current->hardirq_context)) return; + current->hardirq_chain_key = current->curr_chain_key; + current->lockdep_recursion = 1; - __trace_hardirqs_on_caller(ip); + __trace_hardirqs_on_caller(); current->lockdep_recursion = 0; } -NOKPROBE_SYMBOL(lockdep_hardirqs_on); +EXPORT_SYMBOL_GPL(lockdep_hardirqs_on_prepare); + +void noinstr lockdep_hardirqs_on(unsigned long ip) +{ + struct task_struct *curr = current; + + if (unlikely(!debug_locks || curr->lockdep_recursion)) + return; + + if (curr->hardirqs_enabled) { + /* + * Neither irq nor preemption are disabled here + * so this is racy by nature but losing one hit + * in a stat is not a big deal. + */ + __debug_atomic_inc(redundant_hardirqs_on); + return; + } + + /* + * We're enabling irqs and according to our state above irqs weren't + * already enabled, yet we find the hardware thinks they are in fact + * enabled.. someone messed up their IRQ state tracing. + */ + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + + /* + * Ensure the lock stack remained unchanged between + * lockdep_hardirqs_on_prepare() and lockdep_hardirqs_on(). + */ + DEBUG_LOCKS_WARN_ON(current->hardirq_chain_key != + current->curr_chain_key); + + /* we'll do an OFF -> ON transition: */ + curr->hardirqs_enabled = 1; + curr->hardirq_enable_ip = ip; + curr->hardirq_enable_event = ++curr->irq_events; + debug_atomic_inc(hardirqs_on_events); +} +EXPORT_SYMBOL_GPL(lockdep_hardirqs_on); /* * Hardirqs were disabled: */ -void lockdep_hardirqs_off(unsigned long ip) +void noinstr lockdep_hardirqs_off(unsigned long ip) { struct task_struct *curr = current; - if (unlikely(!debug_locks || current->lockdep_recursion)) + if (unlikely(!debug_locks || curr->lockdep_recursion)) return; /* @@ -3583,10 +3626,11 @@ void lockdep_hardirqs_off(unsigned long ip) curr->hardirq_disable_ip = ip; curr->hardirq_disable_event = ++curr->irq_events; debug_atomic_inc(hardirqs_off_events); - } else + } else { debug_atomic_inc(redundant_hardirqs_off); + } } -NOKPROBE_SYMBOL(lockdep_hardirqs_off); +EXPORT_SYMBOL_GPL(lockdep_hardirqs_off); /* * Softirqs will be enabled: @@ -4131,8 +4175,8 @@ static void print_unlock_imbalance_bug(struct task_struct *curr, dump_stack(); } -static int match_held_lock(const struct held_lock *hlock, - const struct lockdep_map *lock) +static noinstr int match_held_lock(const struct held_lock *hlock, + const struct lockdep_map *lock) { if (hlock->instance == lock) return 1; @@ -4419,7 +4463,7 @@ __lock_release(struct lockdep_map *lock, unsigned long ip) return 0; } -static nokprobe_inline +static __always_inline int __lock_is_held(const struct lockdep_map *lock, int read) { struct task_struct *curr = current; @@ -4633,7 +4677,7 @@ void lock_release(struct lockdep_map *lock, int nested, } EXPORT_SYMBOL_GPL(lock_release); -int lock_is_held_type(const struct lockdep_map *lock, int read) +noinstr int lock_is_held_type(const struct lockdep_map *lock, int read) { unsigned long flags; int ret = 0; diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index 05f2093d7815..465e261abbbd 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -46,6 +46,7 @@ void trace_hardirqs_on(void) this_cpu_write(tracing_irq_cpu, 0); } + lockdep_hardirqs_on_prepare(CALLER_ADDR0); lockdep_hardirqs_on(CALLER_ADDR0); } EXPORT_SYMBOL(trace_hardirqs_on); @@ -93,6 +94,7 @@ __visible void trace_hardirqs_on_caller(unsigned long caller_addr) this_cpu_write(tracing_irq_cpu, 0); } + lockdep_hardirqs_on_prepare(caller_addr); lockdep_hardirqs_on(caller_addr); } EXPORT_SYMBOL(trace_hardirqs_on_caller); diff --git a/lib/debug_locks.c b/lib/debug_locks.c index a75ee30b77cb..06d3135bd184 100644 --- a/lib/debug_locks.c +++ b/lib/debug_locks.c @@ -36,7 +36,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent); /* * Generic 'turn off all lock debugging' function: */ -int debug_locks_off(void) +noinstr int debug_locks_off(void) { if (debug_locks && __debug_locks_off()) { if (!debug_locks_silent) { -- Gitee From aaa7d5bb461ecc3163f4614c9127b57bdbe5ef57 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:16:07 +0100 Subject: [PATCH 020/113] x86/traps: Split trap numbers out in a separate header commit 877f183f83cc33b1b09313b9c18ab7ee5abda44f upstream. So they can be used in ASM code. For this it is also necessary to convert them to defines. Will be used for the rework of the entry code. Hygon-SIG: commit 877f183f83cc upstream x86/traps: Split trap numbers out in a separate header Signed-off-by: Thomas Gleixner Reviewed-by: Andy Lutomirski Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134903.731004084@linutronix.de Signed-off-by: hanliyang --- arch/x86/include/asm/trapnr.h | 31 +++++++++++++++++++++++++++++++ arch/x86/include/asm/traps.h | 26 +------------------------- 2 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 arch/x86/include/asm/trapnr.h diff --git a/arch/x86/include/asm/trapnr.h b/arch/x86/include/asm/trapnr.h new file mode 100644 index 000000000000..082f45631fa9 --- /dev/null +++ b/arch/x86/include/asm/trapnr.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_TRAPNR_H +#define _ASM_X86_TRAPNR_H + +/* Interrupts/Exceptions */ + +#define X86_TRAP_DE 0 /* Divide-by-zero */ +#define X86_TRAP_DB 1 /* Debug */ +#define X86_TRAP_NMI 2 /* Non-maskable Interrupt */ +#define X86_TRAP_BP 3 /* Breakpoint */ +#define X86_TRAP_OF 4 /* Overflow */ +#define X86_TRAP_BR 5 /* Bound Range Exceeded */ +#define X86_TRAP_UD 6 /* Invalid Opcode */ +#define X86_TRAP_NM 7 /* Device Not Available */ +#define X86_TRAP_DF 8 /* Double Fault */ +#define X86_TRAP_OLD_MF 9 /* Coprocessor Segment Overrun */ +#define X86_TRAP_TS 10 /* Invalid TSS */ +#define X86_TRAP_NP 11 /* Segment Not Present */ +#define X86_TRAP_SS 12 /* Stack Segment Fault */ +#define X86_TRAP_GP 13 /* General Protection Fault */ +#define X86_TRAP_PF 14 /* Page Fault */ +#define X86_TRAP_SPURIOUS 15 /* Spurious Interrupt */ +#define X86_TRAP_MF 16 /* x87 Floating-Point Exception */ +#define X86_TRAP_AC 17 /* Alignment Check */ +#define X86_TRAP_MC 18 /* Machine Check */ +#define X86_TRAP_XF 19 /* SIMD Floating-Point Exception */ +#define X86_TRAP_VE 20 /* Virtualization Exception */ +#define X86_TRAP_CP 21 /* Control Protection Exception */ +#define X86_TRAP_IRET 32 /* IRET Exception */ + +#endif diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 71abccd78a4f..09d769366d4b 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -7,6 +7,7 @@ #include #include /* TRAP_TRACE, ... */ +#include #define dotraplinkage __visible @@ -126,31 +127,6 @@ void __noreturn handle_stack_overflow(const char *message, unsigned long fault_address); #endif -/* Interrupts/Exceptions */ -enum { - X86_TRAP_DE = 0, /* 0, Divide-by-zero */ - X86_TRAP_DB, /* 1, Debug */ - X86_TRAP_NMI, /* 2, Non-maskable Interrupt */ - X86_TRAP_BP, /* 3, Breakpoint */ - X86_TRAP_OF, /* 4, Overflow */ - X86_TRAP_BR, /* 5, Bound Range Exceeded */ - X86_TRAP_UD, /* 6, Invalid Opcode */ - X86_TRAP_NM, /* 7, Device Not Available */ - X86_TRAP_DF, /* 8, Double Fault */ - X86_TRAP_OLD_MF, /* 9, Coprocessor Segment Overrun */ - X86_TRAP_TS, /* 10, Invalid TSS */ - X86_TRAP_NP, /* 11, Segment Not Present */ - X86_TRAP_SS, /* 12, Stack Segment Fault */ - X86_TRAP_GP, /* 13, General Protection Fault */ - X86_TRAP_PF, /* 14, Page Fault */ - X86_TRAP_SPURIOUS, /* 15, Spurious Interrupt */ - X86_TRAP_MF, /* 16, x87 Floating-Point Exception */ - X86_TRAP_AC, /* 17, Alignment Check */ - X86_TRAP_MC, /* 18, Machine Check */ - X86_TRAP_XF, /* 19, SIMD Floating-Point Exception */ - X86_TRAP_IRET = 32, /* 32, IRET Exception */ -}; - /* * Page fault error code bits: * -- Gitee From b5b41fa74e3f69bda2db560b04ace6c4c7793583 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:16:09 +0100 Subject: [PATCH 021/113] x86/entry/64: Reorder idtentries commit 67f1386616dc0c70f30f214245521c582666edac upstream. Move them all together so verifying the cleanup patches for binary equivalence will be easier. Hygon-SIG: commit 67f1386616dc upstream x86/entry/64: Reorder idtentries Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Andy Lutomirski Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134903.841853522@linutronix.de [hanliyang: Fix conflict on arch/x86/entry/entry_64.S] Signed-off-by: hanliyang --- arch/x86/entry/entry_64.S | 44 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 7f27011f5461..b1769121d825 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1027,20 +1027,40 @@ _ASM_NOKPROBE(\sym) END(\sym) .endm + idtentry divide_error do_divide_error has_error_code=0 idtentry overflow do_overflow has_error_code=0 +idtentry int3 do_int3 has_error_code=0 create_gap=1 idtentry bounds do_bounds has_error_code=0 idtentry invalid_op do_invalid_op has_error_code=0 idtentry device_not_available do_device_not_available has_error_code=0 -idtentry double_fault do_double_fault has_error_code=1 paranoid=2 read_cr2=1 idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 idtentry invalid_TSS do_invalid_TSS has_error_code=1 idtentry segment_not_present do_segment_not_present has_error_code=1 +idtentry stack_segment do_stack_segment has_error_code=1 +idtentry general_protection do_general_protection has_error_code=1 idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0 idtentry coprocessor_error do_coprocessor_error has_error_code=0 idtentry alignment_check do_alignment_check has_error_code=1 idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0 +idtentry page_fault do_page_fault has_error_code=1 read_cr2=1 + +#ifdef CONFIG_KVM_GUEST +idtentry async_page_fault do_async_page_fault has_error_code=1 read_cr2=1 +#endif + +#ifdef CONFIG_X86_MCE +idtentry machine_check do_mce has_error_code=0 paranoid=1 +#endif +idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=IST_INDEX_DB ist_offset=DB_STACK_OFFSET +idtentry double_fault do_double_fault has_error_code=1 paranoid=2 read_cr2=1 + +#ifdef CONFIG_XEN_PV +idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 +idtentry xennmi do_nmi has_error_code=0 +idtentry xendebug do_debug has_error_code=0 +#endif /* * Reload gs selector with exception handling @@ -1092,8 +1112,6 @@ ENTRY(do_softirq_own_stack) ENDPROC(do_softirq_own_stack) #ifdef CONFIG_XEN_PV -idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 - /* * A note on the "critical region" in our callback handler. * We want to avoid stacking callback handlers due to events occurring @@ -1195,26 +1213,6 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ acrn_hv_callback_vector acrn_hv_vector_handler #endif -idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=IST_INDEX_DB ist_offset=DB_STACK_OFFSET -idtentry int3 do_int3 has_error_code=0 create_gap=1 -idtentry stack_segment do_stack_segment has_error_code=1 - -#ifdef CONFIG_XEN_PV -idtentry xennmi do_nmi has_error_code=0 -idtentry xendebug do_debug has_error_code=0 -#endif - -idtentry general_protection do_general_protection has_error_code=1 -idtentry page_fault do_page_fault has_error_code=1 read_cr2=1 - -#ifdef CONFIG_KVM_GUEST -idtentry async_page_fault do_async_page_fault has_error_code=1 read_cr2=1 -#endif - -#ifdef CONFIG_X86_MCE -idtentry machine_check do_mce has_error_code=0 paranoid=1 -#endif - /* * Save all registers in pt_regs. Return GSBASE related information * in EBX depending on the availability of the FSGSBASE instructions: -- Gitee From b7d210619d0b7de99e05da2aaef2f8126e19ff2d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:16:12 +0100 Subject: [PATCH 022/113] x86/idtentry: Provide macros to define/declare IDT entry points commit 53aaf262c66ee237e4163f1af347939ebd9c51c2 upstream. Provide DECLARE/DEFINE_IDTENTRY() macros. DEFINE_IDTENTRY() provides a wrapper which acts as the function definition. The exception handler body is just appended to it with curly brackets. The entry point is marked noinstr so that irq tracing and the enter_from_user_mode() can be moved into the C-entry point. As all C-entries use the same macro (or a later variant) the necessary entry handling can be implemented at one central place. DECLARE_IDTENTRY() provides the function prototypes: - The C entry point cfunc - The ASM entry point asm_cfunc - The XEN/PV entry point xen_asm_cfunc They all follow the same naming convention. When included from ASM code DECLARE_IDTENTRY() is a macro which emits the low level entry point in assembly by instantiating idtentry. IDTENTRY is the simplest variant which just has a pt_regs argument. It's going to be used for all exceptions which have no error code. Hygon-SIG: commit 53aaf262c66e upstream x86/idtentry: Provide macros to define/declare IDT entry points Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Andy Lutomirski Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134904.273363275@linutronix.de [hanliyang: Fix conflict on arch/x86/entry/entry_32.S, arch/x86/entry/entry_64.S, arch/x86/include/asm/idtentry.h, arch/x86/include/asm/traps.h] Signed-off-by: hanliyang --- arch/x86/entry/entry_32.S | 6 +++ arch/x86/entry/entry_64.S | 6 +++ arch/x86/include/asm/idtentry.h | 67 +++++++++++++++++++++++++++++++++ arch/x86/include/asm/traps.h | 2 +- 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 arch/x86/include/asm/idtentry.h diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 740df9cc2196..a9d20754afed 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -726,6 +726,12 @@ .Lend_\@: .endm +/* + * Include the defines which emit the idt entries which are shared + * shared between 32 and 64 bit. + */ +#include + /* * %eax: prev task * %edx: next task diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index b1769121d825..1d2385a389fa 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -498,6 +498,12 @@ END(spurious_entries_start) decl PER_CPU_VAR(irq_count) .endm +/* + * Include the defines which emit the idt entries which are shared + * shared between 32 and 64 bit. + */ +#include + /* * Interrupt entry helper function. * diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h new file mode 100644 index 000000000000..bbd81e200b32 --- /dev/null +++ b/arch/x86/include/asm/idtentry.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_IDTENTRY_H +#define _ASM_X86_IDTENTRY_H + +/* Interrupts/Exceptions */ +#include + +#ifndef __ASSEMBLY__ + +/** + * DECLARE_IDTENTRY - Declare functions for simple IDT entry points + * No error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it + * declares the entry points for usage in C code. There is an ASM variant + * as well which is used to emit the entry stubs in entry_32/64.S. + */ +#define DECLARE_IDTENTRY(vector, func) \ + asmlinkage void asm_##func(void); \ + asmlinkage void xen_asm_##func(void); \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_IDTENTRY - Emit code for simple IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * idtentry_enter() contains common code which has to be invoked before + * arbitrary code in the body. idtentry_exit() contains common code + * which has to run before returning to the low level assembly code. + */ +#define DEFINE_IDTENTRY(func) \ +static __always_inline void __##func(struct pt_regs *regs); \ + \ +__visible noinstr void func(struct pt_regs *regs) \ +{ \ + idtentry_enter(regs); \ + instrumentation_begin(); \ + __##func (regs); \ + instrumentation_end(); \ + idtentry_exit(regs); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs) + +#else /* !__ASSEMBLY__ */ + +/* + * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs. + */ +#define DECLARE_IDTENTRY(vector, func) \ + idtentry vector asm_##func func has_error_code=0 sane=1 + +#endif /* __ASSEMBLY__ */ + +#endif diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 09d769366d4b..6aeb4af71291 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -6,8 +6,8 @@ #include #include +#include #include /* TRAP_TRACE, ... */ -#include #define dotraplinkage __visible -- Gitee From 95315cc4004ffc47340370be7576946791d31fc8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 26 Mar 2020 16:28:52 +0100 Subject: [PATCH 023/113] x86/entry/common: Provide idtentry_enter/exit() commit 0ba50e861ae9f91ffb6589e3a8ecbd47859b0275 upstream. Provide functions which handle the low level entry and exit similar to enter/exit from user mode. Hygon-SIG: commit 0ba50e861ae9 upstream x86/entry/common: Provide idtentry_enter/exit() Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Acked-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20200505134904.457578656@linutronix.de [hanliyang: Fix conflict on arch/x86/entry/common.c, arch/x86/include/asm/idtentry.h] Signed-off-by: hanliyang --- arch/x86/entry/common.c | 103 ++++++++++++++++++++++++++++++++ arch/x86/include/asm/idtentry.h | 3 + 2 files changed, 106 insertions(+) diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 47179aae6828..f48d9d293f78 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -435,3 +435,106 @@ __visible long do_fast_syscall_32(struct pt_regs *regs) #endif } #endif + +/** + * idtentry_enter - Handle state tracking on idtentry + * @regs: Pointer to pt_regs of interrupted context + * + * Invokes: + * - lockdep irqflag state tracking as low level ASM entry disabled + * interrupts. + * + * - Context tracking if the exception hit user mode. + * + * - RCU notification if the exception hit kernel mode. + * + * - The hardirq tracer to keep the state consistent as low level ASM + * entry disabled interrupts. + */ +void noinstr idtentry_enter(struct pt_regs *regs) +{ + if (user_mode(regs)) { + enter_from_user_mode(); + } else { + lockdep_hardirqs_off(CALLER_ADDR0); + rcu_irq_enter(); + instrumentation_begin(); + trace_hardirqs_off_prepare(); + instrumentation_end(); + } +} + +/** + * idtentry_exit - Common code to handle return from exceptions + * @regs: Pointer to pt_regs (exception entry regs) + * + * Depending on the return target (kernel/user) this runs the necessary + * preemption and work checks if possible and required and returns to + * the caller with interrupts disabled and no further work pending. + * + * This is the last action before returning to the low level ASM code which + * just needs to return to the appropriate context. + * + * Invoked by all exception/interrupt IDTENTRY handlers which are not + * returning through the paranoid exit path (all except NMI, #DF and the IST + * variants of #MC and #DB) and are therefore on the thread stack. + */ +void noinstr idtentry_exit(struct pt_regs *regs) +{ + lockdep_assert_irqs_disabled(); + + /* Check whether this returns to user mode */ + if (user_mode(regs)) { + prepare_exit_to_usermode(regs); + } else if (regs->flags & X86_EFLAGS_IF) { + /* Check kernel preemption, if enabled */ + if (IS_ENABLED(CONFIG_PREEMPTION)) { + /* + * This needs to be done very carefully. + * idtentry_enter() invoked rcu_irq_enter(). This + * needs to be undone before scheduling. + * + * Preemption is disabled inside of RCU idle + * sections. When the task returns from + * preempt_schedule_irq(), RCU is still watching. + * + * rcu_irq_exit_preempt() has additional state + * checking if CONFIG_PROVE_RCU=y + * + * NOTE: this tree has no rcu_irq_exit_preempt(); + * the rcu_irq_exit() call below is logically + * equivalent. + */ + if (!preempt_count()) { + if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) + WARN_ON_ONCE(!on_thread_stack()); + instrumentation_begin(); + rcu_irq_exit(); + if (need_resched()) + preempt_schedule_irq(); + /* Covers both tracing and lockdep */ + trace_hardirqs_on(); + instrumentation_end(); + return; + } + } + /* + * If preemption is disabled then this needs to be done + * carefully with respect to RCU. The exception might come + * from a RCU idle section in the idle task due to the fact + * that safe_halt() enables interrupts. So this needs the + * same ordering of lockdep/tracing and RCU as the return + * to user mode path. + */ + instrumentation_begin(); + /* Tell the tracer that IRET will enable interrupts */ + trace_hardirqs_on_prepare(); + lockdep_hardirqs_on_prepare(CALLER_ADDR0); + instrumentation_end(); + rcu_irq_exit(); + lockdep_hardirqs_on(CALLER_ADDR0); + } else { + /* IRQ flags state is correct already. Just tell RCU */ + rcu_irq_exit(); + } +} diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index bbd81e200b32..2adfd80ea2f3 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -7,6 +7,9 @@ #ifndef __ASSEMBLY__ +void idtentry_enter(struct pt_regs *regs); +void idtentry_exit(struct pt_regs *regs); + /** * DECLARE_IDTENTRY - Declare functions for simple IDT entry points * No error code pushed by hardware -- Gitee From 8bac1dc5467cc2f78e77f56e1a85f19cdf9ea8a4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:16:21 +0100 Subject: [PATCH 024/113] x86/idtentry: Provide IDTENTRY_ERRORCODE commit aabfe5383ec7b480ca222ac05d49eb3c83dc022a upstream. Same as IDTENTRY but the C entry point has an error code argument. Hygon-SIG: commit aabfe5383ec7 upstream x86/idtentry: Provide IDTENTRY_ERRORCODE Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Andy Lutomirski Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134905.258989060@linutronix.de [hanliyang: Fix conflict on arch/x86/include/asm/idtentry.h] Signed-off-by: hanliyang --- arch/x86/include/asm/idtentry.h | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 2adfd80ea2f3..61d05ff1dc2c 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -57,6 +57,49 @@ __visible noinstr void func(struct pt_regs *regs) \ \ static __always_inline void __##func(struct pt_regs *regs) +/** + * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points + * Error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the + * C-handler. + */ +#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ + asmlinkage void asm_##func(void); \ + asmlinkage void xen_asm_##func(void); \ + __visible void func(struct pt_regs *regs, unsigned long error_code) + +/** + * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points + * Error code pushed by hardware + * @func: Function name of the entry point + * + * Same as DEFINE_IDTENTRY, but has an extra error_code argument + */ +#define DEFINE_IDTENTRY_ERRORCODE(func) \ +static __always_inline void __##func(struct pt_regs *regs, \ + unsigned long error_code); \ + \ +__visible noinstr void func(struct pt_regs *regs, \ + unsigned long error_code) \ +{ \ + idtentry_enter(regs); \ + instrumentation_begin(); \ + __##func (regs, error_code); \ + instrumentation_end(); \ + idtentry_exit(regs); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs, \ + unsigned long error_code) + #else /* !__ASSEMBLY__ */ /* @@ -65,6 +108,9 @@ static __always_inline void __##func(struct pt_regs *regs) #define DECLARE_IDTENTRY(vector, func) \ idtentry vector asm_##func func has_error_code=0 sane=1 +#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ + idtentry vector asm_##func func has_error_code=1 sane=1 + #endif /* __ASSEMBLY__ */ #endif -- Gitee From c765471feac5789150dde6d51d020a7457fef71e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 4 Mar 2020 15:22:09 +0100 Subject: [PATCH 025/113] x86/idtentry: Provide IDTENTRY_RAW commit 0dc6cdc21b94eed8cdacf34eabb4175cebd13775 upstream. Some exception handlers need to do extra work before any of the entry helpers are invoked. Provide IDTENTRY_RAW for this. Hygon-SIG: commit 0dc6cdc21b94 upstream x86/idtentry: Provide IDTENTRY_RAW Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Acked-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20200505135313.830540017@linutronix.de [hanliyang: Fix conflict on arch/x86/include/asm/idtentry.h] Signed-off-by: hanliyang --- arch/x86/include/asm/idtentry.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 61d05ff1dc2c..32986eadccd6 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -100,6 +100,34 @@ __visible noinstr void func(struct pt_regs *regs, \ static __always_inline void __##func(struct pt_regs *regs, \ unsigned long error_code) +/** + * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points + * No error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY(). + */ +#define DECLARE_IDTENTRY_RAW(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +/** + * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * Contrary to DEFINE_IDTENTRY() this does not invoke the + * idtentry_enter/exit() helpers before and after the body invocation. This + * needs to be done in the body itself if applicable. Use if extra work + * is required before the enter/exit() helpers are invoked. + */ +#define DEFINE_IDTENTRY_RAW(func) \ +__visible noinstr void func(struct pt_regs *regs) + #else /* !__ASSEMBLY__ */ /* @@ -111,6 +139,9 @@ static __always_inline void __##func(struct pt_regs *regs, \ #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ idtentry vector asm_##func func has_error_code=1 sane=1 +#define DECLARE_IDTENTRY_RAW(vector, func) \ + DECLARE_IDTENTRY(vector, func) + #endif /* __ASSEMBLY__ */ #endif -- Gitee From f039d9dc9316d7d7d41b49604fafc27f04f130e4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:33:22 +0100 Subject: [PATCH 026/113] x86/idtentry: Provide IDTENTRY_IST commit 2c058b03cc06ba485169778a271f87e5ac57dd83 upstream. Same as IDTENTRY but for exceptions which run on Interrupt Stacks (IST) on 64bit. For 32bit this maps to IDTENTRY. There are 3 variants which will be used: IDTENTRY_MCE IDTENTRY_DB IDTENTRY_NMI These map to IDTENTRY_IST, but only the MCE and DB variants are emitting ASM code as the NMI entry needs hand crafted ASM still. The function defines do not contain any idtenter/exit calls as these exceptions need special treatment. Hygon-SIG: commit 2c058b03cc06 upstream x86/idtentry: Provide IDTENTRY_IST Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Acked-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20200505135314.137125609@linutronix.de [hanliyang: Fix conflict on arch/x86/include/asm/idtentry.h] Signed-off-by: hanliyang --- arch/x86/include/asm/idtentry.h | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 32986eadccd6..8a08e3ca8443 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -128,6 +128,42 @@ static __always_inline void __##func(struct pt_regs *regs, \ #define DEFINE_IDTENTRY_RAW(func) \ __visible noinstr void func(struct pt_regs *regs) +#ifdef CONFIG_X86_64 +/** + * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW + */ +#define DECLARE_IDTENTRY_IST(vector, func) \ + DECLARE_IDTENTRY_RAW(vector, func) + +/** + * DEFINE_IDTENTRY_IST - Emit code for IST entry points + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW + */ +#define DEFINE_IDTENTRY_IST(func) \ + DEFINE_IDTENTRY_RAW(func) + +#else /* CONFIG_X86_64 */ +/* Maps to a regular IDTENTRY on 32bit for now */ +# define DECLARE_IDTENTRY_IST DECLARE_IDTENTRY +# define DEFINE_IDTENTRY_IST DEFINE_IDTENTRY +#endif /* !CONFIG_X86_64 */ + +/* C-Code mapping */ +#define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST +#define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST + +#define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_IST +#define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_IST + +#define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST +#define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST + #else /* !__ASSEMBLY__ */ /* @@ -142,6 +178,24 @@ __visible noinstr void func(struct pt_regs *regs) #define DECLARE_IDTENTRY_RAW(vector, func) \ DECLARE_IDTENTRY(vector, func) +#ifdef CONFIG_X86_64 +# define DECLARE_IDTENTRY_MCE(vector, func) \ + idtentry_mce_db vector asm_##func func + +# define DECLARE_IDTENTRY_DEBUG(vector, func) \ + idtentry_mce_db vector asm_##func func + +#else +# define DECLARE_IDTENTRY_MCE(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +# define DECLARE_IDTENTRY_DEBUG(vector, func) \ + DECLARE_IDTENTRY(vector, func) +#endif + +/* No ASM code emitted for NMI */ +#define DECLARE_IDTENTRY_NMI(vector, func) + #endif /* __ASSEMBLY__ */ #endif -- Gitee From 33904bf979aa81f3b903af835f68f0d1a3b46a1f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:33:28 +0100 Subject: [PATCH 027/113] x86/idtentry: Provide IDTRENTRY_NOIST variants for #DB and #MC commit f08e32ec3cfcf9e6d3640007de590c225ab2e201 upstream. Provide NOIST entry point macros which allows to implement NOIST variants of the C entry points. These are invoked when #DB or #MC enter from user space. This allows explicit handling of the difference between user mode and kernel mode entry later. Hygon-SIG: commit f08e32ec3cfc upstream x86/idtentry: Provide IDTRENTRY_NOIST variants for #DB and #MC Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Acked-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20200505135315.084882104@linutronix.de [hanliyang: Fix conflict on arch/x86/include/asm/idtentry.h] Signed-off-by: hanliyang --- arch/x86/include/asm/idtentry.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 8a08e3ca8443..bf0944915f0f 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -134,10 +134,12 @@ __visible noinstr void func(struct pt_regs *regs) * @vector: Vector number (ignored for C) * @func: Function name of the entry point * - * Maps to DECLARE_IDTENTRY_RAW + * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler + * which is called from the ASM entry point on user mode entry */ #define DECLARE_IDTENTRY_IST(vector, func) \ - DECLARE_IDTENTRY_RAW(vector, func) + DECLARE_IDTENTRY_RAW(vector, func); \ + __visible void noist_##func(struct pt_regs *regs) /** * DEFINE_IDTENTRY_IST - Emit code for IST entry points @@ -148,6 +150,17 @@ __visible noinstr void func(struct pt_regs *regs) #define DEFINE_IDTENTRY_IST(func) \ DEFINE_IDTENTRY_RAW(func) +/** + * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which + * belong to a IST entry point (MCE, DB) + * @func: Function name of the entry point. Must be the same as + * the function name of the corresponding IST variant + * + * Maps to DEFINE_IDTENTRY_RAW(). + */ +#define DEFINE_IDTENTRY_NOIST(func) \ + DEFINE_IDTENTRY_RAW(noist_##func) + #else /* CONFIG_X86_64 */ /* Maps to a regular IDTENTRY on 32bit for now */ # define DECLARE_IDTENTRY_IST DECLARE_IDTENTRY @@ -157,12 +170,14 @@ __visible noinstr void func(struct pt_regs *regs) /* C-Code mapping */ #define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST #define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST +#define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST #define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_IST #define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_IST #define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST #define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST +#define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST #else /* !__ASSEMBLY__ */ -- Gitee From a0551b685ed37280412f208d93975e8be1289ca4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 25 Feb 2020 23:33:30 +0100 Subject: [PATCH 028/113] x86/idtentry: Provide IDTENTRY_DF commit 6a8dfa8e4053adfcf02ee4d96287943064166beb upstream. Provide a separate macro for #DF as this needs to emit paranoid only code and has also a special ASM stub in 32bit. Hygon-SIG: commit 6a8dfa8e4053 upstream x86/idtentry: Provide IDTENTRY_DF Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Acked-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20200505135315.583415264@linutronix.de Signed-off-by: hanliyang --- arch/x86/include/asm/idtentry.h | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index bf0944915f0f..55550afb852f 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -128,6 +128,35 @@ static __always_inline void __##func(struct pt_regs *regs, \ #define DEFINE_IDTENTRY_RAW(func) \ __visible noinstr void func(struct pt_regs *regs) +/** + * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points + * Error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_ERRORCODE() + */ +#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ + DECLARE_IDTENTRY_ERRORCODE(vector, func) + +/** + * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the + * idtentry_enter/exit() helpers before and after the body invocation. This + * needs to be done in the body itself if applicable. Use if extra work + * is required before the enter/exit() helpers are invoked. + */ +#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \ +__visible noinstr void func(struct pt_regs *regs, unsigned long error_code) + + #ifdef CONFIG_X86_64 /** * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points @@ -161,10 +190,58 @@ __visible noinstr void func(struct pt_regs *regs) #define DEFINE_IDTENTRY_NOIST(func) \ DEFINE_IDTENTRY_RAW(noist_##func) +/** + * DECLARE_IDTENTRY_DF - Declare functions for double fault + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE + */ +#define DECLARE_IDTENTRY_DF(vector, func) \ + DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) + +/** + * DEFINE_IDTENTRY_DF - Emit code for double fault + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_DF(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(func) + #else /* CONFIG_X86_64 */ + /* Maps to a regular IDTENTRY on 32bit for now */ # define DECLARE_IDTENTRY_IST DECLARE_IDTENTRY # define DEFINE_IDTENTRY_IST DEFINE_IDTENTRY + +/** + * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares two functions: + * - The ASM entry point: asm_##func + * - The C handler called from the C shim + */ +#define DECLARE_IDTENTRY_DF(vector, func) \ + asmlinkage void asm_##func(void); \ + __visible void func(struct pt_regs *regs, \ + unsigned long error_code, \ + unsigned long address) + +/** + * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit + * @func: Function name of the entry point + * + * This is called through the doublefault shim which already provides + * cr2 in the address argument. + */ +#define DEFINE_IDTENTRY_DF(func) \ +__visible noinstr void func(struct pt_regs *regs, \ + unsigned long error_code, \ + unsigned long address) + #endif /* !CONFIG_X86_64 */ /* C-Code mapping */ @@ -193,6 +270,9 @@ __visible noinstr void func(struct pt_regs *regs) #define DECLARE_IDTENTRY_RAW(vector, func) \ DECLARE_IDTENTRY(vector, func) +#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ + DECLARE_IDTENTRY_ERRORCODE(vector, func) + #ifdef CONFIG_X86_64 # define DECLARE_IDTENTRY_MCE(vector, func) \ idtentry_mce_db vector asm_##func func @@ -200,12 +280,19 @@ __visible noinstr void func(struct pt_regs *regs) # define DECLARE_IDTENTRY_DEBUG(vector, func) \ idtentry_mce_db vector asm_##func func +# define DECLARE_IDTENTRY_DF(vector, func) \ + idtentry_df vector asm_##func func + #else # define DECLARE_IDTENTRY_MCE(vector, func) \ DECLARE_IDTENTRY(vector, func) # define DECLARE_IDTENTRY_DEBUG(vector, func) \ DECLARE_IDTENTRY(vector, func) + +/* No ASM emitted for DF as this goes through a C shim */ +# define DECLARE_IDTENTRY_DF(vector, func) + #endif /* No ASM code emitted for NMI */ -- Gitee From f8ccc3245541ab6d5d470b07c61d4997b979237a Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 25 May 2020 23:22:06 -0700 Subject: [PATCH 029/113] KVM: SVM: fix svn_pin_memory()'s use of get_user_pages_fast() commit 78824fabc72e5e37d51e6e567fde70a4fc41a6d7 upstream. There are two problems in svn_pin_memory(): 1) The return value of get_user_pages_fast() is stored in an unsigned long, although the declared return value is of type int. This will not cause any symptoms, but it is misleading. Fix this by changing the type of npinned to "int". 2) The number of pages passed into get_user_pages_fast() is stored in an unsigned long, even though get_user_pages_fast() accepts an int. This means that it is possible to silently overflow the number of pages. Fix this by adding a WARN_ON_ONCE() and an early error return. The npages variable is left as an unsigned long for convenience in checking for overflow. Hygon-SIG: commit 78824fabc72e upstream KVM: SVM: fix svn_pin_memory()'s use of get_user_pages_fast() Fixes: 89c505809052 ("KVM: SVM: Add support for KVM_SEV_LAUNCH_UPDATE_DATA command") Cc: Ingo Molnar Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Vitaly Kuznetsov Cc: Wanpeng Li Cc: Jim Mattson Cc: Joerg Roedel Cc: H. Peter Anvin Cc: x86@kernel.org Cc: kvm@vger.kernel.org Signed-off-by: John Hubbard Message-Id: <20200526062207.1360225-2-jhubbard@nvidia.com> Signed-off-by: Paolo Bonzini [hanliyang: Fix conflict on arch/x86/kvm/svm/svm.c] Signed-off-by: hanliyang --- arch/x86/kvm/svm/svm.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 033b7bb1c476..47b91a6e2ac8 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2025,7 +2025,8 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, int write) { struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - unsigned long npages, npinned, size; + unsigned long npages, size; + int npinned; unsigned long locked, lock_limit; struct page **pages; unsigned long first, last; @@ -2047,6 +2048,9 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, return NULL; } + if (WARN_ON_ONCE(npages > INT_MAX)) + return NULL; + /* Avoid using vmalloc for smaller buffers. */ size = npages * sizeof(struct page *); if (size > PAGE_SIZE) -- Gitee From d0e76b0a85698f58b3a1180e1b2fab1c9f49f4b0 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:06 +0200 Subject: [PATCH 030/113] x86/cpufeatures: Add SEV-ES CPU feature commit 360e7c5c4ca4fd8e627781ed42f95d58bc3bb732 upstream. Add CPU feature detection for Secure Encrypted Virtualization with Encrypted State. This feature enhances SEV by also encrypting the guest register state, making it in-accessible to the hypervisor. Hygon-SIG: commit 360e7c5c4ca4 upstream x86/cpufeatures: Add SEV-ES CPU feature Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-6-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/cpu/amd.c, arch/x86/kernel/cpu/scattered.c] Signed-off-by: hanliyang --- arch/x86/kernel/cpu/amd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index b953bfc739b3..13b298e69be8 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -649,7 +649,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) * If BIOS has not enabled SME then don't advertise the * SME feature (set in scattered.c). * For SEV: If BIOS has not enabled SEV then don't advertise the - * SEV feature (set in scattered.c). + * SEV and SEV_ES feature (set in scattered.c). * * In all cases, since support for SME and SEV requires long mode, * don't advertise the feature under CONFIG_X86_32. @@ -680,6 +680,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) setup_clear_cpu_cap(X86_FEATURE_SME); clear_sev: setup_clear_cpu_cap(X86_FEATURE_SEV); + setup_clear_cpu_cap(X86_FEATURE_SEV_ES); } } -- Gitee From 2ad659a0cd788d7eaa5cacc9afa5f00daf4e57c0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:07 +0200 Subject: [PATCH 031/113] x86/traps: Move pf error codes to commit 05a2fdf3230306daee1def019b8f52cd06bd2e48 upstream. Move the definition of the x86 page-fault error code bits to a new header file asm/trap_pf.h. This makes it easier to include them into pre-decompression boot code. No functional changes. Hygon-SIG: commit 05a2fdf32303 upstream x86/traps: Move pf error codes to Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-7-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/trap_pf.h, arch/x86/include/asm/traps.h] Signed-off-by: hanliyang --- arch/x86/include/asm/trap_pf.h | 26 ++++++++++++++++++++++++++ arch/x86/include/asm/traps.h | 21 +-------------------- 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 arch/x86/include/asm/trap_pf.h diff --git a/arch/x86/include/asm/trap_pf.h b/arch/x86/include/asm/trap_pf.h new file mode 100644 index 000000000000..10b1de500ab1 --- /dev/null +++ b/arch/x86/include/asm/trap_pf.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_TRAP_PF_H +#define _ASM_X86_TRAP_PF_H + +/* + * Page fault error code bits: + * + * bit 0 == 0: no page found 1: protection fault + * bit 1 == 0: read access 1: write access + * bit 2 == 0: kernel-mode access 1: user-mode access + * bit 3 == 1: use of reserved bit detected + * bit 4 == 1: fault was an instruction fetch + * bit 5 == 1: protection keys block access + * bit 15 == 1: SGX MMU page-fault + */ +enum x86_pf_error_code { + X86_PF_PROT = 1 << 0, + X86_PF_WRITE = 1 << 1, + X86_PF_USER = 1 << 2, + X86_PF_RSVD = 1 << 3, + X86_PF_INSTR = 1 << 4, + X86_PF_PK = 1 << 5, + X86_PF_SGX = 1 << 15, +}; + +#endif /* _ASM_X86_TRAP_PF_H */ diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 6aeb4af71291..df9eea3ec672 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -8,6 +8,7 @@ #include #include #include /* TRAP_TRACE, ... */ +#include #define dotraplinkage __visible @@ -127,24 +128,4 @@ void __noreturn handle_stack_overflow(const char *message, unsigned long fault_address); #endif -/* - * Page fault error code bits: - * - * bit 0 == 0: no page found 1: protection fault - * bit 1 == 0: read access 1: write access - * bit 2 == 0: kernel-mode access 1: user-mode access - * bit 3 == 1: use of reserved bit detected - * bit 4 == 1: fault was an instruction fetch - * bit 5 == 1: protection keys block access - * bit 15 == 1: SGX MMU page-fault - */ -enum x86_pf_error_code { - X86_PF_PROT = 1 << 0, - X86_PF_WRITE = 1 << 1, - X86_PF_USER = 1 << 2, - X86_PF_RSVD = 1 << 3, - X86_PF_INSTR = 1 << 4, - X86_PF_PK = 1 << 5, - X86_PF_SGX = 1 << 15, -}; #endif /* _ASM_X86_TRAPS_H */ -- Gitee From 26d46db35d4525ea2c52878ac87afece09f4d438 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:08 +0200 Subject: [PATCH 032/113] x86/insn: Make inat-tables.c suitable for pre-decompression code commit 05a2ae7c033ee30f25fbed3ceed549a5cac398a9 upstream. The inat-tables.c file has some arrays in it that contain pointers to other arrays. These pointers need to be relocated when the kernel image is moved to a different location. The pre-decompression boot-code has no support for applying ELF relocations, so initialize these arrays at runtime in the pre-decompression code to make sure all pointers are correctly initialized. Hygon-SIG: commit 05a2ae7c033e upstream x86/insn: Make inat-tables.c suitable for pre-decompression code Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-8-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/tools/gen-insn-attr-x86.awk | 50 +++++++++++++++++++++- tools/arch/x86/tools/gen-insn-attr-x86.awk | 50 +++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index a42015b305f4..af38469afd14 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -362,6 +362,9 @@ function convert_operands(count,opnd, i,j,imm,mod) END { if (awkchecked != "") exit 1 + + print "#ifndef __BOOT_COMPRESSED\n" + # print escape opcode map's array print "/* Escape opcode map array */" print "const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1]" \ @@ -388,6 +391,51 @@ END { for (j = 0; j < max_lprefix; j++) if (atable[i,j]) print " ["i"]["j"] = "atable[i,j]"," - print "};" + print "};\n" + + print "#else /* !__BOOT_COMPRESSED */\n" + + print "/* Escape opcode map array */" + print "static const insn_attr_t *inat_escape_tables[INAT_ESC_MAX + 1]" \ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* Group opcode map array */" + print "static const insn_attr_t *inat_group_tables[INAT_GRP_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* AVX opcode map array */" + print "static const insn_attr_t *inat_avx_tables[X86_VEX_M_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "static void inat_init_tables(void)" + print "{" + + # print escape opcode map's array + print "\t/* Print Escape opcode map array */" + for (i = 0; i < geid; i++) + for (j = 0; j < max_lprefix; j++) + if (etable[i,j]) + print "\tinat_escape_tables["i"]["j"] = "etable[i,j]";" + print "" + + # print group opcode map's array + print "\t/* Print Group opcode map array */" + for (i = 0; i < ggid; i++) + for (j = 0; j < max_lprefix; j++) + if (gtable[i,j]) + print "\tinat_group_tables["i"]["j"] = "gtable[i,j]";" + print "" + # print AVX opcode map's array + print "\t/* Print AVX opcode map array */" + for (i = 0; i < gaid; i++) + for (j = 0; j < max_lprefix; j++) + if (atable[i,j]) + print "\tinat_avx_tables["i"]["j"] = "atable[i,j]";" + + print "}" + print "#endif" } diff --git a/tools/arch/x86/tools/gen-insn-attr-x86.awk b/tools/arch/x86/tools/gen-insn-attr-x86.awk index a42015b305f4..af38469afd14 100644 --- a/tools/arch/x86/tools/gen-insn-attr-x86.awk +++ b/tools/arch/x86/tools/gen-insn-attr-x86.awk @@ -362,6 +362,9 @@ function convert_operands(count,opnd, i,j,imm,mod) END { if (awkchecked != "") exit 1 + + print "#ifndef __BOOT_COMPRESSED\n" + # print escape opcode map's array print "/* Escape opcode map array */" print "const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1]" \ @@ -388,6 +391,51 @@ END { for (j = 0; j < max_lprefix; j++) if (atable[i,j]) print " ["i"]["j"] = "atable[i,j]"," - print "};" + print "};\n" + + print "#else /* !__BOOT_COMPRESSED */\n" + + print "/* Escape opcode map array */" + print "static const insn_attr_t *inat_escape_tables[INAT_ESC_MAX + 1]" \ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* Group opcode map array */" + print "static const insn_attr_t *inat_group_tables[INAT_GRP_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* AVX opcode map array */" + print "static const insn_attr_t *inat_avx_tables[X86_VEX_M_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "static void inat_init_tables(void)" + print "{" + + # print escape opcode map's array + print "\t/* Print Escape opcode map array */" + for (i = 0; i < geid; i++) + for (j = 0; j < max_lprefix; j++) + if (etable[i,j]) + print "\tinat_escape_tables["i"]["j"] = "etable[i,j]";" + print "" + + # print group opcode map's array + print "\t/* Print Group opcode map array */" + for (i = 0; i < ggid; i++) + for (j = 0; j < max_lprefix; j++) + if (gtable[i,j]) + print "\tinat_group_tables["i"]["j"] = "gtable[i,j]";" + print "" + # print AVX opcode map's array + print "\t/* Print AVX opcode map array */" + for (i = 0; i < gaid; i++) + for (j = 0; j < max_lprefix; j++) + if (atable[i,j]) + print "\tinat_avx_tables["i"]["j"] = "atable[i,j]";" + + print "}" + print "#endif" } -- Gitee From 16abcc37330011cd5050feb7d120ee6efc9b029d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:09 +0200 Subject: [PATCH 033/113] x86/umip: Factor out instruction fetch commit 172b75e56b08846e6fb07a88e5685ce4e24f4620 upstream. Factor out the code to fetch the instruction from user-space to a helper function. No functional changes. Hygon-SIG: commit 172b75e56b08 upstream x86/umip: Factor out instruction fetch Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-9-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/insn-eval.h | 2 ++ arch/x86/kernel/umip.c | 26 +++++----------------- arch/x86/lib/insn-eval.c | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 2b6ccf2c49f1..b8b9ef1bbd06 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -19,5 +19,7 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); +int insn_fetch_from_user(struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE]); #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index 548fefed71ee..f3d067b51e08 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -317,11 +317,11 @@ static void force_sig_info_umip_fault(void __user *addr, struct pt_regs *regs) */ bool fixup_umip_exception(struct pt_regs *regs) { - int not_copied, nr_copied, reg_offset, dummy_data_size, umip_inst; - unsigned long seg_base = 0, *reg_addr; + int nr_copied, reg_offset, dummy_data_size, umip_inst; /* 10 bytes is the maximum size of the result of UMIP instructions */ unsigned char dummy_data[10] = { 0 }; unsigned char buf[MAX_INSN_SIZE]; + unsigned long *reg_addr; void __user *uaddr; struct insn insn; int seg_defs; @@ -329,26 +329,12 @@ bool fixup_umip_exception(struct pt_regs *regs) if (!regs) return false; - /* - * If not in user-space long mode, a custom code segment could be in - * use. This is true in protected mode (if the process defined a local - * descriptor table), or virtual-8086 mode. In most of the cases - * seg_base will be zero as in USER_CS. - */ - if (!user_64bit_mode(regs)) - seg_base = insn_get_seg_base(regs, INAT_SEG_REG_CS); - - if (seg_base == -1L) - return false; - - not_copied = copy_from_user(buf, (void __user *)(seg_base + regs->ip), - sizeof(buf)); - nr_copied = sizeof(buf) - not_copied; + nr_copied = insn_fetch_from_user(regs, buf); /* - * The copy_from_user above could have failed if user code is protected - * by a memory protection key. Give up on emulation in such a case. - * Should we issue a page fault? + * The insn_fetch_from_user above could have failed if user code + * is protected by a memory protection key. Give up on emulation + * in such a case. Should we issue a page fault? */ if (!nr_copied) return false; diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index ecdd7b5beda5..26a9c2b9826c 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -1370,3 +1370,41 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs) return (void __user *)-1L; } } + +/** + * insn_fetch_from_user() - Copy instruction bytes from user-space memory + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Array to store the fetched instruction + * + * Gets the linear address of the instruction and copies the instruction bytes + * to the buf. + * + * Returns: + * + * Number of instruction bytes copied. + * + * 0 if nothing was copied. + */ +int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) +{ + unsigned long seg_base = 0; + int not_copied; + + /* + * If not in user-space long mode, a custom code segment could be in + * use. This is true in protected mode (if the process defined a local + * descriptor table), or virtual-8086 mode. In most of the cases + * seg_base will be zero as in USER_CS. + */ + if (!user_64bit_mode(regs)) { + seg_base = insn_get_seg_base(regs, INAT_SEG_REG_CS); + if (seg_base == -1L) + return 0; + } + + + not_copied = copy_from_user(buf, (void __user *)(seg_base + regs->ip), + MAX_INSN_SIZE); + + return MAX_INSN_SIZE - not_copied; +} -- Gitee From e846de57fa41d91a8ec75d2e97f0003e255081bb Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:03 +0200 Subject: [PATCH 034/113] KVM: SVM: Add GHCB definitions commit d07f46f9f51afc7fc9f021eae19eba3c2e7870ac upstream. Extend the vmcb_safe_area with SEV-ES fields and add a new 'struct ghcb' which will be used for guest-hypervisor communication. Hygon-SIG: commit d07f46f9f51a upstream KVM: SVM: Add GHCB definitions Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-3-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/svm.h, arch/x86/kvm/svm/svm.c] Signed-off-by: hanliyang --- arch/x86/include/asm/svm.h | 54 ++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 2 ++ 2 files changed, 56 insertions(+) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index c29d8fb0ffbe..ce5da81ace36 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -203,8 +203,62 @@ struct __attribute__ ((__packed__)) vmcb_save_area { u64 br_to; u64 last_excp_from; u64 last_excp_to; + + /* + * The following part of the save area is valid only for + * SEV-ES guests when referenced through the GHCB. + */ + u8 reserved_7[104]; + u64 reserved_8; /* rax already available at 0x01f8 */ + u64 rcx; + u64 rdx; + u64 rbx; + u64 reserved_9; /* rsp already available at 0x01d8 */ + u64 rbp; + u64 rsi; + u64 rdi; + u64 r8; + u64 r9; + u64 r10; + u64 r11; + u64 r12; + u64 r13; + u64 r14; + u64 r15; + u8 reserved_10[16]; + u64 sw_exit_code; + u64 sw_exit_info_1; + u64 sw_exit_info_2; + u64 sw_scratch; + u8 reserved_11[56]; + u64 xcr0; + u8 valid_bitmap[16]; + u64 x87_state_gpa; }; +struct ghcb { + struct vmcb_save_area save; + u8 reserved_save[2048 - sizeof(struct vmcb_save_area)]; + + u8 shared_buffer[2032]; + + u8 reserved_1[10]; + u16 protocol_version; /* negotiated SEV-ES/GHCB protocol version */ + u32 ghcb_usage; +} __packed; + + +#define EXPECTED_VMCB_SAVE_AREA_SIZE 1032 +#define EXPECTED_VMCB_CONTROL_AREA_SIZE 1024 +#define EXPECTED_GHCB_SIZE PAGE_SIZE + +static inline void __unused_size_checks(void) +{ + BUILD_BUG_ON(sizeof(struct vmcb_save_area) != EXPECTED_VMCB_SAVE_AREA_SIZE); + BUILD_BUG_ON(sizeof(struct vmcb_control_area) != EXPECTED_VMCB_CONTROL_AREA_SIZE); + BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE); +} + struct __attribute__ ((__packed__)) vmcb { struct vmcb_control_area control; struct vmcb_save_area save; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 47b91a6e2ac8..89beecad9f08 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -8244,6 +8244,8 @@ void sev_install_hooks(void) static int __init svm_init(void) { + __unused_size_checks(); + /* Register CSV specific interface for Hygon CPUs */ if (is_x86_vendor_hygon()) csv_init(&svm_x86_ops); -- Gitee From ff4c9383a200c2743845cbd2d78f62d37cd380ca Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:10 +0200 Subject: [PATCH 035/113] x86/umip: Factor out instruction decoding commit 172639d79977ca7b5ce6f84f6606262f4081718f upstream. Factor out the code used to decode an instruction with the correct address and operand sizes to a helper function. No functional changes. Hygon-SIG: commit 172639d79977 upstream x86/umip: Factor out instruction decoding Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-10-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/insn-eval.h | 2 ++ arch/x86/kernel/umip.c | 23 +--------------- arch/x86/lib/insn-eval.c | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index b8b9ef1bbd06..392b4fe377f9 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -21,5 +21,7 @@ unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]); +bool insn_decode(struct insn *insn, struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE], int buf_size); #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index f3d067b51e08..8547d338914d 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -324,7 +324,6 @@ bool fixup_umip_exception(struct pt_regs *regs) unsigned long *reg_addr; void __user *uaddr; struct insn insn; - int seg_defs; if (!regs) return false; @@ -339,27 +338,7 @@ bool fixup_umip_exception(struct pt_regs *regs) if (!nr_copied) return false; - insn_init(&insn, buf, nr_copied, user_64bit_mode(regs)); - - /* - * Override the default operand and address sizes with what is specified - * in the code segment descriptor. The instruction decoder only sets - * the address size it to either 4 or 8 address bytes and does nothing - * for the operand bytes. This OK for most of the cases, but we could - * have special cases where, for instance, a 16-bit code segment - * descriptor is used. - * If there is an address override prefix, the instruction decoder - * correctly updates these values, even for 16-bit defaults. - */ - seg_defs = insn_get_code_seg_params(regs); - if (seg_defs == -EINVAL) - return false; - - insn.addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs); - insn.opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs); - - insn_get_length(&insn); - if (nr_copied < insn.length) + if (!insn_decode(&insn, regs, buf, nr_copied)) return false; umip_inst = identify_insn(&insn); diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 26a9c2b9826c..650153ad89c4 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -1408,3 +1408,48 @@ int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) return MAX_INSN_SIZE - not_copied; } + +/** + * insn_decode() - Decode an instruction + * @insn: Structure to store decoded instruction + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Buffer containing the instruction bytes + * @buf_size: Number of instruction bytes available in buf + * + * Decodes the instruction provided in buf and stores the decoding results in + * insn. Also determines the correct address and operand sizes. + * + * Returns: + * + * True if instruction was decoded, False otherwise. + */ +bool insn_decode(struct insn *insn, struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE], int buf_size) +{ + int seg_defs; + + insn_init(insn, buf, buf_size, user_64bit_mode(regs)); + + /* + * Override the default operand and address sizes with what is specified + * in the code segment descriptor. The instruction decoder only sets + * the address size it to either 4 or 8 address bytes and does nothing + * for the operand bytes. This OK for most of the cases, but we could + * have special cases where, for instance, a 16-bit code segment + * descriptor is used. + * If there is an address override prefix, the instruction decoder + * correctly updates these values, even for 16-bit defaults. + */ + seg_defs = insn_get_code_seg_params(regs); + if (seg_defs == -EINVAL) + return false; + + insn->addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs); + insn->opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs); + + insn_get_length(insn); + if (buf_size < insn->length) + return false; + + return true; +} -- Gitee From 82348177654fd020aa8c0c8da1776f56f7c5b976 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:11 +0200 Subject: [PATCH 036/113] x86/insn: Add insn_get_modrm_reg_off() commit 7af1bd822dd45a669fc178a35cc8183922333d56 upstream. Add a function to the instruction decoder which returns the pt_regs offset of the register specified in the reg field of the modrm byte. Hygon-SIG: commit 7af1bd822dd4 upstream x86/insn: Add insn_get_modrm_reg_off() Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-11-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/insn-eval.h | 1 + arch/x86/lib/insn-eval.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 392b4fe377f9..f748f57f1491 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -17,6 +17,7 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); +int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs); unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); int insn_fetch_from_user(struct pt_regs *regs, diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 650153ad89c4..bbcd79943f46 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -20,6 +20,7 @@ enum reg_type { REG_TYPE_RM = 0, + REG_TYPE_REG, REG_TYPE_INDEX, REG_TYPE_BASE, }; @@ -442,6 +443,13 @@ static int get_reg_offset(struct insn *insn, struct pt_regs *regs, regno += 8; break; + case REG_TYPE_REG: + regno = X86_MODRM_REG(insn->modrm.value); + + if (X86_REX_R(insn->rex_prefix.value)) + regno += 8; + break; + case REG_TYPE_INDEX: regno = X86_SIB_INDEX(insn->sib.value); if (X86_REX_X(insn->rex_prefix.value)) @@ -810,6 +818,21 @@ int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs) return get_reg_offset(insn, regs, REG_TYPE_RM); } +/** + * insn_get_modrm_reg_off() - Obtain register in reg part of the ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the reg part of the ModRM byte. The + * register is obtained as an offset from the base of pt_regs. + */ +int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs) +{ + return get_reg_offset(insn, regs, REG_TYPE_REG); +} + /** * get_seg_base_limit() - obtain base address and limit of a segment * @insn: Instruction. Must be valid. -- Gitee From fe514d2b399cfbb2903ee86e741ef3022e60ca97 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:04 +0200 Subject: [PATCH 037/113] KVM: SVM: Add GHCB Accessor functions commit 3702c2f4eed2188440f65ecdfc89165106fe565d upstream. Building a correct GHCB for the hypervisor requires setting valid bits in the GHCB. Simplify that process by providing accessor functions to set values and to update the valid bitmap and to check the valid bitmap in KVM. Hygon-SIG: commit 3702c2f4eed2 upstream KVM: SVM: Add GHCB Accessor functions Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-4-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/svm.h | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index ce5da81ace36..b4210192619c 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -347,4 +347,47 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP) +/* GHCB Accessor functions */ + +#define GHCB_BITMAP_IDX(field) \ + (offsetof(struct vmcb_save_area, field) / sizeof(u64)) + +#define DEFINE_GHCB_ACCESSORS(field) \ + static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \ + { \ + return test_bit(GHCB_BITMAP_IDX(field), \ + (unsigned long *)&ghcb->save.valid_bitmap); \ + } \ + \ + static inline void ghcb_set_##field(struct ghcb *ghcb, u64 value) \ + { \ + __set_bit(GHCB_BITMAP_IDX(field), \ + (unsigned long *)&ghcb->save.valid_bitmap); \ + ghcb->save.field = value; \ + } + +DEFINE_GHCB_ACCESSORS(cpl) +DEFINE_GHCB_ACCESSORS(rip) +DEFINE_GHCB_ACCESSORS(rsp) +DEFINE_GHCB_ACCESSORS(rax) +DEFINE_GHCB_ACCESSORS(rcx) +DEFINE_GHCB_ACCESSORS(rdx) +DEFINE_GHCB_ACCESSORS(rbx) +DEFINE_GHCB_ACCESSORS(rbp) +DEFINE_GHCB_ACCESSORS(rsi) +DEFINE_GHCB_ACCESSORS(rdi) +DEFINE_GHCB_ACCESSORS(r8) +DEFINE_GHCB_ACCESSORS(r9) +DEFINE_GHCB_ACCESSORS(r10) +DEFINE_GHCB_ACCESSORS(r11) +DEFINE_GHCB_ACCESSORS(r12) +DEFINE_GHCB_ACCESSORS(r13) +DEFINE_GHCB_ACCESSORS(r14) +DEFINE_GHCB_ACCESSORS(r15) +DEFINE_GHCB_ACCESSORS(sw_exit_code) +DEFINE_GHCB_ACCESSORS(sw_exit_info_1) +DEFINE_GHCB_ACCESSORS(sw_exit_info_2) +DEFINE_GHCB_ACCESSORS(sw_scratch) +DEFINE_GHCB_ACCESSORS(xcr0) + #endif -- Gitee From 3d64f7206ea28ddb83d4944f5f13864bdca5e3c4 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 7 Sep 2020 15:15:05 +0200 Subject: [PATCH 038/113] KVM: SVM: Use __packed shorthand commit 976bc5e2aceedef13e0ba1f0e6e372a22164aa0c upstream. Use the shorthand to make it more readable. No functional changes. Hygon-SIG: commit 976bc5e2acee upstream KVM: SVM: Use __packed shorthand Signed-off-by: Borislav Petkov Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-5-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/svm.h] Signed-off-by: hanliyang --- arch/x86/include/asm/svm.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index b4210192619c..cb547c8226d3 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -153,14 +153,14 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_NESTED_CTL_NP_ENABLE BIT(0) #define SVM_NESTED_CTL_SEV_ENABLE BIT(1) -struct __attribute__ ((__packed__)) vmcb_seg { +struct vmcb_seg { u16 selector; u16 attrib; u32 limit; u64 base; -}; +} __packed; -struct __attribute__ ((__packed__)) vmcb_save_area { +struct vmcb_save_area { struct vmcb_seg es; struct vmcb_seg cs; struct vmcb_seg ss; @@ -234,7 +234,7 @@ struct __attribute__ ((__packed__)) vmcb_save_area { u64 xcr0; u8 valid_bitmap[16]; u64 x87_state_gpa; -}; +} __packed; struct ghcb { struct vmcb_save_area save; @@ -259,10 +259,10 @@ static inline void __unused_size_checks(void) BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE); } -struct __attribute__ ((__packed__)) vmcb { +struct vmcb { struct vmcb_control_area control; struct vmcb_save_area save; -}; +} __packed; #define SVM_CPUID_FUNC 0x8000000a -- Gitee From 02004f713107dc2e197ab7f56aab64a397fd6b11 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:12 +0200 Subject: [PATCH 039/113] x86/insn: Add insn_has_rep_prefix() helper commit 5901781a11175a5e5ee91746ec8627f18d47eebd upstream. Add a function to check whether an instruction has a REP prefix. Hygon-SIG: commit 5901781a1117 upstream x86/insn: Add insn_has_rep_prefix() helper Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-12-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/insn-eval.h | 1 + arch/x86/lib/insn-eval.c | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index f748f57f1491..a0f839aa144d 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -15,6 +15,7 @@ #define INSN_CODE_SEG_OPND_SZ(params) (params & 0xf) #define INSN_CODE_SEG_PARAMS(oper_sz, addr_sz) (oper_sz | (addr_sz << 4)) +bool insn_has_rep_prefix(struct insn *insn); void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs); diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index bbcd79943f46..748063c410c4 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -53,6 +53,30 @@ static bool is_string_insn(struct insn *insn) } } +/** + * insn_has_rep_prefix() - Determine if instruction has a REP prefix + * @insn: Instruction containing the prefix to inspect + * + * Returns: + * + * true if the instruction has a REP prefix, false if not. + */ +bool insn_has_rep_prefix(struct insn *insn) +{ + int i; + + insn_get_prefixes(insn); + + for (i = 0; i < insn->prefixes.nbytes; i++) { + insn_byte_t p = insn->prefixes.bytes[i]; + + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + /** * get_seg_reg_override_idx() - obtain segment register override index * @insn: Valid instruction with segment override prefixes -- Gitee From de8bc99fec9f038c3664c5ffb544764e58a9100b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:13 +0200 Subject: [PATCH 040/113] x86/boot/compressed/64: Disable red-zone usage commit 6ba0efa46047936afa81460489cfd24bc95dd863 upstream. The x86-64 ABI defines a red-zone on the stack: The 128-byte area beyond the location pointed to by %rsp is considered to be reserved and shall not be modified by signal or interrupt handlers. Therefore, functions may use this area for temporary data that is not needed across function calls. In particular, leaf functions may use this area for their entire stack frame, rather than adjusting the stack pointer in the prologue and epilogue. This area is known as the red zone. This is not compatible with exception handling, because the IRET frame written by the hardware at the stack pointer and the functions to handle the exception will overwrite the temporary variables of the interrupted function, causing undefined behavior. So disable red-zones for the pre-decompression boot code. Hygon-SIG: commit 6ba0efa46047 upstream x86/boot/compressed/64: Disable red-zone usage Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-13-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/Makefile] Signed-off-by: hanliyang --- arch/x86/boot/compressed/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 16af006316c8..79fad86aae9e 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -30,7 +30,7 @@ KBUILD_CFLAGS := -m$(BITS) -O2 KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC) KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING cflags-$(CONFIG_X86_32) := -march=i386 -cflags-$(CONFIG_X86_64) := -mcmodel=small +cflags-$(CONFIG_X86_64) := -mcmodel=small -mno-red-zone KBUILD_CFLAGS += $(cflags-y) KBUILD_CFLAGS += -mno-mmx -mno-sse KBUILD_CFLAGS += $(call cc-option,-ffreestanding) -- Gitee From e47143ca58d62b4835aed4644d22e9fbe9a31e0d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:14 +0200 Subject: [PATCH 041/113] x86/boot/compressed/64: Add IDT Infrastructure commit 64e682638eb51070ba6044535b250aad43c5564e upstream. Add code needed to setup an IDT in the early pre-decompression boot-code. The IDT is loaded first in startup_64, which is after EfiExitBootServices() has been called, and later reloaded when the kernel image has been relocated to the end of the decompression area. This allows to setup different IDT handlers before and after the relocation. Hygon-SIG: commit 64e682638eb5 upstream x86/boot/compressed/64: Add IDT Infrastructure Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-14-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/Makefile, arch/x86/boot/compressed/head_64.S, arch/x86/boot/compressed/idt_64.c, arch/x86/boot/compressed/idt_handlers_64.S, arch/x86/boot/compressed/misc.h and 1 more] Signed-off-by: hanliyang --- arch/x86/boot/compressed/Makefile | 1 + arch/x86/boot/compressed/head_64.S | 24 ++++++++ arch/x86/boot/compressed/idt_64.c | 44 ++++++++++++++ arch/x86/boot/compressed/idt_handlers_64.S | 70 ++++++++++++++++++++++ arch/x86/boot/compressed/misc.h | 5 ++ arch/x86/include/asm/desc_defs.h | 3 + 6 files changed, 147 insertions(+) create mode 100644 arch/x86/boot/compressed/idt_64.c create mode 100644 arch/x86/boot/compressed/idt_handlers_64.S diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 79fad86aae9e..c1fa7ba814db 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -87,6 +87,7 @@ vmlinux-objs-$(CONFIG_EARLY_PRINTK) += $(obj)/early_serial_console.o vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr.o ifdef CONFIG_X86_64 vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr_64.o + vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o endif diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index ba30d3da4aba..0395dcab5a4e 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -33,6 +33,7 @@ #include #include #include +#include #include "pgtable.h" /* @@ -346,6 +347,10 @@ ENTRY(startup_64) .Lon_kernel_cs: + pushq %rsi + call load_stage1_idt + popq %rsi + /* * paging_prepare() sets up the trampoline and checks if we need to * enable 5-level paging. @@ -510,6 +515,13 @@ ENDPROC(efi64_stub_entry) shrq $3, %rcx rep stosq +/* + * Load stage2 IDT + */ + pushq %rsi + call load_stage2_idt + popq %rsi + /* * Do the extraction, and jump to the new kernel.. */ @@ -638,6 +650,18 @@ SYM_DATA_START_LOCAL(gdt) .quad 0x0000000000000000 /* TS continued */ SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end) +SYM_DATA_START(boot_idt_desc) + .word boot_idt_end - boot_idt - 1 + .quad 0 +SYM_DATA_END(boot_idt_desc) + .balign 8 +SYM_DATA_START(boot_idt) + .rept BOOT_IDT_ENTRIES + .quad 0 + .quad 0 + .endr +SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBAL, boot_idt_end) + #ifdef CONFIG_EFI_STUB SYM_DATA_LOCAL(efi_config, .quad 0) diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c new file mode 100644 index 000000000000..082cd6bca033 --- /dev/null +++ b/arch/x86/boot/compressed/idt_64.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include "misc.h" + +static void set_idt_entry(int vector, void (*handler)(void)) +{ + unsigned long address = (unsigned long)handler; + gate_desc entry; + + memset(&entry, 0, sizeof(entry)); + + entry.offset_low = (u16)(address & 0xffff); + entry.segment = __KERNEL_CS; + entry.bits.type = GATE_TRAP; + entry.bits.p = 1; + entry.offset_middle = (u16)((address >> 16) & 0xffff); + entry.offset_high = (u32)(address >> 32); + + memcpy(&boot_idt[vector], &entry, sizeof(entry)); +} + +/* Have this here so we don't need to include */ +static void load_boot_idt(const struct desc_ptr *dtr) +{ + asm volatile("lidt %0"::"m" (*dtr)); +} + +/* Setup IDT before kernel jumping to .Lrelocated */ +void load_stage1_idt(void) +{ + boot_idt_desc.address = (unsigned long)boot_idt; + + load_boot_idt(&boot_idt_desc); +} + +/* Setup IDT after kernel jumping to .Lrelocated */ +void load_stage2_idt(void) +{ + boot_idt_desc.address = (unsigned long)boot_idt; + + load_boot_idt(&boot_idt_desc); +} diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S new file mode 100644 index 000000000000..36dee2f40a8b --- /dev/null +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Early IDT handler entry points + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#include + +/* For ORIG_RAX */ +#include "../../entry/calling.h" + +.macro EXCEPTION_HANDLER name function error_code=0 +SYM_FUNC_START(\name) + + /* Build pt_regs */ + .if \error_code == 0 + pushq $0 + .endif + + pushq %rdi + pushq %rsi + pushq %rdx + pushq %rcx + pushq %rax + pushq %r8 + pushq %r9 + pushq %r10 + pushq %r11 + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + /* Call handler with pt_regs */ + movq %rsp, %rdi + /* Error code is second parameter */ + movq ORIG_RAX(%rsp), %rsi + call \function + + /* Restore regs */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + + /* Remove error code and return */ + addq $8, %rsp + + iretq +SYM_FUNC_END(\name) + .endm + + .text + .code64 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index c8181392f70d..1d6300689da0 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -23,6 +23,7 @@ #include #include #include +#include #define BOOT_CTYPE_H #include @@ -133,4 +134,8 @@ int count_immovable_mem_regions(void); static inline int count_immovable_mem_regions(void) { return 0; } #endif +/* idt_64.c */ +extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; +extern struct desc_ptr boot_idt_desc; + #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h index a91f3b6e4f2a..5621fb3f2d1a 100644 --- a/arch/x86/include/asm/desc_defs.h +++ b/arch/x86/include/asm/desc_defs.h @@ -109,6 +109,9 @@ struct desc_ptr { #endif /* !__ASSEMBLY__ */ +/* Boot IDT definitions */ +#define BOOT_IDT_ENTRIES 32 + /* Access rights as returned by LAR */ #define AR_TYPE_RODATA (0 * (1 << 9)) #define AR_TYPE_RWDATA (1 * (1 << 9)) -- Gitee From 45e36d1fa44a3fe8132262f3b9b9a71abda83e7b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:15 +0200 Subject: [PATCH 042/113] x86/boot/compressed/64: Rename kaslr_64.c to ident_map_64.c commit 5f2bb01682b7b067783207994c7b8a3dbeb1cd83 upstream. The file contains only code related to identity-mapped page tables. Rename the file and compile it always in. Hygon-SIG: commit 5f2bb01682b7 upstream x86/boot/compressed/64: Rename kaslr_64.c to ident_map_64.c Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-15-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/Makefile, arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/kaslr.c, arch/x86/boot/compressed/misc.h] Signed-off-by: hanliyang --- arch/x86/boot/compressed/Makefile | 2 +- arch/x86/boot/compressed/{kaslr_64.c => ident_map_64.c} | 9 +++++++++ arch/x86/boot/compressed/kaslr.c | 9 --------- arch/x86/boot/compressed/misc.h | 8 ++++++++ 4 files changed, 18 insertions(+), 10 deletions(-) rename arch/x86/boot/compressed/{kaslr_64.c => ident_map_64.c} (95%) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index c1fa7ba814db..219b426ce287 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -86,7 +86,7 @@ vmlinux-objs-y := $(obj)/vmlinux.lds $(obj)/head_$(BITS).o $(obj)/misc.o \ vmlinux-objs-$(CONFIG_EARLY_PRINTK) += $(obj)/early_serial_console.o vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr.o ifdef CONFIG_X86_64 - vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr_64.o + vmlinux-objs-y += $(obj)/ident_map_64.o vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o diff --git a/arch/x86/boot/compressed/kaslr_64.c b/arch/x86/boot/compressed/ident_map_64.c similarity index 95% rename from arch/x86/boot/compressed/kaslr_64.c rename to arch/x86/boot/compressed/ident_map_64.c index 9557c5a15b91..89a44598ed65 100644 --- a/arch/x86/boot/compressed/kaslr_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -29,6 +29,15 @@ #define __PAGE_OFFSET __PAGE_OFFSET_BASE #include "../../mm/ident_map.c" +#ifdef CONFIG_X86_5LEVEL +unsigned int __pgtable_l5_enabled; +unsigned int pgdir_shift = 39; +unsigned int ptrs_per_p4d = 1; +#endif + +/* Used by PAGE_KERN* macros: */ +pteval_t __default_kernel_pte_mask __read_mostly = ~0; + /* Used to track our page table allocation area. */ struct alloc_pgt_data { unsigned char *pgt_buf; diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 2e53c056ba20..e121bde52f95 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -43,17 +43,8 @@ #define STATIC #include -#ifdef CONFIG_X86_5LEVEL -unsigned int __pgtable_l5_enabled; -unsigned int pgdir_shift __ro_after_init = 39; -unsigned int ptrs_per_p4d __ro_after_init = 1; -#endif - extern unsigned long get_cmd_line_ptr(void); -/* Used by PAGE_KERN* macros: */ -pteval_t __default_kernel_pte_mask __read_mostly = ~0; - /* Simplified build-specific string for starting entropy. */ static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@" LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION; diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 1d6300689da0..b7d302ce3733 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -134,6 +134,14 @@ int count_immovable_mem_regions(void); static inline int count_immovable_mem_regions(void) { return 0; } #endif +/* ident_map_64.c */ +#ifdef CONFIG_X86_5LEVEL +extern unsigned int __pgtable_l5_enabled, pgdir_shift, ptrs_per_p4d; +#endif + +/* Used by PAGE_KERN* macros: */ +extern pteval_t __default_kernel_pte_mask; + /* idt_64.c */ extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; extern struct desc_ptr boot_idt_desc; -- Gitee From 0ab6be2788a19870b8ca62e10ab720fef5d5d742 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:16 +0200 Subject: [PATCH 043/113] x86/boot/compressed/64: Add page-fault handler commit 8b0d3b3b41ab6f14f1ce6d4a6b1c5f60b825123f upstream. Install a page-fault handler to add an identity mapping to addresses not yet mapped. Also do some checking whether the error code is sane. This makes non SEV-ES machines use the exception handling infrastructure in the pre-decompressions boot code too, making it less likely to break in the future. Hygon-SIG: commit 8b0d3b3b41ab upstream x86/boot/compressed/64: Add page-fault handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-16-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/idt_64.c, arch/x86/boot/compressed/idt_handlers_64.S, arch/x86/boot/compressed/misc.h] Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 39 ++++++++++++++++++++++ arch/x86/boot/compressed/idt_64.c | 2 ++ arch/x86/boot/compressed/idt_handlers_64.S | 2 ++ arch/x86/boot/compressed/misc.h | 6 ++++ 4 files changed, 49 insertions(+) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 89a44598ed65..b079c0360ad0 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -19,9 +19,12 @@ /* No PAGE_TABLE_ISOLATION support needed either: */ #undef CONFIG_PAGE_TABLE_ISOLATION +#include "error.h" #include "misc.h" /* These actually do the work of building the kernel identity maps. */ +#include +#include #include #include /* Use the static base for this part of the boot process */ @@ -160,3 +163,39 @@ void finalize_identity_maps(void) { write_cr3(top_level_pgt); } + +static void do_pf_error(const char *msg, unsigned long error_code, + unsigned long address, unsigned long ip) +{ + error_putstr(msg); + + error_putstr("\nError Code: "); + error_puthex(error_code); + error_putstr("\nCR2: 0x"); + error_puthex(address); + error_putstr("\nRIP relative to _head: 0x"); + error_puthex(ip - (unsigned long)_head); + error_putstr("\n"); + + error("Stopping.\n"); +} + +void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) +{ + unsigned long address = native_read_cr2(); + + /* + * Check for unexpected error codes. Unexpected are: + * - Faults on present pages + * - User faults + * - Reserved bits set + */ + if (error_code & (X86_PF_PROT | X86_PF_USER | X86_PF_RSVD)) + do_pf_error("Unexpected page-fault:", error_code, address, regs->ip); + + /* + * Error code is sane - now identity map the 2M region around + * the faulting address. + */ + add_identity_map(address & PMD_MASK, PMD_SIZE); +} diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 082cd6bca033..5f083092a86d 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -40,5 +40,7 @@ void load_stage2_idt(void) { boot_idt_desc.address = (unsigned long)boot_idt; + set_idt_entry(X86_TRAP_PF, boot_page_fault); + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 36dee2f40a8b..b20e57504a94 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -68,3 +68,5 @@ SYM_FUNC_END(\name) .text .code64 + +EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index b7d302ce3733..9646e5add1f3 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -37,6 +37,9 @@ #define memptr unsigned #endif +/* boot/compressed/vmlinux start and end markers */ +extern char _head[], _end[]; + /* misc.c */ extern memptr free_mem_ptr; extern memptr free_mem_end_ptr; @@ -146,4 +149,7 @@ extern pteval_t __default_kernel_pte_mask; extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; extern struct desc_ptr boot_idt_desc; +/* IDT Entry Points */ +void boot_page_fault(void); + #endif /* BOOT_COMPRESSED_MISC_H */ -- Gitee From c668d04093dcd3188ab36a4bfb1f5dc3a8a6b3ee Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:17 +0200 Subject: [PATCH 044/113] x86/boot/compressed/64: Always switch to own page table commit ca0e22d4f011a56e974fa3a712d76e86a791559d upstream. When booted through startup_64(), the kernel keeps running on the EFI page table until the KASLR code sets up its own page table. Without KASLR, the pre-decompression boot code never switches off the EFI page table. Change that by unconditionally switching to a kernel-controlled page table after relocation. This makes sure the kernel can make changes to the mapping when necessary, for example map pages unencrypted in SEV and SEV-ES guests. Also, remove the debug_putstr() calls in initialize_identity_maps() because the function now runs before console_init() is called. [ bp: Massage commit message. ] Hygon-SIG: commit ca0e22d4f011 upstream x86/boot/compressed/64: Always switch to own page table Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-17-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/head_64.S, arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/kaslr.c] Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 3 +- arch/x86/boot/compressed/ident_map_64.c | 51 +++++++++++++++---------- arch/x86/boot/compressed/kaslr.c | 3 -- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 0395dcab5a4e..5abed84184d6 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -516,10 +516,11 @@ ENDPROC(efi64_stub_entry) rep stosq /* - * Load stage2 IDT + * Load stage2 IDT and switch to our own page-table */ pushq %rsi call load_stage2_idt + call initialize_identity_maps popq %rsi /* diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index b079c0360ad0..96e325d4c885 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -86,9 +86,31 @@ phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; */ static struct x86_mapping_info mapping_info; +/* + * Adds the specified range to what will become the new identity mappings. + * Once all ranges have been added, the new mapping is activated by calling + * finalize_identity_maps() below. + */ +void add_identity_map(unsigned long start, unsigned long size) +{ + unsigned long end = start + size; + + /* Align boundary to 2M. */ + start = round_down(start, PMD_SIZE); + end = round_up(end, PMD_SIZE); + if (start >= end) + return; + + /* Build the mapping. */ + kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, + start, end); +} + /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { + unsigned long start, size; + /* If running as an SEV guest, the encryption mask is required. */ set_sev_encryption_mask(); @@ -121,37 +143,24 @@ void initialize_identity_maps(void) */ top_level_pgt = read_cr3_pa(); if (p4d_offset((pgd_t *)top_level_pgt, 0) == (p4d_t *)_pgtable) { - debug_putstr("booted via startup_32()\n"); pgt_data.pgt_buf = _pgtable + BOOT_INIT_PGT_SIZE; pgt_data.pgt_buf_size = BOOT_PGT_SIZE - BOOT_INIT_PGT_SIZE; memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size); } else { - debug_putstr("booted via startup_64()\n"); pgt_data.pgt_buf = _pgtable; pgt_data.pgt_buf_size = BOOT_PGT_SIZE; memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size); top_level_pgt = (unsigned long)alloc_pgt_page(&pgt_data); } -} -/* - * Adds the specified range to what will become the new identity mappings. - * Once all ranges have been added, the new mapping is activated by calling - * finalize_identity_maps() below. - */ -void add_identity_map(unsigned long start, unsigned long size) -{ - unsigned long end = start + size; - - /* Align boundary to 2M. */ - start = round_down(start, PMD_SIZE); - end = round_up(end, PMD_SIZE); - if (start >= end) - return; - - /* Build the mapping. */ - kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, - start, end); + /* + * New page-table is set up - map the kernel image and load it + * into cr3. + */ + start = (unsigned long)_head; + size = _end - _head; + add_identity_map(start, size); + write_cr3(top_level_pgt); } /* diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index e121bde52f95..63de3c2c3c92 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -859,9 +859,6 @@ void choose_random_location(unsigned long input, boot_params->hdr.loadflags |= KASLR_FLAG; - /* Prepare to add new identity pagetables on demand. */ - initialize_identity_maps(); - /* Record the various known unsafe memory ranges. */ mem_avoid_init(input, input_size, *output); -- Gitee From e50d398edc452fd1cbd2861fedbbfd8c86b7648c Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sat, 10 Oct 2020 15:11:10 -0400 Subject: [PATCH 045/113] x86/boot/64: Initialize 5-level paging variables earlier commit e5ceb9a02402b984feecb95a82239be151c9f4e2 upstream. Commit ca0e22d4f011 ("x86/boot/compressed/64: Always switch to own page table") started using a new set of pagetables even without KASLR. After that commit, initialize_identity_maps() is called before the 5-level paging variables are setup in choose_random_location(), which will not work if 5-level paging is actually enabled. Fix this by moving the initialization of __pgtable_l5_enabled, pgdir_shift and ptrs_per_p4d into cleanup_trampoline(), which is called immediately after the finalization of whether the kernel is executing with 4- or 5-level paging. This will be earlier than anything that might require those variables, and keeps the 4- vs 5-level paging code all in one place. Hygon-SIG: commit e5ceb9a02402 upstream x86/boot/64: Initialize 5-level paging variables earlier Fixes: ca0e22d4f011 ("x86/boot/compressed/64: Always switch to own page table") Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Tested-by: Joerg Roedel Tested-by: Kirill A. Shutemov Link: https://lkml.kernel.org/r/20201010191110.4060905-1-nivedita@alum.mit.edu [hanliyang: Fix conflict on arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/kaslr.c, arch/x86/boot/compressed/pgtable_64.c] Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 6 ------ arch/x86/boot/compressed/kaslr.c | 8 -------- arch/x86/boot/compressed/pgtable_64.c | 16 ++++++++++++++++ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 96e325d4c885..2bf7f56b8adc 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -32,12 +32,6 @@ #define __PAGE_OFFSET __PAGE_OFFSET_BASE #include "../../mm/ident_map.c" -#ifdef CONFIG_X86_5LEVEL -unsigned int __pgtable_l5_enabled; -unsigned int pgdir_shift = 39; -unsigned int ptrs_per_p4d = 1; -#endif - /* Used by PAGE_KERN* macros: */ pteval_t __default_kernel_pte_mask __read_mostly = ~0; diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 63de3c2c3c92..d80e3c28e9fc 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -849,14 +849,6 @@ void choose_random_location(unsigned long input, return; } -#ifdef CONFIG_X86_5LEVEL - if (__read_cr4() & X86_CR4_LA57) { - __pgtable_l5_enabled = 1; - pgdir_shift = 48; - ptrs_per_p4d = 512; - } -#endif - boot_params->hdr.loadflags |= KASLR_FLAG; /* Record the various known unsafe memory ranges. */ diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 7d0394f4ebf9..5def1674d6f1 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -8,6 +8,13 @@ #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ +#ifdef CONFIG_X86_5LEVEL +/* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */ +unsigned int __section(.data) __pgtable_l5_enabled; +unsigned int __section(.data) pgdir_shift = 39; +unsigned int __section(.data) ptrs_per_p4d = 1; +#endif + struct paging_config { unsigned long trampoline_start; unsigned long l5_required; @@ -198,4 +205,13 @@ void cleanup_trampoline(void *pgtable) /* Restore trampoline memory */ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); + + /* Initialize variables for 5-level paging */ +#ifdef CONFIG_X86_5LEVEL + if (__read_cr4() & X86_CR4_LA57) { + __pgtable_l5_enabled = 1; + pgdir_shift = 48; + ptrs_per_p4d = 512; + } +#endif } -- Gitee From d17c8b211b1eecbc5c3cf8d2d8261ad4e30d0304 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:18 +0200 Subject: [PATCH 046/113] x86/boot/compressed/64: Don't pre-map memory in KASLR code commit 8570978ea030757839747aa9944ea576708be3d4 upstream. With the page-fault handler in place, he identity mapping can be built on-demand. So remove the code which manually creates the mappings and unexport/remove the functions used for it. Hygon-SIG: commit 8570978ea030 upstream x86/boot/compressed/64: Don't pre-map memory in KASLR code Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-18-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/kaslr.c, arch/x86/boot/compressed/misc.h] Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 6 ++---- arch/x86/boot/compressed/kaslr.c | 24 +----------------------- arch/x86/boot/compressed/misc.h | 10 ---------- 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 2bf7f56b8adc..8e6c44e994c1 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -81,11 +81,9 @@ phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; static struct x86_mapping_info mapping_info; /* - * Adds the specified range to what will become the new identity mappings. - * Once all ranges have been added, the new mapping is activated by calling - * finalize_identity_maps() below. + * Adds the specified range to the identity mappings. */ -void add_identity_map(unsigned long start, unsigned long size) +static void add_identity_map(unsigned long start, unsigned long size) { unsigned long end = start + size; diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index d80e3c28e9fc..aa6537f6748d 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -371,8 +371,6 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, */ mem_avoid[MEM_AVOID_ZO_RANGE].start = input; mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input; - add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start, - mem_avoid[MEM_AVOID_ZO_RANGE].size); /* Avoid initrd. */ initrd_start = (u64)boot_params->ext_ramdisk_image << 32; @@ -392,14 +390,10 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, ; mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line; mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size; - add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start, - mem_avoid[MEM_AVOID_CMDLINE].size); /* Avoid boot parameters. */ mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params; mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params); - add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start, - mem_avoid[MEM_AVOID_BOOTPARAMS].size); /* We don't need to set a mapping for setup_data. */ @@ -408,11 +402,6 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, /* Enumerate the immovable memory regions */ num_immovable_mem = count_immovable_mem_regions(); - -#ifdef CONFIG_X86_VERBOSE_BOOTUP - /* Make sure video RAM can be used. */ - add_identity_map(0, PMD_SIZE); -#endif } /* @@ -867,19 +856,8 @@ void choose_random_location(unsigned long input, warn("Physical KASLR disabled: no suitable memory region!"); } else { /* Update the new physical address location. */ - if (*output != random_addr) { - add_identity_map(random_addr, output_size); + if (*output != random_addr) *output = random_addr; - } - - /* - * This loads the identity mapping page table. - * This should only be done if a new physical address - * is found for the kernel, otherwise we should keep - * the old page table to make it be like the "nokaslr" - * case. - */ - finalize_identity_maps(); } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 9646e5add1f3..4879cdd2832d 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -98,17 +98,7 @@ static inline void choose_random_location(unsigned long input, #endif #ifdef CONFIG_X86_64 -void initialize_identity_maps(void); -void add_identity_map(unsigned long start, unsigned long size); -void finalize_identity_maps(void); extern unsigned char _pgtable[]; -#else -static inline void initialize_identity_maps(void) -{ } -static inline void add_identity_map(unsigned long start, unsigned long size) -{ } -static inline void finalize_identity_maps(void) -{ } #endif #ifdef CONFIG_EARLY_PRINTK -- Gitee From 678bcaa57691a7f0c8b7375f44d8a9737f00b90a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:19 +0200 Subject: [PATCH 047/113] x86/boot/compressed/64: Change add_identity_map() to take start and end commit 21cf2372618ef167d8c4ae04880fb873b55b2daa upstream. Changing the function to take start and end as parameters instead of start and size simplifies the callers which don't need to calculate the size if they already have start and end. Hygon-SIG: commit 21cf2372618e upstream x86/boot/compressed/64: Change add_identity_map() to take start and end Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-19-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 8e6c44e994c1..756bec2158e3 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -83,10 +83,8 @@ static struct x86_mapping_info mapping_info; /* * Adds the specified range to the identity mappings. */ -static void add_identity_map(unsigned long start, unsigned long size) +static void add_identity_map(unsigned long start, unsigned long end) { - unsigned long end = start + size; - /* Align boundary to 2M. */ start = round_down(start, PMD_SIZE); end = round_up(end, PMD_SIZE); @@ -101,8 +99,6 @@ static void add_identity_map(unsigned long start, unsigned long size) /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { - unsigned long start, size; - /* If running as an SEV guest, the encryption mask is required. */ set_sev_encryption_mask(); @@ -149,9 +145,7 @@ void initialize_identity_maps(void) * New page-table is set up - map the kernel image and load it * into cr3. */ - start = (unsigned long)_head; - size = _end - _head; - add_identity_map(start, size); + add_identity_map((unsigned long)_head, (unsigned long)_end); write_cr3(top_level_pgt); } @@ -183,7 +177,8 @@ static void do_pf_error(const char *msg, unsigned long error_code, void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) { - unsigned long address = native_read_cr2(); + unsigned long address = native_read_cr2() & PMD_MASK; + unsigned long end = address + PMD_SIZE; /* * Check for unexpected error codes. Unexpected are: @@ -198,5 +193,5 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) * Error code is sane - now identity map the 2M region around * the faulting address. */ - add_identity_map(address & PMD_MASK, PMD_SIZE); + add_identity_map(address, end); } -- Gitee From ec0ec8df18feb88f54e7db2a4a5a9ab44240ffad Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:20 +0200 Subject: [PATCH 048/113] x86/boot/compressed/64: Add stage1 #VC handler commit 29dcc60f6a19fb0aaee97bd1ae2ed8a7dc6f0cfe upstream. Add the first handler for #VC exceptions. At stage 1 there is no GHCB yet because the kernel might still be running on the EFI page table. The stage 1 handler is limited to the MSR-based protocol to talk to the hypervisor and can only support CPUID exit-codes, but that is enough to get to stage 2. [ bp: Zap superfluous newlines after rd/wrmsr instruction mnemonics. ] Hygon-SIG: commit 29dcc60f6a19 upstream x86/boot/compressed/64: Add stage1 #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-20-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/Makefile, arch/x86/boot/compressed/idt_64.c, arch/x86/boot/compressed/idt_handlers_64.S, arch/x86/boot/compressed/misc.h, arch/x86/boot/compressed/sev-es.c and 4 more] Signed-off-by: hanliyang --- arch/x86/boot/compressed/Makefile | 1 + arch/x86/boot/compressed/idt_64.c | 4 ++ arch/x86/boot/compressed/idt_handlers_64.S | 4 ++ arch/x86/boot/compressed/misc.h | 1 + arch/x86/boot/compressed/sev-es.c | 45 +++++++++++++++ arch/x86/include/asm/msr-index.h | 1 + arch/x86/include/asm/sev-es.h | 37 ++++++++++++ arch/x86/include/asm/trapnr.h | 1 + arch/x86/kernel/sev-es-shared.c | 66 ++++++++++++++++++++++ 9 files changed, 160 insertions(+) create mode 100644 arch/x86/boot/compressed/sev-es.c create mode 100644 arch/x86/include/asm/sev-es.h create mode 100644 arch/x86/kernel/sev-es-shared.c diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 219b426ce287..494998a915e1 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -90,6 +90,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 5f083092a86d..f3ca7324be44 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -32,6 +32,10 @@ void load_stage1_idt(void) { boot_idt_desc.address = (unsigned long)boot_idt; + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + set_idt_entry(X86_TRAP_VC, boot_stage1_vc); + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index b20e57504a94..92eb4df478a1 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -70,3 +70,7 @@ SYM_FUNC_END(\name) .code64 EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 + +#ifdef CONFIG_AMD_MEM_ENCRYPT +EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +#endif diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 4879cdd2832d..15f09c07e545 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -141,5 +141,6 @@ extern struct desc_ptr boot_idt_desc; /* IDT Entry Points */ void boot_page_fault(void); +void boot_stage1_vc(void); #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c new file mode 100644 index 000000000000..99c3bcd4d61f --- /dev/null +++ b/arch/x86/boot/compressed/sev-es.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + */ + +/* + * misc.h needs to be first because it knows how to include the other kernel + * headers in the pre-decompression code in a way that does not break + * compilation. + */ +#include "misc.h" + +#include +#include +#include +#include + +static inline u64 sev_es_rd_ghcb_msr(void) +{ + unsigned long low, high; + + asm volatile("rdmsr" : "=a" (low), "=d" (high) : + "c" (MSR_AMD64_SEV_ES_GHCB)); + + return ((high << 32) | low); +} + +static inline void sev_es_wr_ghcb_msr(u64 val) +{ + u32 low, high; + + low = val & 0xffffffffUL; + high = val >> 32; + + asm volatile("wrmsr" : : "c" (MSR_AMD64_SEV_ES_GHCB), + "a"(low), "d" (high) : "memory"); +} + +#undef __init +#define __init + +/* Include code for early handlers */ +#include "../../kernel/sev-es-shared.c" diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 220448e729b5..371dfbaba396 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -499,6 +499,7 @@ #define MSR_AMD64_ICIBSEXTDCTL 0xc001103c #define MSR_AMD64_IBSOPDATA4 0xc001103d #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ +#define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h new file mode 100644 index 000000000000..48a44038b5d1 --- /dev/null +++ b/arch/x86/include/asm/sev-es.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + */ + +#ifndef __ASM_ENCRYPTED_STATE_H +#define __ASM_ENCRYPTED_STATE_H + +#include + +#define GHCB_SEV_CPUID_REQ 0x004UL +#define GHCB_CPUID_REQ_EAX 0 +#define GHCB_CPUID_REQ_EBX 1 +#define GHCB_CPUID_REQ_ECX 2 +#define GHCB_CPUID_REQ_EDX 3 +#define GHCB_CPUID_REQ(fn, reg) (GHCB_SEV_CPUID_REQ | \ + (((unsigned long)reg & 3) << 30) | \ + (((unsigned long)fn) << 32)) + +#define GHCB_SEV_CPUID_RESP 0x005UL +#define GHCB_SEV_TERMINATE 0x100UL + +#define GHCB_SEV_GHCB_RESP_CODE(v) ((v) & 0xfff) +#define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } + +void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); + +static inline u64 lower_bits(u64 val, unsigned int bits) +{ + u64 mask = (1ULL << bits) - 1; + + return (val & mask); +} + +#endif diff --git a/arch/x86/include/asm/trapnr.h b/arch/x86/include/asm/trapnr.h index 082f45631fa9..f5d2325aa0b7 100644 --- a/arch/x86/include/asm/trapnr.h +++ b/arch/x86/include/asm/trapnr.h @@ -26,6 +26,7 @@ #define X86_TRAP_XF 19 /* SIMD Floating-Point Exception */ #define X86_TRAP_VE 20 /* Virtualization Exception */ #define X86_TRAP_CP 21 /* Control Protection Exception */ +#define X86_TRAP_VC 29 /* VMM Communication Exception */ #define X86_TRAP_IRET 32 /* IRET Exception */ #endif diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c new file mode 100644 index 000000000000..0bea32341afa --- /dev/null +++ b/arch/x86/kernel/sev-es-shared.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + * + * This file is not compiled stand-alone. It contains code shared + * between the pre-decompression boot code and the running Linux kernel + * and is included directly into both code-bases. + */ + +/* + * Boot VC Handler - This is the first VC handler during boot, there is no GHCB + * page yet, so it only supports the MSR based communication with the + * hypervisor and only the CPUID exit-code. + */ +void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) +{ + unsigned int fn = lower_bits(regs->ax, 32); + unsigned long val; + + /* Only CPUID is supported via MSR protocol */ + if (exit_code != SVM_EXIT_CPUID) + goto fail; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->ax = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->bx = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->cx = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->dx = val >> 32; + + /* Skip over the CPUID two-byte opcode */ + regs->ip += 2; + + return; + +fail: + sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE); + VMGEXIT(); + + /* Shouldn't get here - if we do halt the machine */ + while (true) + asm volatile("hlt\n"); +} -- Gitee From 8df482f55cfc363ddc8f93ffb9f5ab73c252a459 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:21 +0200 Subject: [PATCH 049/113] x86/boot/compressed/64: Call set_sev_encryption_mask() earlier commit c2a0304a286f386e45cea3f4b0617f0813de67fd upstream. Call set_sev_encryption_mask() while still on the stage 1 #VC-handler because the stage 2 handler needs the kernel's own page tables to be set up, to which calling set_sev_encryption_mask() is a prerequisite. Hygon-SIG: commit c2a0304a286f upstream x86/boot/compressed/64: Call set_sev_encryption_mask() earlier Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-21-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 9 ++++++++- arch/x86/boot/compressed/ident_map_64.c | 3 --- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 5abed84184d6..ef521541e0af 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -516,9 +516,16 @@ ENDPROC(efi64_stub_entry) rep stosq /* - * Load stage2 IDT and switch to our own page-table + * If running as an SEV guest, the encryption mask is required in the + * page-table setup code below. When the guest also has SEV-ES enabled + * set_sev_encryption_mask() will cause #VC exceptions, but the stage2 + * handler can't map its GHCB because the page-table is not set up yet. + * So set up the encryption mask here while still on the stage1 #VC + * handler. Then load stage2 IDT and switch to the kernel's own + * page-table. */ pushq %rsi + call set_sev_encryption_mask call load_stage2_idt call initialize_identity_maps popq %rsi diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 756bec2158e3..33461efe989a 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -99,9 +99,6 @@ static void add_identity_map(unsigned long start, unsigned long end) /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { - /* If running as an SEV guest, the encryption mask is required. */ - set_sev_encryption_mask(); - /* Exclude the encryption mask from __PHYSICAL_MASK */ physical_mask &= ~sme_me_mask; -- Gitee From e7afb4ff2b270023ffcb5f951ca697fb58ebee90 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:22 +0200 Subject: [PATCH 050/113] x86/boot/compressed/64: Check return value of kernel_ident_mapping_init() commit 4b3fdca64a7e8ad90c87cad1fbc6991471f48dc7 upstream. The function can fail to create an identity mapping, check for that and bail out if it happens. Hygon-SIG: commit 4b3fdca64a7e upstream x86/boot/compressed/64: Check return value of kernel_ident_mapping_init() Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-22-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 33461efe989a..04189971041b 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -85,6 +85,8 @@ static struct x86_mapping_info mapping_info; */ static void add_identity_map(unsigned long start, unsigned long end) { + int ret; + /* Align boundary to 2M. */ start = round_down(start, PMD_SIZE); end = round_up(end, PMD_SIZE); @@ -92,8 +94,9 @@ static void add_identity_map(unsigned long start, unsigned long end) return; /* Build the mapping. */ - kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, - start, end); + ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); + if (ret) + error("Error: kernel_ident_mapping_init() failed\n"); } /* Locates and clears a region for a new top level page table. */ -- Gitee From a1e90bf829dfa8883d095336d9572176cd479afc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:23 +0200 Subject: [PATCH 051/113] x86/boot/compressed/64: Add set_page_en/decrypted() helpers commit c81d60029a1393183d2125fcb4b64831629b8864 upstream. The functions are needed to map the GHCB for SEV-ES guests. The GHCB is used for communication with the hypervisor, so its content must not be encrypted. After the GHCB is not needed anymore it must be mapped encrypted again so that the running kernel image can safely re-use the memory. Hygon-SIG: commit c81d60029a13 upstream x86/boot/compressed/64: Add set_page_en/decrypted() helpers Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-23-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/misc.h] Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 133 ++++++++++++++++++++++++ arch/x86/boot/compressed/misc.h | 2 + 2 files changed, 135 insertions(+) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 04189971041b..2488c3f945c2 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -23,6 +23,7 @@ #include "misc.h" /* These actually do the work of building the kernel identity maps. */ +#include #include #include #include @@ -159,6 +160,138 @@ void finalize_identity_maps(void) write_cr3(top_level_pgt); } +static pte_t *split_large_pmd(struct x86_mapping_info *info, + pmd_t *pmdp, unsigned long __address) +{ + unsigned long page_flags; + unsigned long address; + pte_t *pte; + pmd_t pmd; + int i; + + pte = (pte_t *)info->alloc_pgt_page(info->context); + if (!pte) + return NULL; + + address = __address & PMD_MASK; + /* No large page - clear PSE flag */ + page_flags = info->page_flag & ~_PAGE_PSE; + + /* Populate the PTEs */ + for (i = 0; i < PTRS_PER_PMD; i++) { + set_pte(&pte[i], __pte(address | page_flags)); + address += PAGE_SIZE; + } + + /* + * Ideally we need to clear the large PMD first and do a TLB + * flush before we write the new PMD. But the 2M range of the + * PMD might contain the code we execute and/or the stack + * we are on, so we can't do that. But that should be safe here + * because we are going from large to small mappings and we are + * also the only user of the page-table, so there is no chance + * of a TLB multihit. + */ + pmd = __pmd((unsigned long)pte | info->kernpg_flag); + set_pmd(pmdp, pmd); + /* Flush TLB to establish the new PMD */ + write_cr3(top_level_pgt); + + return pte + pte_index(__address); +} + +static void clflush_page(unsigned long address) +{ + unsigned int flush_size; + char *cl, *start, *end; + + /* + * Hardcode cl-size to 64 - CPUID can't be used here because that might + * cause another #VC exception and the GHCB is not ready to use yet. + */ + flush_size = 64; + start = (char *)(address & PAGE_MASK); + end = start + PAGE_SIZE; + + /* + * First make sure there are no pending writes on the cache-lines to + * flush. + */ + asm volatile("mfence" : : : "memory"); + + for (cl = start; cl != end; cl += flush_size) + clflush(cl); +} + +static int set_clr_page_flags(struct x86_mapping_info *info, + unsigned long address, + pteval_t set, pteval_t clr) +{ + pgd_t *pgdp = (pgd_t *)top_level_pgt; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep, pte; + + /* + * First make sure there is a PMD mapping for 'address'. + * It should already exist, but keep things generic. + * + * To map the page just read from it and fault it in if there is no + * mapping yet. add_identity_map() can't be called here because that + * would unconditionally map the address on PMD level, destroying any + * PTE-level mappings that might already exist. Use assembly here so + * the access won't be optimized away. + */ + asm volatile("mov %[address], %%r9" + :: [address] "g" (*(unsigned long *)address) + : "r9", "memory"); + + /* + * The page is mapped at least with PMD size - so skip checks and walk + * directly to the PMD. + */ + p4dp = p4d_offset(pgdp, address); + pudp = pud_offset(p4dp, address); + pmdp = pmd_offset(pudp, address); + + if (pmd_large(*pmdp)) + ptep = split_large_pmd(info, pmdp, address); + else + ptep = pte_offset_kernel(pmdp, address); + + if (!ptep) + return -ENOMEM; + + /* + * Changing encryption attributes of a page requires to flush it from + * the caches. + */ + if ((set | clr) & _PAGE_ENC) + clflush_page(address); + + /* Update PTE */ + pte = *ptep; + pte = pte_set_flags(pte, set); + pte = pte_clear_flags(pte, clr); + set_pte(ptep, pte); + + /* Flush TLB after changing encryption attribute */ + write_cr3(top_level_pgt); + + return 0; +} + +int set_page_decrypted(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, 0, _PAGE_ENC); +} + +int set_page_encrypted(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0); +} + static void do_pf_error(const char *msg, unsigned long error_code, unsigned long address, unsigned long ip) { diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 15f09c07e545..251a28247a13 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -98,6 +98,8 @@ static inline void choose_random_location(unsigned long input, #endif #ifdef CONFIG_X86_64 +extern int set_page_decrypted(unsigned long address); +extern int set_page_encrypted(unsigned long address); extern unsigned char _pgtable[]; #endif -- Gitee From 4ab617dc18886a4c2d49eb900be07bef4fa48790 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:24 +0200 Subject: [PATCH 052/113] x86/boot/compressed/64: Setup a GHCB-based VC Exception handler commit 597cfe48212a3f110ab0f918bf59791f453e65b7 upstream. Install an exception handler for #VC exception that uses a GHCB. Also add the infrastructure for handling different exit-codes by decoding the instruction that caused the exception and error handling. Hygon-SIG: commit 597cfe48212a upstream x86/boot/compressed/64: Setup a GHCB-based VC Exception handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-24-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/Kconfig, arch/x86/boot/compressed/Makefile, arch/x86/boot/compressed/idt_64.c, arch/x86/boot/compressed/idt_handlers_64.S, arch/x86/boot/compressed/misc.c and 5 more] Signed-off-by: hanliyang --- arch/x86/Kconfig | 1 + arch/x86/boot/compressed/Makefile | 5 + arch/x86/boot/compressed/idt_64.c | 4 + arch/x86/boot/compressed/idt_handlers_64.S | 3 +- arch/x86/boot/compressed/misc.c | 7 + arch/x86/boot/compressed/misc.h | 7 + arch/x86/boot/compressed/sev-es.c | 111 +++++++++++++++ arch/x86/include/asm/sev-es.h | 39 ++++++ arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/sev-es-shared.c | 154 +++++++++++++++++++++ 10 files changed, 331 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index cdc3b4f600ed..c0bb48595e40 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1555,6 +1555,7 @@ config AMD_MEM_ENCRYPT select DYNAMIC_PHYSICAL_MASK select ARCH_USE_MEMREMAP_PROT select ARCH_HAS_FORCE_DMA_UNENCRYPTED + select INSTRUCTION_DECODER ---help--- Say yes to enable support for the encryption of system memory. This requires an AMD processor that supports Secure Memory diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 494998a915e1..0709b396f714 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -42,6 +42,11 @@ KBUILD_CFLAGS += -Wno-pointer-sign KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no) KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h +# sev-es.c indirectly inludes inat-table.h which is generated during +# compilation and stored in $(objtree). Add the directory to the includes so +# that the compiler finds it even with out-of-tree builds (make O=/some/path). +CFLAGS_sev-es.o += -I$(objtree)/arch/x86/lib/ + KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n UBSAN_SANITIZE :=n diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index f3ca7324be44..804a502ee0d2 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -46,5 +46,9 @@ void load_stage2_idt(void) set_idt_entry(X86_TRAP_PF, boot_page_fault); +#ifdef CONFIG_AMD_MEM_ENCRYPT + set_idt_entry(X86_TRAP_VC, boot_stage2_vc); +#endif + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 92eb4df478a1..22890e199f5b 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -72,5 +72,6 @@ SYM_FUNC_END(\name) EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 #ifdef CONFIG_AMD_MEM_ENCRYPT -EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +EXCEPTION_HANDLER boot_stage2_vc do_boot_stage2_vc error_code=1 #endif diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 9652d5c2afda..dba49e75095a 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -441,6 +441,13 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, parse_elf(output); handle_relocations(output, output_len, virt_addr); debug_putstr("done.\nBooting the kernel.\n"); + + /* + * Flush GHCB from cache and map it encrypted again when running as + * SEV-ES guest. + */ + sev_es_shutdown_ghcb(); + return output; } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 251a28247a13..88b5fbd16371 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -115,6 +115,12 @@ static inline void console_init(void) void set_sev_encryption_mask(void); +#ifdef CONFIG_AMD_MEM_ENCRYPT +void sev_es_shutdown_ghcb(void); +#else +static inline void sev_es_shutdown_ghcb(void) { } +#endif + /* acpi.c */ #ifdef CONFIG_ACPI acpi_physical_address get_rsdp_addr(void); @@ -144,5 +150,6 @@ extern struct desc_ptr boot_idt_desc; /* IDT Entry Points */ void boot_page_fault(void); void boot_stage1_vc(void); +void boot_stage2_vc(void); #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 99c3bcd4d61f..fa62af771dd5 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -13,10 +13,17 @@ #include "misc.h" #include +#include +#include #include #include #include +#include "error.h" + +struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); +struct ghcb *boot_ghcb; + static inline u64 sev_es_rd_ghcb_msr(void) { unsigned long low, high; @@ -38,8 +45,112 @@ static inline void sev_es_wr_ghcb_msr(u64 val) "a"(low), "d" (high) : "memory"); } +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + enum es_result ret; + + memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE, 1); + insn_get_length(&ctxt->insn); + + ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; + + return ret; +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + void *dst, char *buf, size_t size) +{ + memcpy(dst, buf, size); + + return ES_OK; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + void *src, char *buf, size_t size) +{ + memcpy(buf, src, size); + + return ES_OK; +} + #undef __init +#undef __pa #define __init +#define __pa(x) ((unsigned long)(x)) + +#define __BOOT_COMPRESSED + +/* Basic instruction decoding support needed */ +#include "../../lib/inat.c" +#include "../../lib/insn.c" /* Include code for early handlers */ #include "../../kernel/sev-es-shared.c" + +static bool early_setup_sev_es(void) +{ + if (!sev_es_negotiate_protocol()) + sev_es_terminate(GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED); + + if (set_page_decrypted((unsigned long)&boot_ghcb_page)) + return false; + + /* Page is now mapped decrypted, clear it */ + memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page)); + + boot_ghcb = &boot_ghcb_page; + + /* Initialize lookup tables for the instruction decoder */ + inat_init_tables(); + + return true; +} + +void sev_es_shutdown_ghcb(void) +{ + if (!boot_ghcb) + return; + + /* + * GHCB Page must be flushed from the cache and mapped encrypted again. + * Otherwise the running kernel will see strange cache effects when + * trying to use that page. + */ + if (set_page_encrypted((unsigned long)&boot_ghcb_page)) + error("Can't map GHCB page encrypted"); +} + +void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) +{ + struct es_em_ctxt ctxt; + enum es_result result; + + if (!boot_ghcb && !early_setup_sev_es()) + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + vc_ghcb_invalidate(boot_ghcb); + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result != ES_OK) + goto finish; + + switch (exit_code) { + default: + result = ES_UNSUPPORTED; + break; + } + +finish: + if (result == ES_OK) { + vc_finish_insn(&ctxt); + } else if (result != ES_RETRY) { + /* + * For now, just halt the machine. That makes debugging easier, + * later we just call sev_es_terminate() here. + */ + while (true) + asm volatile("hlt\n"); + } +} diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 48a44038b5d1..6dc52440c4b4 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -9,7 +9,14 @@ #define __ASM_ENCRYPTED_STATE_H #include +#include +#define GHCB_SEV_INFO 0x001UL +#define GHCB_SEV_INFO_REQ 0x002UL +#define GHCB_INFO(v) ((v) & 0xfffUL) +#define GHCB_PROTO_MAX(v) (((v) >> 48) & 0xffffUL) +#define GHCB_PROTO_MIN(v) (((v) >> 32) & 0xffffUL) +#define GHCB_PROTO_OUR 0x0001UL #define GHCB_SEV_CPUID_REQ 0x004UL #define GHCB_CPUID_REQ_EAX 0 #define GHCB_CPUID_REQ_EBX 1 @@ -19,12 +26,44 @@ (((unsigned long)reg & 3) << 30) | \ (((unsigned long)fn) << 32)) +#define GHCB_PROTOCOL_MAX 0x0001UL +#define GHCB_DEFAULT_USAGE 0x0000UL + #define GHCB_SEV_CPUID_RESP 0x005UL #define GHCB_SEV_TERMINATE 0x100UL +#define GHCB_SEV_TERMINATE_REASON(reason_set, reason_val) \ + (((((u64)reason_set) & 0x7) << 12) | \ + ((((u64)reason_val) & 0xff) << 16)) +#define GHCB_SEV_ES_REASON_GENERAL_REQUEST 0 +#define GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED 1 #define GHCB_SEV_GHCB_RESP_CODE(v) ((v) & 0xfff) #define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } +enum es_result { + ES_OK, /* All good */ + ES_UNSUPPORTED, /* Requested operation not supported */ + ES_VMM_ERROR, /* Unexpected state from the VMM */ + ES_DECODE_FAILED, /* Instruction decoding failed */ + ES_EXCEPTION, /* Instruction caused exception */ + ES_RETRY, /* Retry instruction emulation */ +}; + +struct es_fault_info { + unsigned long vector; + unsigned long error_code; + unsigned long cr2; +}; + +struct pt_regs; + +/* ES instruction emulation context */ +struct es_em_ctxt { + struct pt_regs *regs; + struct insn insn; + struct es_fault_info fi; +}; + void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); static inline u64 lower_bits(u64 val, unsigned int bits) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 2e8a30f06c74..c68d1618c9b0 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -29,6 +29,7 @@ #define SVM_EXIT_WRITE_DR6 0x036 #define SVM_EXIT_WRITE_DR7 0x037 #define SVM_EXIT_EXCP_BASE 0x040 +#define SVM_EXIT_LAST_EXCP 0x05f #define SVM_EXIT_INTR 0x060 #define SVM_EXIT_NMI 0x061 #define SVM_EXIT_SMI 0x062 diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 0bea32341afa..7ac6e6b0ae57 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,6 +9,118 @@ * and is included directly into both code-bases. */ +static void sev_es_terminate(unsigned int reason) +{ + u64 val = GHCB_SEV_TERMINATE; + + /* + * Tell the hypervisor what went wrong - only reason-set 0 is + * currently supported. + */ + val |= GHCB_SEV_TERMINATE_REASON(0, reason); + + /* Request Guest Termination from Hypvervisor */ + sev_es_wr_ghcb_msr(val); + VMGEXIT(); + + while (true) + asm volatile("hlt\n" : : : "memory"); +} + +static bool sev_es_negotiate_protocol(void) +{ + u64 val; + + /* Do the GHCB protocol version negotiation */ + sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + + if (GHCB_INFO(val) != GHCB_SEV_INFO) + return false; + + if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR || + GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR) + return false; + + return true; +} + +static void vc_ghcb_invalidate(struct ghcb *ghcb) +{ + memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); +} + +static bool vc_decoding_needed(unsigned long exit_code) +{ + /* Exceptions don't require to decode the instruction */ + return !(exit_code >= SVM_EXIT_EXCP_BASE && + exit_code <= SVM_EXIT_LAST_EXCP); +} + +static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) +{ + enum es_result ret = ES_OK; + + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->regs = regs; + + if (vc_decoding_needed(exit_code)) + ret = vc_decode_insn(ctxt); + + return ret; +} + +static void vc_finish_insn(struct es_em_ctxt *ctxt) +{ + ctxt->regs->ip += ctxt->insn.length; +} + +static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2) +{ + enum es_result ret; + + /* Fill in protocol and format specifiers */ + ghcb->protocol_version = GHCB_PROTOCOL_MAX; + ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; + + ghcb_set_sw_exit_code(ghcb, exit_code); + ghcb_set_sw_exit_info_1(ghcb, exit_info_1); + ghcb_set_sw_exit_info_2(ghcb, exit_info_2); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) { + u64 info = ghcb->save.sw_exit_info_2; + unsigned long v; + + info = ghcb->save.sw_exit_info_2; + v = info & SVM_EVTINJ_VEC_MASK; + + /* Check if exception information from hypervisor is sane. */ + if ((info & SVM_EVTINJ_VALID) && + ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && + ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { + ctxt->fi.vector = v; + if (info & SVM_EVTINJ_VALID_ERR) + ctxt->fi.error_code = info >> 32; + ret = ES_EXCEPTION; + } else { + ret = ES_VMM_ERROR; + } + } else { + ret = ES_OK; + } + + return ret; +} + /* * Boot VC Handler - This is the first VC handler during boot, there is no GHCB * page yet, so it only supports the MSR based communication with the @@ -64,3 +176,45 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) while (true) asm volatile("hlt\n"); } + +static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, + void *src, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, b = backwards ? -1 : 1; + enum es_result ret = ES_OK; + + for (i = 0; i < count; i++) { + void *s = src + (i * data_size * b); + char *d = buf + (i * data_size); + + ret = vc_read_mem(ctxt, s, d, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} + +static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, + void *dst, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, s = backwards ? -1 : 1; + enum es_result ret = ES_OK; + + for (i = 0; i < count; i++) { + void *d = dst + (i * data_size * s); + char *b = buf + (i * data_size); + + ret = vc_write_mem(ctxt, d, b, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} -- Gitee From 9581da333ff852bcf3021d7b9e4e28f39b7504fa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:25 +0200 Subject: [PATCH 053/113] x86/boot/compressed/64: Unmap GHCB page before booting the kernel commit 69add17a7c1992593a7cf775a66e0256ad4b3ef8 upstream. Force a page-fault on any further accesses to the GHCB page when they shouldn't happen anymore. This will catch any bugs where a #VC exception is raised even though none is expected anymore. Hygon-SIG: commit 69add17a7c19 upstream x86/boot/compressed/64: Unmap GHCB page before booting the kernel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-25-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 17 +++++++++++++++-- arch/x86/boot/compressed/misc.h | 6 ++++++ arch/x86/boot/compressed/sev-es.c | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 2488c3f945c2..207101a06f63 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -292,6 +292,11 @@ int set_page_encrypted(unsigned long address) return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0); } +int set_page_non_present(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, 0, _PAGE_PRESENT); +} + static void do_pf_error(const char *msg, unsigned long error_code, unsigned long address, unsigned long ip) { @@ -310,8 +315,14 @@ static void do_pf_error(const char *msg, unsigned long error_code, void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) { - unsigned long address = native_read_cr2() & PMD_MASK; - unsigned long end = address + PMD_SIZE; + unsigned long address = native_read_cr2(); + unsigned long end; + bool ghcb_fault; + + ghcb_fault = sev_es_check_ghcb_fault(address); + + address &= PMD_MASK; + end = address + PMD_SIZE; /* * Check for unexpected error codes. Unexpected are: @@ -321,6 +332,8 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) */ if (error_code & (X86_PF_PROT | X86_PF_USER | X86_PF_RSVD)) do_pf_error("Unexpected page-fault:", error_code, address, regs->ip); + else if (ghcb_fault) + do_pf_error("Page-fault on GHCB page:", error_code, address, regs->ip); /* * Error code is sane - now identity map the 2M region around diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 88b5fbd16371..6fd41171bcfc 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -100,6 +100,7 @@ static inline void choose_random_location(unsigned long input, #ifdef CONFIG_X86_64 extern int set_page_decrypted(unsigned long address); extern int set_page_encrypted(unsigned long address); +extern int set_page_non_present(unsigned long address); extern unsigned char _pgtable[]; #endif @@ -117,8 +118,13 @@ void set_sev_encryption_mask(void); #ifdef CONFIG_AMD_MEM_ENCRYPT void sev_es_shutdown_ghcb(void); +extern bool sev_es_check_ghcb_fault(unsigned long address); #else static inline void sev_es_shutdown_ghcb(void) { } +static inline bool sev_es_check_ghcb_fault(unsigned long address) +{ + return false; +} #endif /* acpi.c */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index fa62af771dd5..1e1fab583302 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -121,6 +121,20 @@ void sev_es_shutdown_ghcb(void) */ if (set_page_encrypted((unsigned long)&boot_ghcb_page)) error("Can't map GHCB page encrypted"); + + /* + * GHCB page is mapped encrypted again and flushed from the cache. + * Mark it non-present now to catch bugs when #VC exceptions trigger + * after this point. + */ + if (set_page_non_present((unsigned long)&boot_ghcb_page)) + error("Can't unmap GHCB page"); +} + +bool sev_es_check_ghcb_fault(unsigned long address) +{ + /* Check whether the fault was on the GHCB page */ + return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page); } void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) -- Gitee From 02a41fa04775cfd4fba00783302787fea7811414 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:26 +0200 Subject: [PATCH 054/113] x86/sev-es: Add support for handling IOIO exceptions commit 25189d08e5168c098c307a0eaae5b30c13a331ef upstream. Add support for decoding and handling #VC exceptions for IOIO events. Hygon-SIG: commit 25189d08e516 upstream x86/sev-es: Add support for handling IOIO exceptions [ jroedel@suse.de: Adapted code to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-26-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/sev-es.c | 32 +++++ arch/x86/kernel/sev-es-shared.c | 214 ++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 1e1fab583302..61504eb1ab46 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -24,6 +24,35 @@ struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); struct ghcb *boot_ghcb; +/* + * Copy a version of this function here - insn-eval.c can't be used in + * pre-decompression code. + */ +static bool insn_has_rep_prefix(struct insn *insn) +{ + int i; + + insn_get_prefixes(insn); + + for (i = 0; i < insn->prefixes.nbytes; i++) { + insn_byte_t p = insn->prefixes.bytes[i]; + + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + +/* + * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and + * doesn't use segments. + */ +static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) +{ + return 0UL; +} + static inline u64 sev_es_rd_ghcb_msr(void) { unsigned long low, high; @@ -151,6 +180,9 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) goto finish; switch (exit_code) { + case SVM_EXIT_IOIO: + result = vc_handle_ioio(boot_ghcb, &ctxt); + break; default: result = ES_UNSUPPORTED; break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 7ac6e6b0ae57..bae7cf28455b 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -218,3 +218,217 @@ static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, return ret; } + +#define IOIO_TYPE_STR BIT(2) +#define IOIO_TYPE_IN 1 +#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) +#define IOIO_TYPE_OUT 0 +#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) + +#define IOIO_REP BIT(3) + +#define IOIO_ADDR_64 BIT(9) +#define IOIO_ADDR_32 BIT(8) +#define IOIO_ADDR_16 BIT(7) + +#define IOIO_DATA_32 BIT(6) +#define IOIO_DATA_16 BIT(5) +#define IOIO_DATA_8 BIT(4) + +#define IOIO_SEG_ES (0 << 10) +#define IOIO_SEG_DS (3 << 10) + +static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) +{ + struct insn *insn = &ctxt->insn; + *exitinfo = 0; + + switch (insn->opcode.bytes[0]) { + /* INS opcodes */ + case 0x6c: + case 0x6d: + *exitinfo |= IOIO_TYPE_INS; + *exitinfo |= IOIO_SEG_ES; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* OUTS opcodes */ + case 0x6e: + case 0x6f: + *exitinfo |= IOIO_TYPE_OUTS; + *exitinfo |= IOIO_SEG_DS; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* IN immediate opcodes */ + case 0xe4: + case 0xe5: + *exitinfo |= IOIO_TYPE_IN; + *exitinfo |= (u64)insn->immediate.value << 16; + break; + + /* OUT immediate opcodes */ + case 0xe6: + case 0xe7: + *exitinfo |= IOIO_TYPE_OUT; + *exitinfo |= (u64)insn->immediate.value << 16; + break; + + /* IN register opcodes */ + case 0xec: + case 0xed: + *exitinfo |= IOIO_TYPE_IN; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* OUT register opcodes */ + case 0xee: + case 0xef: + *exitinfo |= IOIO_TYPE_OUT; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + default: + return ES_DECODE_FAILED; + } + + switch (insn->opcode.bytes[0]) { + case 0x6c: + case 0x6e: + case 0xe4: + case 0xe6: + case 0xec: + case 0xee: + /* Single byte opcodes */ + *exitinfo |= IOIO_DATA_8; + break; + default: + /* Length determined by instruction parsing */ + *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 + : IOIO_DATA_32; + } + switch (insn->addr_bytes) { + case 2: + *exitinfo |= IOIO_ADDR_16; + break; + case 4: + *exitinfo |= IOIO_ADDR_32; + break; + case 8: + *exitinfo |= IOIO_ADDR_64; + break; + } + + if (insn_has_rep_prefix(insn)) + *exitinfo |= IOIO_REP; + + return ES_OK; +} + +static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u64 exit_info_1, exit_info_2; + enum es_result ret; + + ret = vc_ioio_exitinfo(ctxt, &exit_info_1); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_STR) { + + /* (REP) INS/OUTS */ + + bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); + unsigned int io_bytes, exit_bytes; + unsigned int ghcb_count, op_count; + unsigned long es_base; + u64 sw_scratch; + + /* + * For the string variants with rep prefix the amount of in/out + * operations per #VC exception is limited so that the kernel + * has a chance to take interrupts and re-schedule while the + * instruction is emulated. + */ + io_bytes = (exit_info_1 >> 4) & 0x7; + ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; + + op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; + exit_info_2 = min(op_count, ghcb_count); + exit_bytes = exit_info_2 * io_bytes; + + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + /* Read bytes of OUTS into the shared buffer */ + if (!(exit_info_1 & IOIO_TYPE_IN)) { + ret = vc_insn_string_read(ctxt, + (void *)(es_base + regs->si), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + } + + /* + * Issue an VMGEXIT to the HV to consume the bytes from the + * shared buffer or to have it write them into the shared buffer + * depending on the instruction: OUTS or INS. + */ + sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); + ghcb_set_sw_scratch(ghcb, sw_scratch); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, + exit_info_1, exit_info_2); + if (ret != ES_OK) + return ret; + + /* Read bytes from shared buffer into the guest's destination. */ + if (exit_info_1 & IOIO_TYPE_IN) { + ret = vc_insn_string_write(ctxt, + (void *)(es_base + regs->di), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + + if (df) + regs->di -= exit_bytes; + else + regs->di += exit_bytes; + } else { + if (df) + regs->si -= exit_bytes; + else + regs->si += exit_bytes; + } + + if (exit_info_1 & IOIO_REP) + regs->cx -= exit_info_2; + + ret = regs->cx ? ES_RETRY : ES_OK; + + } else { + + /* IN/OUT into/from rAX */ + + int bits = (exit_info_1 & 0x70) >> 1; + u64 rax = 0; + + if (!(exit_info_1 & IOIO_TYPE_IN)) + rax = lower_bits(regs->ax, bits); + + ghcb_set_rax(ghcb, rax); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_IN) { + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + regs->ax = lower_bits(ghcb->save.rax, bits); + } + } + + return ret; +} -- Gitee From 6b03ce4357a819259f48ac63781a4891af3e301a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:27 +0200 Subject: [PATCH 055/113] x86/fpu: Move xgetbv()/xsetbv() into a separate header commit 1b4fb8545f2b00f2844c4b7619d64d98440a477c upstream. The xgetbv() function is needed in the pre-decompression boot code, but asm/fpu/internal.h can't be included there directly. Doing so opens the door to include-hell due to various include-magic in boot/compressed/misc.h. Avoid that by moving xgetbv()/xsetbv() to a separate header file and include it instead. Hygon-SIG: commit 1b4fb8545f2b upstream x86/fpu: Move xgetbv()/xsetbv() into a separate header Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-27-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/fpu/internal.h, arch/x86/include/asm/fpu/xcr.h] Signed-off-by: hanliyang --- arch/x86/include/asm/fpu/internal.h | 33 +--------------------------- arch/x86/include/asm/fpu/xcr.h | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 arch/x86/include/asm/fpu/xcr.h diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 20e3f67845f1..ee9658a3ec37 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -664,36 +665,4 @@ static inline void switch_kernel_fpu_finish(struct task_struct *next) } #endif -/* - * MXCSR and XCR definitions: - */ - -static inline void ldmxcsr(u32 mxcsr) -{ - asm volatile("ldmxcsr %0" :: "m" (mxcsr)); -} - -extern unsigned int mxcsr_feature_mask; - -#define XCR_XFEATURE_ENABLED_MASK 0x00000000 - -static inline u64 xgetbv(u32 index) -{ - u32 eax, edx; - - asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */ - : "=a" (eax), "=d" (edx) - : "c" (index)); - return eax + ((u64)edx << 32); -} - -static inline void xsetbv(u32 index, u64 value) -{ - u32 eax = value; - u32 edx = value >> 32; - - asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */ - : : "a" (eax), "d" (edx), "c" (index)); -} - #endif /* _ASM_X86_FPU_INTERNAL_H */ diff --git a/arch/x86/include/asm/fpu/xcr.h b/arch/x86/include/asm/fpu/xcr.h new file mode 100644 index 000000000000..1c7ab8d95da5 --- /dev/null +++ b/arch/x86/include/asm/fpu/xcr.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_FPU_XCR_H +#define _ASM_X86_FPU_XCR_H + +/* + * MXCSR and XCR definitions: + */ + +static inline void ldmxcsr(u32 mxcsr) +{ + asm volatile("ldmxcsr %0" :: "m" (mxcsr)); +} + +extern unsigned int mxcsr_feature_mask; + +#define XCR_XFEATURE_ENABLED_MASK 0x00000000 + +static inline u64 xgetbv(u32 index) +{ + u32 eax, edx; + + asm volatile("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index)); + return eax + ((u64)edx << 32); +} + +static inline void xsetbv(u32 index, u64 value) +{ + u32 eax = value; + u32 edx = value >> 32; + + asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); +} + +#endif /* _ASM_X86_FPU_XCR_H */ -- Gitee From 7bce24a86aed42ab3522dd568c4b7a0f9ce65de1 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:28 +0200 Subject: [PATCH 056/113] x86/sev-es: Add CPUID handling to #VC handler commit a7de15d489d956217b47671705ac2218ca50eaae upstream. Handle #VC exceptions caused by CPUID instructions. These happen in early boot code when the KASLR code checks for RDTSC. Hygon-SIG: commit a7de15d489d9 upstream x86/sev-es: Add CPUID handling to #VC handler Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-28-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/sev-es.c | 4 ++++ arch/x86/kernel/sev-es-shared.c | 35 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 61504eb1ab46..b1790f487456 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -183,6 +184,9 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) case SVM_EXIT_IOIO: result = vc_handle_ioio(boot_ghcb, &ctxt); break; + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(boot_ghcb, &ctxt); + break; default: result = ES_UNSUPPORTED; break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index bae7cf28455b..a6b41910b8ab 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -432,3 +432,38 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } + +static enum es_result vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u32 cr4 = native_read_cr4(); + enum es_result ret; + + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rcx(ghcb, regs->cx); + + if (cr4 & X86_CR4_OSXSAVE) + /* Safe to read xcr0 */ + ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); + else + /* xgetbv will cause #GP - use reset value for xcr0 */ + ghcb_set_xcr0(ghcb, 1); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && + ghcb_rbx_is_valid(ghcb) && + ghcb_rcx_is_valid(ghcb) && + ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + regs->ax = ghcb->save.rax; + regs->bx = ghcb->save.rbx; + regs->cx = ghcb->save.rcx; + regs->dx = ghcb->save.rdx; + + return ES_OK; +} -- Gitee From 0a067bea52c0151b3413f99c4f6371eca1ec1e56 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:29 +0200 Subject: [PATCH 057/113] x86/idt: Split idt_data setup out of set_intr_gate() commit 4bed2266cc6f9c3f6cd91378ea4fc76edde674cf upstream. The code to setup idt_data is needed for early exception handling, but set_intr_gate() can't be used that early because it has pv-ops in its code path which don't work that early. Split out the idt_data initialization part from set_intr_gate() so that it can be used separately. Hygon-SIG: commit 4bed2266cc6f upstream x86/idt: Split idt_data setup out of set_intr_gate() Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-29-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/idt.c] Signed-off-by: hanliyang --- arch/x86/kernel/idt.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 7bb4c3cbf4dc..530ae8aa1d03 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -227,18 +227,24 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy } } +static void init_idt_data(struct idt_data *data, unsigned int n, + const void *addr) +{ + BUG_ON(n > 0xFF); + + memset(data, 0, sizeof(*data)); + data->vector = n; + data->addr = addr; + data->segment = __KERNEL_CS; + data->bits.type = GATE_INTERRUPT; + data->bits.p = 1; +} + static void set_intr_gate(unsigned int n, const void *addr) { struct idt_data data; - BUG_ON(n > 0xFF); - - memset(&data, 0, sizeof(data)); - data.vector = n; - data.addr = addr; - data.segment = __KERNEL_CS; - data.bits.type = GATE_INTERRUPT; - data.bits.p = 1; + init_idt_data(&data, n, addr); idt_setup_from_table(idt_table, &data, 1, false); } -- Gitee From 15ac48dcbffd195dc5b09ea069d3774724f2f2b0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:30 +0200 Subject: [PATCH 058/113] x86/head/64: Install startup GDT commit 866b556efa1295934ed0bc20c2f208c93a873fb0 upstream. Handling exceptions during boot requires a working GDT. The kernel GDT can't be used on the direct mapping, so load a startup GDT and setup segments. Hygon-SIG: commit 866b556efa12 upstream x86/head/64: Install startup GDT Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-30-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/setup.h | 1 + arch/x86/kernel/head64.c | 33 +++++++++++++++++++++++++++++++++ arch/x86/kernel/head_64.S | 14 ++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index ed8ec011a9fd..6dffbae9153a 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -48,6 +48,7 @@ extern void reserve_standard_io_resources(void); extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp); extern unsigned long __startup_secondary_64(void); +extern void startup_64_setup_env(unsigned long physbase); extern int early_make_pgtable(unsigned long address); #ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 950286016f63..b02274f57804 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -61,6 +61,24 @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); #endif +/* + * GDT used on the boot CPU before switching to virtual addresses. + */ +static struct desc_struct startup_gdt[GDT_ENTRIES] = { + [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff), +}; + +/* + * Address needs to be set at runtime because it references the startup_gdt + * while the kernel still uses a direct mapping. + */ +static struct desc_ptr startup_gdt_descr = { + .size = sizeof(startup_gdt), + .address = 0, +}; + #define __head __section(.head.text) static void __head *fixup_pointer(void *ptr, unsigned long physaddr) @@ -491,3 +509,18 @@ void __init x86_64_start_reservations(char *real_mode_data) start_kernel(); } + +/* + * Setup boot CPU state needed before kernel switches to virtual addresses. + */ +void __head startup_64_setup_env(unsigned long physbase) +{ + /* Load GDT */ + startup_gdt_descr.address = (unsigned long)fixup_pointer(startup_gdt, physbase); + native_load_gdt(&startup_gdt_descr); + + /* New GDT is live - reload data segment registers */ + asm volatile("movl %%eax, %%ds\n" + "movl %%eax, %%ss\n" + "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); +} diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 1a02ef0e2d9f..a35674babe01 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -73,6 +73,20 @@ startup_64: /* Set up the stack for verify_cpu(), similar to initial_stack below */ leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp + leaq _text(%rip), %rdi + pushq %rsi + call startup_64_setup_env + popq %rsi + + /* Now switch to __KERNEL_CS so IRET works reliably */ + pushq $__KERNEL_CS + leaq .Lon_kernel_cs(%rip), %rax + pushq %rax + lretq + +.Lon_kernel_cs: + UNWIND_HINT_EMPTY + /* Sanitize CPU configuration */ call verify_cpu -- Gitee From 9829058e3b8ef3cef5ea45bdb392177087577bf7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:31 +0200 Subject: [PATCH 059/113] x86/head/64: Load GDT after switch to virtual addresses commit e04b88336360e101329add0c05e5cb1cebae64fd upstream. Load the GDT right after switching to virtual addresses to make sure there is a defined GDT for exception handling. Hygon-SIG: commit e04b88336360 upstream x86/head/64: Load GDT after switch to virtual addresses Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-31-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/head_64.S | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index a35674babe01..955969244a42 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -156,6 +156,14 @@ ENTRY(secondary_startup_64) 1: UNWIND_HINT_EMPTY + /* + * We must switch to a new descriptor in kernel space for the GDT + * because soon the kernel won't have access anymore to the userspace + * addresses where we're currently running on. We have to do that here + * because in 32bit we couldn't load a 64bit linear address. + */ + lgdt early_gdt_descr(%rip) + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -183,14 +191,6 @@ ENTRY(secondary_startup_64) pushq $0 popfq - /* - * We must switch to a new descriptor in kernel space for the GDT - * because soon the kernel won't have access anymore to the userspace - * addresses where we're currently running on. We have to do that here - * because in 32bit we couldn't load a 64bit linear address. - */ - lgdt early_gdt_descr(%rip) - /* set up data segments */ xorl %eax,%eax movl %eax,%ds -- Gitee From 7b7573c8f57c79e84443b9d3ccba21ed08340508 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:32 +0200 Subject: [PATCH 060/113] x86/head/64: Load segment registers earlier commit 7b99819dfb60268cc1c75f83c949bc4a09221bea upstream. Make sure segments are properly set up before setting up an IDT and doing anything that might cause a #VC exception. This is later needed for early exception handling. Hygon-SIG: commit 7b99819dfb60 upstream x86/head/64: Load segment registers earlier Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-32-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/head_64.S | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 955969244a42..b3a8bda1bbd6 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -164,6 +164,32 @@ ENTRY(secondary_startup_64) */ lgdt early_gdt_descr(%rip) + /* set up data segments */ + xorl %eax,%eax + movl %eax,%ds + movl %eax,%ss + movl %eax,%es + + /* + * We don't really need to load %fs or %gs, but load them anyway + * to kill any stale realmode selectors. This allows execution + * under VT hardware. + */ + movl %eax,%fs + movl %eax,%gs + + /* Set up %gs. + * + * The base of %gs always points to fixed_percpu_data. If the + * stack protector canary is enabled, it is located at %gs:40. + * Note that, on SMP, the boot cpu uses init data section until + * the per cpu areas are set up. + */ + movl $MSR_GS_BASE,%ecx + movl initial_gs(%rip),%eax + movl initial_gs+4(%rip),%edx + wrmsr + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -191,32 +217,6 @@ ENTRY(secondary_startup_64) pushq $0 popfq - /* set up data segments */ - xorl %eax,%eax - movl %eax,%ds - movl %eax,%ss - movl %eax,%es - - /* - * We don't really need to load %fs or %gs, but load them anyway - * to kill any stale realmode selectors. This allows execution - * under VT hardware. - */ - movl %eax,%fs - movl %eax,%gs - - /* Set up %gs. - * - * The base of %gs always points to fixed_percpu_data. If the - * stack protector canary is enabled, it is located at %gs:40. - * Note that, on SMP, the boot cpu uses init data section until - * the per cpu areas are set up. - */ - movl $MSR_GS_BASE,%ecx - movl initial_gs(%rip),%eax - movl initial_gs+4(%rip),%edx - wrmsr - /* rsi is pointer to real mode structure with interesting info. pass it to C */ movq %rsi, %rdi -- Gitee From 7b9ef2885fa1f4186a86b7eeacc35887adb8463f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:33 +0200 Subject: [PATCH 061/113] x86/head/64: Switch to initial stack earlier commit 3add38cb96a1ae7d152db69ab4329e809c2af2d4 upstream. Make sure there is a stack once the kernel runs from virtual addresses. At this stage any secondary CPU which boots will have lost its stack because the kernel switched to a new page-table which does not map the real-mode stack anymore. This is needed for handling early #VC exceptions caused by instructions like CPUID. Hygon-SIG: commit 3add38cb96a1 upstream x86/head/64: Switch to initial stack earlier Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-33-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/head_64.S | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index b3a8bda1bbd6..6c027bd2b470 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -190,6 +190,12 @@ ENTRY(secondary_startup_64) movl initial_gs+4(%rip),%edx wrmsr + /* + * Setup a boot time stack - Any secondary CPU will have lost its stack + * by now because the cr3-switch above unmaps the real-mode stack + */ + movq initial_stack(%rip), %rsp + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -210,9 +216,6 @@ ENTRY(secondary_startup_64) /* Make changes effective */ movq %rax, %cr0 - /* Setup a boot time stack */ - movq initial_stack(%rip), %rsp - /* zero EFLAGS after setting rsp */ pushq $0 popfq -- Gitee From 9579e9923f28728ea9783e6e7e690ed27e01a0f2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:34 +0200 Subject: [PATCH 062/113] x86/head/64: Install a CPU bringup IDT commit f5963ba7a45fc6ff298a34976064354be437e1d8 upstream. Add a separate bringup IDT for the CPU bringup code that will be used until the kernel switches to the idt_table. There are two reasons for a separate IDT: 1) When the idt_table is set up and the secondary CPUs are booted, it contains entries (e.g. IST entries) which require certain CPU state to be set up. This includes a working TSS (for IST), MSR_GS_BASE (for stack protector) or CR4.FSGSBASE (for paranoid_entry) path. By using a dedicated IDT for early boot this state need not to be set up early. 2) The idt_table is static to idt.c, so any function using/modifying must be in idt.c too. That means that all compiler driven instrumentation like tracing or KASAN is also active in this code. But during early CPU bringup the environment is not set up for this instrumentation to work correctly. To avoid all of these hassles and make early exception handling robust, use a dedicated bringup IDT. The IDT is loaded two times, first on the boot CPU while the kernel is still running on direct mapped addresses, and again later after the switch to kernel addresses has happened. The second IDT load happens on the boot and secondary CPUs. Hygon-SIG: commit f5963ba7a45f upstream x86/head/64: Install a CPU bringup IDT Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-34-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/setup.h | 1 + arch/x86/kernel/head64.c | 39 ++++++++++++++++++++++++++++++++++++ arch/x86/kernel/head_64.S | 5 +++++ 3 files changed, 45 insertions(+) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 6dffbae9153a..97ef7613ba89 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -50,6 +50,7 @@ extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp extern unsigned long __startup_secondary_64(void); extern void startup_64_setup_env(unsigned long physbase); extern int early_make_pgtable(unsigned long address); +extern void early_setup_idt(void); #ifdef CONFIG_X86_INTEL_MID extern void x86_intel_mid_early_setup(void); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index b02274f57804..19ff46c18a17 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include /* * Manage page tables very early on. @@ -510,6 +512,41 @@ void __init x86_64_start_reservations(char *real_mode_data) start_kernel(); } +/* + * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is + * used until the idt_table takes over. On the boot CPU this happens in + * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases + * this happens in the functions called from head_64.S. + * + * The idt_table can't be used that early because all the code modifying it is + * in idt.c and can be instrumented by tracing or KASAN, which both don't work + * during early CPU bringup. Also the idt_table has the runtime vectors + * configured which require certain CPU state to be setup already (like TSS), + * which also hasn't happened yet in early CPU bringup. + */ +static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; + +static struct desc_ptr bringup_idt_descr = { + .size = (NUM_EXCEPTION_VECTORS * sizeof(gate_desc)) - 1, + .address = 0, /* Set at runtime */ +}; + +/* This runs while still in the direct mapping */ +static void startup_64_load_idt(unsigned long physbase) +{ + struct desc_ptr *desc = fixup_pointer(&bringup_idt_descr, physbase); + + desc->address = (unsigned long)fixup_pointer(bringup_idt_table, physbase); + native_load_idt(desc); +} + +/* This is used when running on kernel addresses */ +void early_setup_idt(void) +{ + bringup_idt_descr.address = (unsigned long)bringup_idt_table; + native_load_idt(&bringup_idt_descr); +} + /* * Setup boot CPU state needed before kernel switches to virtual addresses. */ @@ -523,4 +560,6 @@ void __head startup_64_setup_env(unsigned long physbase) asm volatile("movl %%eax, %%ds\n" "movl %%eax, %%ss\n" "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); + + startup_64_load_idt(physbase); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 6c027bd2b470..0cff33a97145 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -196,6 +196,11 @@ ENTRY(secondary_startup_64) */ movq initial_stack(%rip), %rsp + /* Setup and Load IDT */ + pushq %rsi + call early_setup_idt + popq %rsi + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid -- Gitee From b21c3cd852c3fe6e3f6a4cc22b2f197d5f635873 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:35 +0200 Subject: [PATCH 063/113] x86/idt: Make IDT init functions static inlines commit 097ee5b778b8970e1c2ed3ca1631b297d90acd61 upstream. Move these two functions from kernel/idt.c to include/asm/desc.h: * init_idt_data() * idt_init_desc() These functions are needed to setup IDT entries very early and need to be called from head64.c. To be usable this early, these functions need to be compiled without instrumentation and the stack-protector feature. These features need to be kept enabled for kernel/idt.c, so head64.c must use its own versions. [ bp: Take Kees' suggested patch title and add his Rev-by. ] Hygon-SIG: commit 097ee5b778b8 upstream x86/idt: Make IDT init functions static inlines Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-35-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/desc.h, arch/x86/include/asm/desc_defs.h, arch/x86/kernel/idt.c] Signed-off-by: hanliyang --- arch/x86/include/asm/desc.h | 27 +++++++++++++++++++++++++ arch/x86/include/asm/desc_defs.h | 7 +++++++ arch/x86/kernel/idt.c | 34 -------------------------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 68a99d2a5f33..80bf63c08007 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -389,6 +389,33 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) void update_intr_gate(unsigned int n, const void *addr); void alloc_intr_gate(unsigned int n, const void *addr); +static inline void init_idt_data(struct idt_data *data, unsigned int n, + const void *addr) +{ + BUG_ON(n > 0xFF); + + memset(data, 0, sizeof(*data)); + data->vector = n; + data->addr = addr; + data->segment = __KERNEL_CS; + data->bits.type = GATE_INTERRUPT; + data->bits.p = 1; +} + +static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) +{ + unsigned long addr = (unsigned long) d->addr; + + gate->offset_low = (u16) addr; + gate->segment = (u16) d->segment; + gate->bits = d->bits; + gate->offset_middle = (u16) (addr >> 16); +#ifdef CONFIG_X86_64 + gate->offset_high = (u32) (addr >> 32); + gate->reserved = 0; +#endif +} + extern unsigned long system_vectors[]; #ifdef CONFIG_X86_64 diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h index 5621fb3f2d1a..f7e7099af595 100644 --- a/arch/x86/include/asm/desc_defs.h +++ b/arch/x86/include/asm/desc_defs.h @@ -74,6 +74,13 @@ struct idt_bits { p : 1; } __attribute__((packed)); +struct idt_data { + unsigned int vector; + unsigned int segment; + struct idt_bits bits; + const void *addr; +}; + struct gate_struct { u16 offset_low; u16 segment; diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 530ae8aa1d03..1520b28b82e2 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -9,13 +9,6 @@ #include #include -struct idt_data { - unsigned int vector; - unsigned int segment; - struct idt_bits bits; - const void *addr; -}; - #define DPL0 0x0 #define DPL3 0x3 @@ -200,20 +193,6 @@ const struct desc_ptr debug_idt_descr = { }; #endif -static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) -{ - unsigned long addr = (unsigned long) d->addr; - - gate->offset_low = (u16) addr; - gate->segment = (u16) d->segment; - gate->bits = d->bits; - gate->offset_middle = (u16) (addr >> 16); -#ifdef CONFIG_X86_64 - gate->offset_high = (u32) (addr >> 32); - gate->reserved = 0; -#endif -} - static void idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) { @@ -227,19 +206,6 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy } } -static void init_idt_data(struct idt_data *data, unsigned int n, - const void *addr) -{ - BUG_ON(n > 0xFF); - - memset(data, 0, sizeof(*data)); - data->vector = n; - data->addr = addr; - data->segment = __KERNEL_CS; - data->bits.type = GATE_INTERRUPT; - data->bits.p = 1; -} - static void set_intr_gate(unsigned int n, const void *addr) { struct idt_data data; -- Gitee From 28b6f11800ebd89cb9f799a4a256f307c77d7d48 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:36 +0200 Subject: [PATCH 064/113] x86/head/64: Move early exception dispatch to C code commit 4b47cdbda6f1ad73b08dc7d497bac12b8f26ae0d upstream. Move the assembly coded dispatch between page-faults and all other exceptions to C code to make it easier to maintain and extend. Also change the return-type of early_make_pgtable() to bool and make it static. Hygon-SIG: commit 4b47cdbda6f1 upstream x86/head/64: Move early exception dispatch to C code Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-36-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/pgtable.h, arch/x86/include/asm/setup.h, arch/x86/kernel/head64.c, arch/x86/kernel/head_64.S] Signed-off-by: hanliyang --- arch/x86/include/asm/pgtable.h | 2 +- arch/x86/include/asm/setup.h | 4 +++- arch/x86/kernel/head64.c | 19 +++++++++++++++---- arch/x86/kernel/head_64.S | 11 +---------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 60fe13c52b09..ef37dd64fe13 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -27,7 +27,7 @@ #include extern pgd_t early_top_pgt[PTRS_PER_PGD]; -int __init __early_make_pgtable(unsigned long address, pmdval_t pmd); +bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user); diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 97ef7613ba89..4801746c4af8 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -39,6 +39,8 @@ void vsmp_init(void); static inline void vsmp_init(void) { } #endif +struct pt_regs; + void setup_bios_corruption_check(void); void early_platform_quirks(void); @@ -49,8 +51,8 @@ extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp); extern unsigned long __startup_secondary_64(void); extern void startup_64_setup_env(unsigned long physbase); -extern int early_make_pgtable(unsigned long address); extern void early_setup_idt(void); +extern void __init do_early_exception(struct pt_regs *regs, int trapnr); #ifdef CONFIG_X86_INTEL_MID extern void x86_intel_mid_early_setup(void); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 19ff46c18a17..bef72aa05442 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include /* * Manage page tables very early on. @@ -317,7 +319,7 @@ static void __init reset_early_page_tables(void) } /* Create a new PMD entry */ -int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) +bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd) { unsigned long physaddr = address - __PAGE_OFFSET; pgdval_t pgd, *pgd_p; @@ -327,7 +329,7 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) /* Invalid address or early pgt is done ? */ if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt)) - return -1; + return false; again: pgd_p = &early_top_pgt[pgd_index(address)].pgd; @@ -384,10 +386,10 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) } pmd_p[pmd_index(address)] = pmd; - return 0; + return true; } -int __init early_make_pgtable(unsigned long address) +static bool __init early_make_pgtable(unsigned long address) { unsigned long physaddr = address - __PAGE_OFFSET; pmdval_t pmd; @@ -397,6 +399,15 @@ int __init early_make_pgtable(unsigned long address) return __early_make_pgtable(address, pmd); } +void __init do_early_exception(struct pt_regs *regs, int trapnr) +{ + if (trapnr == X86_TRAP_PF && + early_make_pgtable(native_read_cr2())) + return; + + early_fixup_exception(regs, trapnr); +} + /* Don't add a printk in there. printk relies on the PDA which is not initialized yet. */ static void __init clear_bss(void) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 0cff33a97145..9383e4de0975 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -341,18 +341,9 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) pushq %r15 /* pt_regs->r15 */ UNWIND_HINT_REGS - cmpq $14,%rsi /* Page fault? */ - jnz 10f - GET_CR2_INTO(%rdi) /* can clobber %rax if pv */ - call early_make_pgtable - andl %eax,%eax - jz 20f /* All good */ - -10: movq %rsp,%rdi /* RDI = pt_regs; RSI is already trapnr */ - call early_fixup_exception + call do_early_exception -20: decl early_recursion_flag(%rip) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) -- Gitee From f9313de477f067fb39dcef5cc59b9e2c3b9c6e3b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:37 +0200 Subject: [PATCH 065/113] x86/sev-es: Add SEV-ES Feature Detection commit b57de6cd16395be1ebdaa9b489ffbf462bb585c4 upstream. Add a sev_es_active() function for checking whether SEV-ES is enabled. Also cache the value of MSR_AMD64_SEV at boot to speed up the feature checking in the running code. [ bp: Remove "!!" in sev_active() too. ] Hygon-SIG: commit b57de6cd1639 upstream x86/sev-es: Add SEV-ES Feature Detection Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-37-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/mem_encrypt.h | 3 +++ arch/x86/include/asm/msr-index.h | 2 ++ arch/x86/mm/mem_encrypt.c | 9 ++++++++- arch/x86/mm/mem_encrypt_identity.c | 3 +++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 454939a0a95e..9d80b43669cd 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -19,6 +19,7 @@ #ifdef CONFIG_AMD_MEM_ENCRYPT extern u64 sme_me_mask; +extern u64 sev_status; extern bool sev_enabled; void sme_encrypt_execute(unsigned long encrypted_kernel_vaddr, @@ -51,6 +52,7 @@ void __init mem_encrypt_free_decrypted_mem(void); bool sme_active(void); bool sev_active(void); +bool sev_es_active(void); #define __bss_decrypted __attribute__((__section__(".bss..decrypted"))) @@ -73,6 +75,7 @@ static inline void __init sme_enable(struct boot_params *bp) { } static inline bool sme_active(void) { return false; } static inline bool sev_active(void) { return false; } +static inline bool sev_es_active(void) { return false; } static inline int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0; } diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 371dfbaba396..3f90bd07451c 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -502,7 +502,9 @@ #define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 +#define MSR_AMD64_SEV_ES_ENABLED_BIT 1 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) +#define MSR_AMD64_SEV_ES_ENABLED BIT_ULL(MSR_AMD64_SEV_ES_ENABLED_BIT) #define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 86aa3d2f14c5..9448a78b47d2 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -40,6 +40,7 @@ * section is later cleared. */ u64 sme_me_mask __section(.data) = 0; +u64 sev_status __section(.data) = 0; EXPORT_SYMBOL(sme_me_mask); DEFINE_STATIC_KEY_FALSE(sev_enable_key); EXPORT_SYMBOL_GPL(sev_enable_key); @@ -403,7 +404,13 @@ bool sme_active(void) bool sev_active(void) { - return sme_me_mask && sev_enabled; + return sev_status & MSR_AMD64_SEV_ENABLED; +} + +/* Needs to be called from non-instrumentable code */ +bool noinstr sev_es_active(void) +{ + return sev_status & MSR_AMD64_SEV_ES_ENABLED; } /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */ diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 77d4aa57fd35..a7cd0466c0fc 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -549,6 +549,9 @@ void __init sme_enable(struct boot_params *bp) if (!(msr & MSR_AMD64_SEV_ENABLED)) return; + /* Save SEV_STATUS to avoid reading MSR again */ + sev_status = msr; + /* SEV state cannot be controlled by a command line option */ sme_me_mask = me_mask; sev_enabled = true; -- Gitee From bb5efc23b07f07141b65079769e88f28593c777c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:38 +0200 Subject: [PATCH 066/113] x86/sev-es: Print SEV-ES info into the kernel log commit c685eb0c12b4d4816d22ee734e91f4005b152fcd upstream. Refactor the message printed to the kernel log which indicates whether SEV or SME, etc is active. This will scale better in the future when more memory encryption features might be added. Also add SEV-ES to the list of features. [ bp: Massage. ] Hygon-SIG: commit c685eb0c12b4 upstream x86/sev-es: Print SEV-ES info into the kernel log Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-38-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/mm/mem_encrypt.c] [hanliyang: Supplement the HYGON CSV2 print to print_hygon_cc_feature_info(), which is lost because the "x86/mm: Print CSV info into the kernel log" patch is ported ahead of this one, where the SEV-ES feature detection is not available yet] Signed-off-by: hanliyang --- arch/x86/mm/mem_encrypt.c | 30 +++++++++++++++++++++++++++--- arch/x86/mm/mem_encrypt_hygon.c | 4 ++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 9448a78b47d2..c3ce11d715bf 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -465,6 +465,32 @@ void __init mem_encrypt_free_decrypted_mem(void) free_init_pages("unused decrypted", vaddr, vaddr_end); } +static void print_mem_encrypt_feature_info(void) +{ + pr_info("AMD Memory Encryption Features active:"); + + /* Secure Memory Encryption */ + if (sme_active()) { + /* + * SME is mutually exclusive with any of the SEV + * features below. + */ + pr_cont(" SME\n"); + return; + } + + /* Secure Encrypted Virtualization */ + if (sev_active()) + pr_cont(" SEV"); + + /* Encrypted Register State */ + if (sev_es_active()) + pr_cont(" SEV-ES"); + + pr_cont("\n"); +} + +/* Architecture __weak replacement functions */ void __init mem_encrypt_init(void) { if (!sme_me_mask) @@ -484,8 +510,6 @@ void __init mem_encrypt_init(void) return; } - pr_info("AMD %s active\n", - sev_active() ? "Secure Encrypted Virtualization (SEV)" - : "Secure Memory Encryption (SME)"); + print_mem_encrypt_feature_info(); } diff --git a/arch/x86/mm/mem_encrypt_hygon.c b/arch/x86/mm/mem_encrypt_hygon.c index cc5e5f2e6344..6a278764d3b0 100644 --- a/arch/x86/mm/mem_encrypt_hygon.c +++ b/arch/x86/mm/mem_encrypt_hygon.c @@ -32,4 +32,8 @@ void print_hygon_cc_feature_info(void) /* Secure Encrypted Virtualization */ pr_info(" HYGON CSV"); } + + /* Encrypted Register State */ + if (sev_es_active()) + pr_info(" HYGON CSV2"); } -- Gitee From 77213a69bfba21c9cbffe02719729c4108deb8da Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:39 +0200 Subject: [PATCH 067/113] x86/sev-es: Compile early handler code into kernel image commit f980f9c31a923e9040dee0bc679a5f5b09e61f40 upstream. Setup sev-es.c and include the code from the pre-decompression stage to also build it into the image of the running kernel. Temporarily add __maybe_unused annotations to avoid build warnings until the functions get used. [ bp: Use the non-tracing rd/wrmsr variants because: vmlinux.o: warning: objtool: __sev_es_nmi_complete()+0x11f: \ call to do_trace_write_msr() leaves .noinstr.text section as __sev_es_nmi_complete() is noinstr due to being called from the NMI handler exc_nmi(). ] Hygon-SIG: commit f980f9c31a92 upstream x86/sev-es: Compile early handler code into kernel image Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-39-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/Makefile, arch/x86/kernel/sev-es-shared.c, arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/sev-es-shared.c | 21 ++-- arch/x86/kernel/sev-es.c | 163 ++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 arch/x86/kernel/sev-es.c diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 52ce1e239525..72a94c0d41e7 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -140,6 +140,7 @@ obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o +obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev-es.o ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index a6b41910b8ab..18619279a46f 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,7 +9,7 @@ * and is included directly into both code-bases. */ -static void sev_es_terminate(unsigned int reason) +static void __maybe_unused sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; @@ -27,7 +27,7 @@ static void sev_es_terminate(unsigned int reason) asm volatile("hlt\n" : : : "memory"); } -static bool sev_es_negotiate_protocol(void) +static bool __maybe_unused sev_es_negotiate_protocol(void) { u64 val; @@ -46,7 +46,7 @@ static bool sev_es_negotiate_protocol(void) return true; } -static void vc_ghcb_invalidate(struct ghcb *ghcb) +static void __maybe_unused vc_ghcb_invalidate(struct ghcb *ghcb) { memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } @@ -58,9 +58,9 @@ static bool vc_decoding_needed(unsigned long exit_code) exit_code <= SVM_EXIT_LAST_EXCP); } -static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, - struct pt_regs *regs, - unsigned long exit_code) +static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) { enum es_result ret = ES_OK; @@ -73,7 +73,7 @@ static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, return ret; } -static void vc_finish_insn(struct es_em_ctxt *ctxt) +static void __maybe_unused vc_finish_insn(struct es_em_ctxt *ctxt) { ctxt->regs->ip += ctxt->insn.length; } @@ -325,7 +325,8 @@ static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) return ES_OK; } -static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +static enum es_result __maybe_unused +vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u64 exit_info_1, exit_info_2; @@ -433,8 +434,8 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } -static enum es_result vc_handle_cpuid(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) +static enum es_result __maybe_unused vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u32 cr4 = native_read_cr4(); diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c new file mode 100644 index 000000000000..c359d9b6d23a --- /dev/null +++ b/arch/x86/kernel/sev-es.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +static inline u64 sev_es_rd_ghcb_msr(void) +{ + return __rdmsr(MSR_AMD64_SEV_ES_GHCB); +} + +static inline void sev_es_wr_ghcb_msr(u64 val) +{ + u32 low, high; + + low = (u32)(val); + high = (u32)(val >> 32); + + native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); +} + +static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, + unsigned char *buffer) +{ + return probe_kernel_read(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); +} + +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + enum es_result ret; + int res; + + res = vc_fetch_insn_kernel(ctxt, buffer); + if (unlikely(res == -EFAULT)) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = 0; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); + insn_get_length(&ctxt->insn); + + ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; + + return ret; +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + char *dst, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; + char __user *target = (char __user *)dst; + u64 d8; + u32 d4; + u16 d2; + u8 d1; + + switch (size) { + case 1: + memcpy(&d1, buf, 1); + if (put_user(d1, target)) + goto fault; + break; + case 2: + memcpy(&d2, buf, 2); + if (put_user(d2, target)) + goto fault; + break; + case 4: + memcpy(&d4, buf, 4); + if (put_user(d4, target)) + goto fault; + break; + case 8: + memcpy(&d8, buf, 8); + if (put_user(d8, target)) + goto fault; + break; + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)dst; + + return ES_EXCEPTION; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + char *src, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT; + char __user *s = (char __user *)src; + u64 d8; + u32 d4; + u16 d2; + u8 d1; + + switch (size) { + case 1: + if (get_user(d1, s)) + goto fault; + memcpy(buf, &d1, 1); + break; + case 2: + if (get_user(d2, s)) + goto fault; + memcpy(buf, &d2, 2); + break; + case 4: + if (get_user(d4, s)) + goto fault; + memcpy(buf, &d4, 4); + break; + case 8: + if (get_user(d8, s)) + goto fault; + memcpy(buf, &d8, 8); + break; + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)src; + + return ES_EXCEPTION; +} + +/* Include code shared with pre-decompression boot stage */ +#include "sev-es-shared.c" -- Gitee From 6e1626c6fe6db52c33bd587a58b894dd1eede584 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 8 Sep 2020 14:35:17 +0200 Subject: [PATCH 068/113] x86/sev-es: Setup an early #VC handler commit 74d8d9d531b4cc945a9f75aa2fc21d99ca5a9fe3 upstream. Setup an early handler for #VC exceptions. There is no GHCB mapped yet, so just re-use the vc_no_ghcb_handler(). It can only handle CPUID exit-codes, but that should be enough to get the kernel through verify_cpu() and __startup_64() until it runs on virtual addresses. Hygon-SIG: commit 74d8d9d531b4 upstream x86/sev-es: Setup an early #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov [ boot failure Error: kernel_ident_mapping_init() failed. ] Reported-by: kernel test robot Link: https://lkml.kernel.org/r/20200908123517.GA3764@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/sev-es.h, arch/x86/kernel/head64.c, arch/x86/kernel/head_64.S] Signed-off-by: hanliyang --- arch/x86/include/asm/sev-es.h | 3 +++ arch/x86/kernel/head64.c | 25 ++++++++++++++++++++++++- arch/x86/kernel/head_64.S | 31 +++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 6dc52440c4b4..7175d432ebfe 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -73,4 +73,7 @@ static inline u64 lower_bits(u64 val, unsigned int bits) return (val & mask); } +/* Early IDT entry points for #VC handler */ +extern void vc_no_ghcb(void); + #endif diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index bef72aa05442..c3dd5367ef28 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -40,6 +40,7 @@ #include #include #include +#include /* * Manage page tables very early on. @@ -542,12 +543,34 @@ static struct desc_ptr bringup_idt_descr = { .address = 0, /* Set at runtime */ }; +static void set_bringup_idt_handler(gate_desc *idt, int n, void *handler) +{ +#ifdef CONFIG_AMD_MEM_ENCRYPT + struct idt_data data; + gate_desc desc; + + init_idt_data(&data, n, handler); + idt_init_desc(&desc, &data); + native_write_idt_entry(idt, n, &desc); +#endif +} + /* This runs while still in the direct mapping */ static void startup_64_load_idt(unsigned long physbase) { struct desc_ptr *desc = fixup_pointer(&bringup_idt_descr, physbase); + gate_desc *idt = fixup_pointer(bringup_idt_table, physbase); + + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + void *handler; + + /* VMM Communication Exception */ + handler = fixup_pointer(vc_no_ghcb, physbase); + set_bringup_idt_handler(idt, X86_TRAP_VC, handler); + } - desc->address = (unsigned long)fixup_pointer(bringup_idt_table, physbase); + desc->address = (unsigned long)idt; native_load_idt(desc); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 9383e4de0975..a78b050aa477 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -348,6 +348,37 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during very early boot. The + * early_idt_handler_array can't be used because it returns via the + * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. + * + * This handler will end up in the .init.text section and not be + * available to boot secondary CPUs. + */ +SYM_CODE_START_NOALIGN(vc_no_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_no_ghcb) +#endif + __INITDATA .balign 4 -- Gitee From 008624037f8efc2130cc645dc99fca7457605ed6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 8 Sep 2020 14:38:16 +0200 Subject: [PATCH 069/113] x86/sev-es: Setup GHCB-based boot #VC handler commit 1aa9aa8ee517e0443b06e816a4fd2d15f2113615 upstream. Add the infrastructure to handle #VC exceptions when the kernel runs on virtual addresses and has mapped a GHCB. This handler will be used until the runtime #VC handler takes over. Since the handler runs very early, disable instrumentation for sev-es.c. [ bp: Make vc_ghcb_invalidate() __always_inline so that it can be inlined in noinstr functions like __sev_es_nmi_complete(). ] Hygon-SIG: commit 1aa9aa8ee517 upstream x86/sev-es: Setup GHCB-based boot #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200908123816.GB3764@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/realmode.h, arch/x86/include/asm/segment.h, arch/x86/include/asm/sev-es.h, arch/x86/kernel/Makefile, arch/x86/kernel/head64.c and 4 more] Signed-off-by: hanliyang --- arch/x86/include/asm/realmode.h | 3 + arch/x86/include/asm/segment.h | 2 +- arch/x86/include/asm/sev-es.h | 2 + arch/x86/kernel/Makefile | 2 + arch/x86/kernel/head64.c | 8 +++ arch/x86/kernel/head_64.S | 37 ++++++++++ arch/x86/kernel/sev-es-shared.c | 14 ++-- arch/x86/kernel/sev-es.c | 116 ++++++++++++++++++++++++++++++++ arch/x86/mm/extable.c | 1 + 9 files changed, 177 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index 52d7512ea91a..a55231c104c7 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -57,6 +57,9 @@ extern unsigned char real_mode_blob_end[]; extern unsigned long initial_code; extern unsigned long initial_gs; extern unsigned long initial_stack; +#ifdef CONFIG_AMD_MEM_ENCRYPT +extern unsigned long initial_vc_handler; +#endif extern unsigned char real_mode_blob[]; extern unsigned char real_mode_relocs[]; diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index 6669164abadc..5b648066504c 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -230,7 +230,7 @@ #define NUM_EXCEPTION_VECTORS 32 /* Bitmask of exception vectors which push an error code on the stack: */ -#define EXCEPTION_ERRCODE_MASK 0x00027d00 +#define EXCEPTION_ERRCODE_MASK 0x20027d00 #define GDT_SIZE (GDT_ENTRIES*8) #define GDT_ENTRY_TLS_ENTRIES 3 diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 7175d432ebfe..9fbeedaa66ee 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -75,5 +75,7 @@ static inline u64 lower_bits(u64 val, unsigned int bits) /* Early IDT entry points for #VC handler */ extern void vc_no_ghcb(void); +extern void vc_boot_ghcb(void); +extern bool handle_vc_boot_ghcb(struct pt_regs *regs); #endif diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 72a94c0d41e7..313aec582bec 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -20,6 +20,7 @@ CFLAGS_REMOVE_kvmclock.o = -pg CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg CFLAGS_REMOVE_head64.o = -pg +CFLAGS_REMOVE_sev-es.o = -pg endif KASAN_SANITIZE_head$(BITS).o := n @@ -27,6 +28,7 @@ KASAN_SANITIZE_dumpstack.o := n KASAN_SANITIZE_dumpstack_$(BITS).o := n KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_paravirt.o := n +KASAN_SANITIZE_sev-es.o := n OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index c3dd5367ef28..4f26a8d0a101 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -406,6 +406,10 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr) early_make_pgtable(native_read_cr2())) return; + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT) && + trapnr == X86_TRAP_VC && handle_vc_boot_ghcb(regs)) + return; + early_fixup_exception(regs, trapnr); } @@ -577,6 +581,10 @@ static void startup_64_load_idt(unsigned long physbase) /* This is used when running on kernel addresses */ void early_setup_idt(void) { + /* VMM Communication Exception */ + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + set_bringup_idt_handler(bringup_idt_table, X86_TRAP_VC, vc_boot_ghcb); + bringup_idt_descr.address = (unsigned long)bringup_idt_table; native_load_idt(&bringup_idt_descr); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index a78b050aa477..d1d2517ec804 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -277,6 +277,39 @@ ENTRY(start_cpu0) movq initial_stack(%rip), %rsp jmp .Ljump_to_C_code END(start_cpu0) +#endif + +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during early boot when running on kernel + * addresses, but before the switch to the idt_table can be made. + * The early_idt_handler_array can't be used here because it calls into a lot + * of __init code and this handler is also used during CPU offlining/onlining. + * Therefore this handler ends up in the .text section so that it stays around + * when .init.text is freed. + */ +SYM_CODE_START_NOALIGN(vc_boot_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + movq initial_vc_handler(%rip), %rax + ANNOTATE_RETPOLINE_SAFE + call *%rax + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_boot_ghcb) #endif /* Both SMP bootup and ACPI suspend change these variables */ @@ -286,6 +319,10 @@ END(start_cpu0) .quad x86_64_start_kernel GLOBAL(initial_gs) .quad INIT_PER_CPU_VAR(fixed_percpu_data) +#ifdef CONFIG_AMD_MEM_ENCRYPT + GLOBAL(initial_vc_handler) + .quad handle_vc_boot_ghcb +#endif GLOBAL(initial_stack) /* * The SIZEOF_PTREGS gap is a convention which helps the in-kernel diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 18619279a46f..53e3de0e8060 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,7 +9,7 @@ * and is included directly into both code-bases. */ -static void __maybe_unused sev_es_terminate(unsigned int reason) +static void sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; @@ -27,7 +27,7 @@ static void __maybe_unused sev_es_terminate(unsigned int reason) asm volatile("hlt\n" : : : "memory"); } -static bool __maybe_unused sev_es_negotiate_protocol(void) +static bool sev_es_negotiate_protocol(void) { u64 val; @@ -46,7 +46,7 @@ static bool __maybe_unused sev_es_negotiate_protocol(void) return true; } -static void __maybe_unused vc_ghcb_invalidate(struct ghcb *ghcb) +static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) { memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } @@ -58,9 +58,9 @@ static bool vc_decoding_needed(unsigned long exit_code) exit_code <= SVM_EXIT_LAST_EXCP); } -static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, - struct pt_regs *regs, - unsigned long exit_code) +static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) { enum es_result ret = ES_OK; @@ -73,7 +73,7 @@ static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, return ret; } -static void __maybe_unused vc_finish_insn(struct es_em_ctxt *ctxt) +static void vc_finish_insn(struct es_em_ctxt *ctxt) { ctxt->regs->ip += ctxt->insn.length; } diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index c359d9b6d23a..2ca0d9478081 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -7,7 +7,9 @@ * Author: Joerg Roedel */ +#include /* For show_regs() */ #include +#include #include #include @@ -18,6 +20,18 @@ #include #include +/* For early boot hypervisor communication in SEV-ES enabled guests */ +static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); + +/* + * Needs to be in the .data section because we need it NULL before bss is + * cleared + */ +static struct ghcb __initdata *boot_ghcb; + +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static inline u64 sev_es_rd_ghcb_msr(void) { return __rdmsr(MSR_AMD64_SEV_ES_GHCB); @@ -161,3 +175,105 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" + +/* + * This function runs on the first #VC exception after the kernel + * switched to virtual addresses. + */ +static bool __init sev_es_setup_ghcb(void) +{ + /* First make sure the hypervisor talks a supported protocol. */ + if (!sev_es_negotiate_protocol()) + return false; + + /* + * Clear the boot_ghcb. The first exception comes in before the bss + * section is cleared. + */ + memset(&boot_ghcb_page, 0, PAGE_SIZE); + + /* Alright - Make the boot-ghcb public */ + boot_ghcb = &boot_ghcb_page; + + return true; +} + +static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) +{ + int trapnr = ctxt->fi.vector; + + if (trapnr == X86_TRAP_PF) + native_write_cr2(ctxt->fi.cr2); + + ctxt->regs->orig_ax = ctxt->fi.error_code; + do_early_exception(ctxt->regs, trapnr); +} + +static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, + struct ghcb *ghcb, + unsigned long exit_code) +{ + enum es_result result; + + switch (exit_code) { + default: + /* + * Unexpected #VC exception + */ + result = ES_UNSUPPORTED; + } + + return result; +} + +bool __init handle_vc_boot_ghcb(struct pt_regs *regs) +{ + unsigned long exit_code = regs->orig_ax; + struct es_em_ctxt ctxt; + enum es_result result; + + /* Do initial setup or terminate the guest */ + if (unlikely(boot_ghcb == NULL && !sev_es_setup_ghcb())) + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + vc_ghcb_invalidate(boot_ghcb); + + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_early_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + BUG(); + } + + return true; + +fail: + show_regs(regs); + + while (true) + halt(); +} diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 615264b2e55e..5aaad4d7adf7 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -5,6 +5,7 @@ #include #include +#include #include #include -- Gitee From 75bf56c0e08d1126cb824d290a3792e63faf2a31 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:42 +0200 Subject: [PATCH 070/113] x86/sev-es: Setup per-CPU GHCBs for the runtime handler commit 885689e47dfa1499b756a07237eb645234d93cf9 upstream. The runtime handler needs one GHCB per-CPU. Set them up and map them unencrypted. [ bp: Touchups and simplification. ] Hygon-SIG: commit 885689e47dfa upstream x86/sev-es: Setup per-CPU GHCBs for the runtime handler Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-42-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/mem_encrypt.h, arch/x86/kernel/sev-es.c, arch/x86/kernel/traps.c] Signed-off-by: hanliyang --- arch/x86/include/asm/mem_encrypt.h | 2 ++ arch/x86/kernel/sev-es.c | 56 +++++++++++++++++++++++++++++- arch/x86/kernel/traps.c | 3 ++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 9d80b43669cd..239e67e0cda4 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -50,6 +50,7 @@ void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, void __init mem_encrypt_init(void); void __init mem_encrypt_free_decrypted_mem(void); +void __init sev_es_init_vc_handling(void); bool sme_active(void); bool sev_active(void); bool sev_es_active(void); @@ -73,6 +74,7 @@ static inline void __init sme_early_init(void) { } static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } static inline void __init sme_enable(struct boot_params *bp) { } +static inline void sev_es_init_vc_handling(void) { } static inline bool sme_active(void) { return false; } static inline bool sev_active(void) { return false; } static inline bool sev_es_active(void) { return false; } diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 2ca0d9478081..438fc4973c05 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -8,8 +8,13 @@ */ #include /* For show_regs() */ -#include +#include +#include #include +#include +#include +#include +#include #include #include @@ -29,6 +34,13 @@ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); */ static struct ghcb __initdata *boot_ghcb; +/* #VC handler runtime per-CPU data */ +struct sev_es_runtime_data { + struct ghcb ghcb_page; +}; + +static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -198,6 +210,48 @@ static bool __init sev_es_setup_ghcb(void) return true; } +static void __init alloc_runtime_data(int cpu) +{ + struct sev_es_runtime_data *data; + + data = memblock_alloc(sizeof(*data), PAGE_SIZE); + if (!data) + panic("Can't allocate SEV-ES runtime data"); + + per_cpu(runtime_data, cpu) = data; +} + +static void __init init_ghcb(int cpu) +{ + struct sev_es_runtime_data *data; + int err; + + data = per_cpu(runtime_data, cpu); + + err = early_set_memory_decrypted((unsigned long)&data->ghcb_page, + sizeof(data->ghcb_page)); + if (err) + panic("Can't map GHCBs unencrypted"); + + memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); +} + +void __init sev_es_init_vc_handling(void) +{ + int cpu; + + BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); + + if (!sev_es_active()) + return; + + /* Initialize per-cpu GHCB pages */ + for_each_possible_cpu(cpu) { + alloc_runtime_data(cpu); + init_ghcb(cpu); + } +} + static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) { int trapnr = ctxt->fi.vector; diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 18648baeccf7..6fafb609716f 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -858,6 +858,9 @@ void __init trap_init(void) /* Init cpu_entry_area before IST entries are set up */ setup_cpu_entry_areas(); + /* Init GHCB memory pages when running as an SEV-ES guest */ + sev_es_init_vc_handling(); + idt_setup_traps(); /* -- Gitee From eff1bd82ca4d66ecb91812774cf8252614440ed7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:43 +0200 Subject: [PATCH 071/113] x86/sev-es: Allocate and map an IST stack for #VC handler commit 02772fb9b68e6a72a5e17f994048df832fe2b15e upstream. Allocate and map an IST stack and an additional fall-back stack for the #VC handler. The memory for the stacks is allocated only when SEV-ES is active. The #VC handler needs to use an IST stack because a #VC exception can be raised from kernel space with unsafe stack, e.g. in the SYSCALL entry path. Since the #VC exception can be nested, the #VC handler switches back to the interrupted stack when entered from kernel space. If switching back is not possible, the fall-back stack is used. Hygon-SIG: commit 02772fb9b68e upstream x86/sev-es: Allocate and map an IST stack for #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-43-joro@8bytes.org [hanliyang: This tree keeps the v5.2 DBn stacks (fd501d4f0399 not backported), so ESTACKS_MEMBERS() gets a separate db2_holesize argument for DB2 while VC/VC2 use optional_stack_size.] [hanliyang: Fix conflict on arch/x86/include/asm/cpu_entry_area.h, arch/x86/include/asm/page_64_types.h, arch/x86/kernel/cpu/common.c, arch/x86/kernel/dumpstack_64.c, arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/include/asm/cpu_entry_area.h | 15 +++++++++--- arch/x86/include/asm/page_64_types.h | 1 + arch/x86/kernel/cpu/common.c | 2 ++ arch/x86/kernel/dumpstack_64.c | 8 +++++-- arch/x86/kernel/sev-es.c | 33 +++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/cpu_entry_area.h b/arch/x86/include/asm/cpu_entry_area.h index 0d1d37d8b279..5c045d762733 100644 --- a/arch/x86/include/asm/cpu_entry_area.h +++ b/arch/x86/include/asm/cpu_entry_area.h @@ -10,7 +10,7 @@ #ifdef CONFIG_X86_64 /* Macro to enforce the same ordering and stack sizes */ -#define ESTACKS_MEMBERS(guardsize, db2_holesize)\ +#define ESTACKS_MEMBERS(guardsize, db2_holesize, optional_stack_size)\ char DF_stack_guard[guardsize]; \ char DF_stack[EXCEPTION_STKSZ]; \ char NMI_stack_guard[guardsize]; \ @@ -23,16 +23,20 @@ char DB_stack[EXCEPTION_STKSZ]; \ char MCE_stack_guard[guardsize]; \ char MCE_stack[EXCEPTION_STKSZ]; \ + char VC_stack_guard[guardsize]; \ + char VC_stack[optional_stack_size]; \ + char VC2_stack_guard[guardsize]; \ + char VC2_stack[optional_stack_size]; \ char IST_top_guard[guardsize]; \ /* The exception stacks' physical storage. No guard pages required */ struct exception_stacks { - ESTACKS_MEMBERS(0, 0) + ESTACKS_MEMBERS(0, 0, 0) }; /* The effective cpu entry area mapping with guard pages. */ struct cea_exception_stacks { - ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ) + ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ, EXCEPTION_STKSZ) }; /* @@ -45,6 +49,8 @@ enum exception_stack_ordering { ESTACK_DB1, ESTACK_DB, ESTACK_MCE, + ESTACK_VC, + ESTACK_VC2, N_EXCEPTION_STACKS }; @@ -141,4 +147,7 @@ static __always_inline struct entry_stack *cpu_entry_stack(int cpu) #define __this_cpu_ist_top_va(name) \ CEA_ESTACK_TOP(__this_cpu_read(cea_exception_stacks), name) +#define __this_cpu_ist_bottom_va(name) \ + CEA_ESTACK_BOT(__this_cpu_read(cea_exception_stacks), name) + #endif diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 9d0b47945272..b661e10fe189 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -28,6 +28,7 @@ #define IST_INDEX_NMI 1 #define IST_INDEX_DB 2 #define IST_INDEX_MCE 3 +#define IST_INDEX_VC 4 /* * Set __PAGE_OFFSET to the most negative possible address + diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 236de8d51553..631f95d1fe8a 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2038,6 +2038,8 @@ void cpu_init(void) t->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI); t->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB); t->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE); + /* Only mapped when SEV-ES is active */ + t->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC); } t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET; diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 87b97897a881..6945f95d5cfd 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -26,11 +26,13 @@ static const char * const exception_stack_names[] = { [ ESTACK_DB1 ] = "#DB1", [ ESTACK_DB ] = "#DB", [ ESTACK_MCE ] = "#MC", + [ ESTACK_VC ] = "#VC", + [ ESTACK_VC2 ] = "#VC2", }; const char *stack_type_name(enum stack_type type) { - BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 8); if (type == STACK_TYPE_IRQ) return "IRQ"; @@ -82,6 +84,8 @@ struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = { EPAGERANGE(DB1), EPAGERANGE(DB), EPAGERANGE(MCE), + EPAGERANGE(VC), + EPAGERANGE(VC2), }; static bool in_exception_stack(unsigned long *stack, struct stack_info *info) @@ -91,7 +95,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info) struct pt_regs *regs; unsigned int k; - BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 8); begin = (unsigned long)__this_cpu_read(cea_exception_stacks); /* diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 438fc4973c05..bbdf7414c0a7 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -37,10 +38,41 @@ static struct ghcb __initdata *boot_ghcb; /* #VC handler runtime per-CPU data */ struct sev_es_runtime_data { struct ghcb ghcb_page; + + /* Physical storage for the per-CPU IST stack of the #VC handler */ + char ist_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); + + /* + * Physical storage for the per-CPU fall-back stack of the #VC handler. + * The fall-back stack is used when it is not safe to switch back to the + * interrupted stack in the #VC entry code. + */ + char fallback_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +static void __init setup_vc_stacks(int cpu) +{ + struct sev_es_runtime_data *data; + struct cpu_entry_area *cea; + unsigned long vaddr; + phys_addr_t pa; + + data = per_cpu(runtime_data, cpu); + cea = get_cpu_entry_area(cpu); + + /* Map #VC IST stack */ + vaddr = CEA_ESTACK_BOT(&cea->estacks, VC); + pa = __pa(data->ist_stack); + cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); + + /* Map VC fall-back stack */ + vaddr = CEA_ESTACK_BOT(&cea->estacks, VC2); + pa = __pa(data->fallback_stack); + cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); +} + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -249,6 +281,7 @@ void __init sev_es_init_vc_handling(void) for_each_possible_cpu(cpu) { alloc_runtime_data(cpu); init_ghcb(cpu); + setup_vc_stacks(cpu); } } -- Gitee From 532313095fc05195c5d20dc97ce8edf74d36ecdc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:44 +0200 Subject: [PATCH 072/113] x86/sev-es: Adjust #VC IST Stack on entering NMI handler commit 315562c9af3d583502b35c4b223a08d95ce69864 upstream. When an NMI hits in the #VC handler entry code before it has switched to another stack, any subsequent #VC exception in the NMI code-path will overwrite the interrupted #VC handler's stack. Make sure this doesn't happen by explicitly adjusting the #VC IST entry in the NMI handler for the time it can cause #VC exceptions. [ bp: Touchups, spelling fixes. ] Hygon-SIG: commit 315562c9af3d upstream x86/sev-es: Adjust #VC IST Stack on entering NMI handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-44-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/sev-es.h, arch/x86/kernel/nmi.c, arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/include/asm/sev-es.h | 19 +++++++++++++ arch/x86/kernel/nmi.c | 9 ++++++ arch/x86/kernel/sev-es.c | 53 +++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 9fbeedaa66ee..59176e8c6b81 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -78,4 +78,23 @@ extern void vc_no_ghcb(void); extern void vc_boot_ghcb(void); extern bool handle_vc_boot_ghcb(struct pt_regs *regs); +#ifdef CONFIG_AMD_MEM_ENCRYPT +extern struct static_key_false sev_es_enable_key; +extern void __sev_es_ist_enter(struct pt_regs *regs); +extern void __sev_es_ist_exit(void); +static __always_inline void sev_es_ist_enter(struct pt_regs *regs) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_ist_enter(regs); +} +static __always_inline void sev_es_ist_exit(void) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_ist_exit(); +} +#else +static inline void sev_es_ist_enter(struct pt_regs *regs) { } +static inline void sev_es_ist_exit(void) { } +#endif + #endif diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 5bb001c0c771..3ac2ac1488b5 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -37,6 +37,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -520,6 +521,12 @@ do_nmi(struct pt_regs *regs, long error_code) this_cpu_write(nmi_cr2, read_cr2()); nmi_restart: + /* + * Needs to happen before DR7 is accessed, because the hypervisor can + * intercept DR7 reads/writes, turning those into #VC exceptions. + */ + sev_es_ist_enter(regs); + #ifdef CONFIG_X86_64 /* * If we interrupted a breakpoint, it is possible that @@ -549,6 +556,8 @@ do_nmi(struct pt_regs *regs, long error_code) } #endif + sev_es_ist_exit(); + if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) write_cr2(this_cpu_read(nmi_cr2)); if (this_cpu_dec_return(nmi_state)) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index bbdf7414c0a7..97b6c4d1d3d4 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -51,6 +51,7 @@ struct sev_es_runtime_data { }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +DEFINE_STATIC_KEY_FALSE(sev_es_enable_key); static void __init setup_vc_stacks(int cpu) { @@ -73,6 +74,55 @@ static void __init setup_vc_stacks(int cpu) cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); } +static __always_inline bool on_vc_stack(unsigned long sp) +{ + return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); +} + +/* + * This function handles the case when an NMI is raised in the #VC exception + * handler entry code. In this case, the IST entry for #VC must be adjusted, so + * that any subsequent #VC exception will not overwrite the stack contents of the + * interrupted #VC handler. + * + * The IST entry is adjusted unconditionally so that it can be also be + * unconditionally adjusted back in sev_es_ist_exit(). Otherwise a nested + * sev_es_ist_exit() call may adjust back the IST entry too early. + */ +void noinstr __sev_es_ist_enter(struct pt_regs *regs) +{ + unsigned long old_ist, new_ist; + + /* Read old IST entry */ + old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + /* Make room on the IST stack */ + if (on_vc_stack(regs->sp)) + new_ist = ALIGN_DOWN(regs->sp, 8) - sizeof(old_ist); + else + new_ist = old_ist - sizeof(old_ist); + + /* Store old IST entry */ + *(unsigned long *)new_ist = old_ist; + + /* Set new IST entry */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); +} + +void noinstr __sev_es_ist_exit(void) +{ + unsigned long ist; + + /* Read IST entry */ + ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) + return; + + /* Read back old IST entry and write it to the TSS */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); +} + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -277,6 +327,9 @@ void __init sev_es_init_vc_handling(void) if (!sev_es_active()) return; + /* Enable SEV-ES special handling */ + static_branch_enable(&sev_es_enable_key); + /* Initialize per-cpu GHCB pages */ for_each_possible_cpu(cpu) { alloc_runtime_data(cpu); -- Gitee From a7a4ba7a44b76d8fa54ddfc0b6cc8b2f15936d8f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:45 +0200 Subject: [PATCH 073/113] x86/dumpstack/64: Add noinstr version of get_stack_info() commit 6b27edd74a5e9669120f7bd0ae1f475d124c1042 upstream. The get_stack_info() functionality is needed in the entry code for the #VC exception handler. Provide a version of it in the .text.noinstr section which can be called safely from there. Hygon-SIG: commit 6b27edd74a5e upstream x86/dumpstack/64: Add noinstr version of get_stack_info() Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-45-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/stacktrace.h, arch/x86/kernel/dumpstack.c, arch/x86/kernel/dumpstack_64.c, arch/x86/mm/cpu_entry_area.c] Signed-off-by: hanliyang --- arch/x86/include/asm/stacktrace.h | 2 ++ arch/x86/kernel/dumpstack.c | 7 +++--- arch/x86/kernel/dumpstack_64.c | 38 ++++++++++++++++++------------- arch/x86/mm/cpu_entry_area.c | 3 ++- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 14db05086bbf..506873a8d5a4 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -35,6 +35,8 @@ bool in_entry_stack(unsigned long *stack, struct stack_info *info); int get_stack_info(unsigned long *stack, struct task_struct *task, struct stack_info *info, unsigned long *visit_mask); +bool get_stack_info_noinstr(unsigned long *stack, struct task_struct *task, + struct stack_info *info); const char *stack_type_name(enum stack_type type); diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index e72042dc9487..73650f55bbbf 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -29,8 +29,8 @@ static int die_counter; static struct pt_regs exec_summary_regs; -bool in_task_stack(unsigned long *stack, struct task_struct *task, - struct stack_info *info) +bool noinstr in_task_stack(unsigned long *stack, struct task_struct *task, + struct stack_info *info) { unsigned long *begin = task_stack_page(task); unsigned long *end = task_stack_page(task) + THREAD_SIZE; @@ -46,7 +46,8 @@ bool in_task_stack(unsigned long *stack, struct task_struct *task, return true; } -bool in_entry_stack(unsigned long *stack, struct stack_info *info) +/* Called from get_stack_info_noinstr - so must be noinstr too */ +bool noinstr in_entry_stack(unsigned long *stack, struct stack_info *info) { struct entry_stack *ss = cpu_entry_stack(smp_processor_id()); diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 6945f95d5cfd..b24c20913c08 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -88,7 +88,7 @@ struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = { EPAGERANGE(VC2), }; -static bool in_exception_stack(unsigned long *stack, struct stack_info *info) +static __always_inline bool in_exception_stack(unsigned long *stack, struct stack_info *info) { unsigned long begin, end, stk = (unsigned long)stack; const struct estack_pages *ep; @@ -129,7 +129,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info) return true; } -static bool in_irq_stack(unsigned long *stack, struct stack_info *info) +static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info) { unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long)); @@ -154,32 +154,38 @@ static bool in_irq_stack(unsigned long *stack, struct stack_info *info) return true; } -int get_stack_info(unsigned long *stack, struct task_struct *task, - struct stack_info *info, unsigned long *visit_mask) +bool noinstr get_stack_info_noinstr(unsigned long *stack, struct task_struct *task, + struct stack_info *info) { - if (!stack) - goto unknown; - - task = task ? : current; - if (in_task_stack(stack, task, info)) - goto recursion_check; + return true; if (task != current) - goto unknown; + return false; if (in_exception_stack(stack, info)) - goto recursion_check; + return true; if (in_irq_stack(stack, info)) - goto recursion_check; + return true; if (in_entry_stack(stack, info)) - goto recursion_check; + return true; + + return false; +} + +int get_stack_info(unsigned long *stack, struct task_struct *task, + struct stack_info *info, unsigned long *visit_mask) +{ + task = task ? : current; - goto unknown; + if (!stack) + goto unknown; + + if (!get_stack_info_noinstr(stack, task, info)) + goto unknown; -recursion_check: /* * Make sure we don't iterate through any given stack more than once. * If it comes up a second time then there's something wrong going on: diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c index d9643647a9ce..e724f173a1bd 100644 --- a/arch/x86/mm/cpu_entry_area.c +++ b/arch/x86/mm/cpu_entry_area.c @@ -17,7 +17,8 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks); DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks); #endif -struct cpu_entry_area *get_cpu_entry_area(int cpu) +/* Is called from entry code, so must be noinstr */ +noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu) { unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE; BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0); -- Gitee From df4e7a6d4434bb016567b313bedd20039bf867cf Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:46 +0200 Subject: [PATCH 074/113] x86/entry/64: Add entry code for #VC handler commit a13644f3a53de4e95a7bce6459f834e832ea44c5 upstream. The #VC handler needs special entry code because: 1. It runs on an IST stack 2. It needs to be able to handle nested #VC exceptions To make this work, the entry code is implemented to pretend it doesn't use an IST stack. When entered from user-mode or early SYSCALL entry path it switches to the task stack. If entered from kernel-mode it tries to switch back to the previous stack in the IRET frame. The stack found in the IRET frame is validated first, and if it is not safe to use it for the #VC handler, the code will switch to a fall-back stack (the #VC2 IST stack). From there, it can cause nested exceptions again. Hygon-SIG: commit a13644f3a53d upstream x86/entry/64: Add entry code for #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-46-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/entry/entry_64.S, arch/x86/include/asm/idtentry.h, arch/x86/include/asm/proto.h, arch/x86/include/asm/traps.h, arch/x86/kernel/traps.c] Signed-off-by: hanliyang --- arch/x86/entry/entry_64.S | 80 +++++++++++++++++++++++++++++++++ arch/x86/include/asm/idtentry.h | 44 ++++++++++++++++++ arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 45 +++++++++++++++++++ 5 files changed, 171 insertions(+) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 1d2385a389fa..c328a897695c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -157,6 +157,8 @@ ENTRY(entry_SYSCALL_64) SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp +SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */ pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ @@ -941,6 +943,84 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt .endm +#ifdef CONFIG_AMD_MEM_ENCRYPT +/** + * idtentry_vc - Macro to generate entry stub for #VC + * @vector: Vector number + * @asmsym: ASM symbol for the entry point + * @cfunc: C function to be called + * + * The macro emits code to set up the kernel context for #VC. The #VC handler + * runs on an IST stack and needs to be able to cause nested #VC exceptions. + * + * To make this work the #VC entry code tries its best to pretend it doesn't use + * an IST stack by switching to the task stack if coming from user-space (which + * includes early SYSCALL entry path) or back to the stack in the IRET frame if + * entered from kernel-mode. + * + * If entered from kernel-mode the return stack is validated first, and if it is + * not safe to use (e.g. because it points to the entry stack) the #VC handler + * will switch to a fall-back stack (VC2) and call a special handler function. + * + * The macro is only used for one vector, but it is planned to be extended in + * the future for the #HV exception. + */ +.macro idtentry_vc vector asmsym cfunc +SYM_CODE_START(\asmsym) + UNWIND_HINT_IRET_REGS + ASM_CLAC + + /* + * If the entry is from userspace, switch stacks and treat it as + * a normal entry. + */ + testb $3, CS-ORIG_RAX(%rsp) + jnz .Lfrom_usermode_switch_stack_\@ + + /* + * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. + * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS + */ + call paranoid_entry + + UNWIND_HINT_REGS + + /* + * Switch off the IST stack to make it free for nested exceptions. The + * vc_switch_off_ist() function will switch back to the interrupted + * stack if it is safe to do so. If not it switches to the VC fall-back + * stack. + */ + movq %rsp, %rdi /* pt_regs pointer */ + call vc_switch_off_ist + movq %rax, %rsp /* Switch to new stack */ + + UNWIND_HINT_REGS + + /* Update pt_regs */ + movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ + movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ + + movq %rsp, %rdi /* pt_regs pointer */ + + call \cfunc + + /* + * No need to switch back to the IST stack. The current stack is either + * identical to the stack in the IRET frame or the VC fall-back stack, + * so it is definitly mapped even with PTI enabled. + */ + jmp paranoid_exit + + /* Switch to the regular task stack */ +.Lfrom_usermode_switch_stack_\@: + idtentry_part safe_stack_\cfunc, has_error_code=1, read_cr2=0, paranoid=0 + +_ASM_NOKPROBE(\asmsym) +SYM_CODE_END(\asmsym) +.endm +#endif + /** * idtentry - Generate an IDT entry stub * @sym: Name of the generated entry point diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 55550afb852f..9c050345b6ca 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -170,6 +170,18 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) DECLARE_IDTENTRY_RAW(vector, func); \ __visible void noist_##func(struct pt_regs *regs) +/** + * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the + * safe_stack C handler. + */ +#define DECLARE_IDTENTRY_VC(vector, func) \ + DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ + __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code) + /** * DEFINE_IDTENTRY_IST - Emit code for IST entry points * @func: Function name of the entry point @@ -209,6 +221,35 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) #define DEFINE_IDTENTRY_DF(func) \ DEFINE_IDTENTRY_RAW_ERRORCODE(func) +/** + * DEFINE_IDTENTRY_VC_SAFE_STACK - Emit code for VMM communication handler + which runs on a safe stack. + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC_SAFE_STACK(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(safe_stack_##func) + +/** + * DEFINE_IDTENTRY_VC_IST - Emit code for VMM communication handler + which runs on the VC fall-back stack + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC_IST(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(ist_##func) + +/** + * DEFINE_IDTENTRY_VC - Emit code for VMM communication handler + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(func) + #else /* CONFIG_X86_64 */ /* Maps to a regular IDTENTRY on 32bit for now */ @@ -283,6 +324,9 @@ __visible noinstr void func(struct pt_regs *regs, \ # define DECLARE_IDTENTRY_DF(vector, func) \ idtentry_df vector asm_##func func +# define DECLARE_IDTENTRY_VC(vector, func) \ + idtentry_vc vector asm_##func func + #else # define DECLARE_IDTENTRY_MCE(vector, func) \ DECLARE_IDTENTRY(vector, func) diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h index 0eaca7a130c9..a27dcb4f4bcc 100644 --- a/arch/x86/include/asm/proto.h +++ b/arch/x86/include/asm/proto.h @@ -12,6 +12,7 @@ void syscall_init(void); #ifdef CONFIG_X86_64 void entry_SYSCALL_64(void); +void entry_SYSCALL_64_safe_stack(void); long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2); #endif diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index df9eea3ec672..63cfa2adffbd 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -81,6 +81,7 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); asmlinkage __visible notrace struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); void __init trap_init(void); +asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs); #endif dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code); dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 6fafb609716f..6aa394c98844 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -556,6 +557,50 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs) } NOKPROBE_SYMBOL(sync_regs); +#ifdef CONFIG_AMD_MEM_ENCRYPT +asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs) +{ + unsigned long sp, *stack; + struct stack_info info; + struct pt_regs *regs_ret; + + /* + * In the SYSCALL entry path the RSP value comes from user-space - don't + * trust it and switch to the current kernel stack + */ + if (regs->ip >= (unsigned long)entry_SYSCALL_64 && + regs->ip < (unsigned long)entry_SYSCALL_64_safe_stack) { + sp = this_cpu_read(cpu_current_top_of_stack); + goto sync; + } + + /* + * From here on the RSP value is trusted. Now check whether entry + * happened from a safe stack. Not safe are the entry or unknown stacks, + * use the fall-back stack instead in this case. + */ + sp = regs->sp; + stack = (unsigned long *)sp; + + if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY || + info.type >= STACK_TYPE_EXCEPTION_LAST) + sp = __this_cpu_ist_top_va(VC2); + +sync: + /* + * Found a safe stack - switch to it as if the entry didn't happen via + * IST stack. The code below only copies pt_regs, the real switch happens + * in assembly code. + */ + sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret); + + regs_ret = (struct pt_regs *)sp; + *regs_ret = *regs; + + return regs_ret; +} +#endif + struct bad_iret_stack { void *error_entry_ret; struct pt_regs regs; -- Gitee From 14347feab6ac72522b4a38344acfa0f2b36f2e70 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:47 +0200 Subject: [PATCH 075/113] x86/sev-es: Add a Runtime #VC Exception Handler commit 0786138c78e79343c7b015d77507cbf9d5f15d00 upstream. Add the handlers for #VC exceptions invoked at runtime. Hygon-SIG: commit 0786138c78e7 upstream x86/sev-es: Add a Runtime #VC Exception Handler Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-47-joro@8bytes.org [hanliyang: Keep the #VC entry point declaration in the shared section of arch/x86/include/asm/idtentry.h as upstream does, so that C and ASM use the same one, and move the asm/idtentry.h include in arch/x86/entry/entry_64.S behind the idtentry macro definitions which the ASM variant of that declaration expands to.] [hanliyang: Fix conflict on arch/x86/entry/entry_64.S, arch/x86/include/asm/idtentry.h, arch/x86/kernel/idt.c, arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/entry/entry_64.S | 11 +- arch/x86/include/asm/idtentry.h | 6 + arch/x86/kernel/idt.c | 3 + arch/x86/kernel/sev-es.c | 246 +++++++++++++++++++++++++++++++- 4 files changed, 256 insertions(+), 10 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index c328a897695c..3ef08d52519b 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -500,12 +500,6 @@ END(spurious_entries_start) decl PER_CPU_VAR(irq_count) .endm -/* - * Include the defines which emit the idt entries which are shared - * shared between 32 and 64 bit. - */ -#include - /* * Interrupt entry helper function. * @@ -1113,6 +1107,11 @@ _ASM_NOKPROBE(\sym) END(\sym) .endm +/* + * Include the defines which emit the idt entries which are shared + * shared between 32 and 64 bit. + */ +#include idtentry divide_error do_divide_error has_error_code=0 idtentry overflow do_overflow has_error_code=0 diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 9c050345b6ca..acf7eee45e91 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -180,6 +180,7 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) */ #define DECLARE_IDTENTRY_VC(vector, func) \ DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ + __visible noinstr void ist_##func(struct pt_regs *regs, unsigned long error_code); \ __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code) /** @@ -344,4 +345,9 @@ __visible noinstr void func(struct pt_regs *regs, \ #endif /* __ASSEMBLY__ */ +/* #VC */ +#ifdef CONFIG_AMD_MEM_ENCRYPT +DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication); +#endif + #endif diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 1520b28b82e2..7b96a1c8da56 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -181,6 +181,9 @@ static const __initconst struct idt_data ist_idts[] = { #ifdef CONFIG_X86_MCE ISTG(X86_TRAP_MC, &machine_check, IST_INDEX_MCE), #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT + ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC), +#endif }; /* diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 97b6c4d1d3d4..8c9334a7b084 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -7,9 +7,12 @@ * Author: Joerg Roedel */ +#define pr_fmt(fmt) "SEV-ES: " fmt + #include /* For show_regs() */ #include #include +#include #include #include #include @@ -22,8 +25,8 @@ #include #include #include -#include -#include +#include +#include #include /* For early boot hypervisor communication in SEV-ES enabled guests */ @@ -48,11 +51,43 @@ struct sev_es_runtime_data { * interrupted stack in the #VC entry code. */ char fallback_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); + + /* + * Reserve one page per CPU as backup storage for the unencrypted GHCB. + * It is needed when an NMI happens while the #VC handler uses the real + * GHCB, and the NMI handler itself is causing another #VC exception. In + * that case the GHCB content of the first handler needs to be backed up + * and restored. + */ + struct ghcb backup_ghcb; + + /* + * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. + * There is no need for it to be atomic, because nothing is written to + * the GHCB between the read and the write of ghcb_active. So it is safe + * to use it when a nested #VC exception happens before the write. + * + * This is necessary for example in the #VC->NMI->#VC case when the NMI + * happens while the first #VC handler uses the GHCB. When the NMI code + * raises a second #VC handler it might overwrite the contents of the + * GHCB written by the first handler. To avoid this the content of the + * GHCB is saved and restored when the GHCB is detected to be in use + * already. + */ + bool ghcb_active; + bool backup_ghcb_active; +}; + +struct ghcb_state { + struct ghcb *ghcb; }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); DEFINE_STATIC_KEY_FALSE(sev_es_enable_key); +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static void __init setup_vc_stacks(int cpu) { struct sev_es_runtime_data *data; @@ -123,8 +158,52 @@ void noinstr __sev_es_ist_exit(void) this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); } -/* Needed in vc_early_forward_exception */ -void do_early_exception(struct pt_regs *regs, int trapnr); +static __always_inline struct ghcb *sev_es_get_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (unlikely(data->ghcb_active)) { + /* GHCB is already in use - save its contents */ + + if (unlikely(data->backup_ghcb_active)) + return NULL; + + /* Mark backup_ghcb active before writing to it */ + data->backup_ghcb_active = true; + + state->ghcb = &data->backup_ghcb; + + /* Backup GHCB content */ + *state->ghcb = *ghcb; + } else { + state->ghcb = NULL; + data->ghcb_active = true; + } + + return ghcb; +} + +static __always_inline void sev_es_put_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (state->ghcb) { + /* Restore GHCB from Backup */ + *ghcb = *state->ghcb; + data->backup_ghcb_active = false; + state->ghcb = NULL; + } else { + data->ghcb_active = false; + } +} static inline u64 sev_es_rd_ghcb_msr(void) { @@ -316,6 +395,9 @@ static void __init init_ghcb(int cpu) panic("Can't map GHCBs unencrypted"); memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); + + data->ghcb_active = false; + data->backup_ghcb_active = false; } void __init sev_es_init_vc_handling(void) @@ -336,6 +418,9 @@ void __init sev_es_init_vc_handling(void) init_ghcb(cpu); setup_vc_stacks(cpu); } + + /* Secondary CPUs use the runtime #VC handler */ + initial_vc_handler = (unsigned long)safe_stack_exc_vmm_communication; } static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) @@ -366,6 +451,159 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, return result; } +static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) +{ + long error_code = ctxt->fi.error_code; + int trapnr = ctxt->fi.vector; + + ctxt->regs->orig_ax = ctxt->fi.error_code; + + switch (trapnr) { + case X86_TRAP_GP: + do_general_protection(ctxt->regs, error_code); + break; + case X86_TRAP_UD: + do_invalid_op(ctxt->regs, error_code); + break; + default: + pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); + BUG(); + } +} + +static __always_inline bool on_vc_fallback_stack(struct pt_regs *regs) +{ + unsigned long sp = (unsigned long)regs; + + return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); +} + +/* + * Main #VC exception handler. It is called when the entry code was able to + * switch off the IST to a safe kernel stack. + * + * With the current implementation it is always possible to switch to a safe + * stack because #VC exceptions only happen at known places, like intercepted + * instructions or accesses to MMIO areas/IO ports. They can also happen with + * code instrumentation when the hypervisor intercepts #DB, but the critical + * paths are forbidden to be instrumented, so #DB exceptions currently also + * only happen in safe places. + */ +DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + struct ghcb_state state; + struct es_em_ctxt ctxt; + enum es_result result; + struct ghcb *ghcb; + + lockdep_assert_irqs_disabled(); + instrumentation_begin(); + + /* + * This is invoked through an interrupt gate, so IRQs are disabled. The + * code below might walk page-tables for user or kernel addresses, so + * keep the IRQs disabled to protect us against concurrent TLB flushes. + */ + + ghcb = sev_es_get_ghcb(&state); + if (!ghcb) { + /* + * Mark GHCBs inactive so that panic() is able to print the + * message. + */ + data->ghcb_active = false; + data->backup_ghcb_active = false; + + panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); + } + + vc_ghcb_invalidate(ghcb); + result = vc_init_em_ctxt(&ctxt, regs, error_code); + + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, ghcb, error_code); + + sev_es_put_ghcb(&state); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + pr_err_ratelimited("Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + pr_emerg("Unknown result in %s():%d\n", __func__, result); + /* + * Emulating the instruction which caused the #VC exception + * failed - can't continue so print debug information + */ + BUG(); + } + +out: + instrumentation_end(); + + return; + +fail: + if (user_mode(regs)) { + /* + * Do not kill the machine if user-space triggered the + * exception. Send SIGBUS instead and let user-space deal with + * it. + */ + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); + } else { + pr_emerg("PANIC: Unhandled #VC exception in kernel space (result=%d)\n", + result); + + /* Show some debug info */ + show_regs(regs); + + /* Ask hypervisor to sev_es_terminate */ + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + /* If that fails and we get here - just panic */ + panic("Returned from Terminate-Request to Hypervisor\n"); + } + + goto out; +} + +/* This handler runs on the #VC fall-back stack. It can cause further #VC exceptions */ +DEFINE_IDTENTRY_VC_IST(exc_vmm_communication) +{ + instrumentation_begin(); + panic("Can't handle #VC exception from unsupported context\n"); + instrumentation_end(); +} + +DEFINE_IDTENTRY_VC(exc_vmm_communication) +{ + if (likely(!on_vc_fallback_stack(regs))) + safe_stack_exc_vmm_communication(regs, error_code); + else + ist_exc_vmm_communication(regs, error_code); +} + bool __init handle_vc_boot_ghcb(struct pt_regs *regs) { unsigned long exit_code = regs->orig_ax; -- Gitee From 25c08d0f9474075aa61fed1b615738f8c5c4e113 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:48 +0200 Subject: [PATCH 076/113] x86/sev-es: Wire up existing #VC exit-code handlers commit d3529bb73f76d0ec8aafaca505226fa0971c1dc9 upstream. Re-use the handlers for CPUID- and IOIO-caused #VC exceptions in the early boot handler. Hygon-SIG: commit d3529bb73f76 upstream x86/sev-es: Wire up existing #VC exit-code handlers Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-48-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es-shared.c | 7 +++---- arch/x86/kernel/sev-es.c | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 53e3de0e8060..491b557bdfba 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -325,8 +325,7 @@ static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) return ES_OK; } -static enum es_result __maybe_unused -vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u64 exit_info_1, exit_info_2; @@ -434,8 +433,8 @@ vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } -static enum es_result __maybe_unused vc_handle_cpuid(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) +static enum es_result vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u32 cr4 = native_read_cr4(); diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 8c9334a7b084..8f3f6302faed 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -441,6 +441,12 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, enum es_result result; switch (exit_code) { + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(ghcb, ctxt); + break; + case SVM_EXIT_IOIO: + result = vc_handle_ioio(ghcb, ctxt); + break; default: /* * Unexpected #VC exception -- Gitee From 25c4b92fe0d99f4af5e789c7d7381d91069d65a6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:49 +0200 Subject: [PATCH 077/113] x86/sev-es: Handle instruction fetches from user-space commit 5e3427a7bc432ed2e5de394ac30f160cc6c37a1f upstream. When a #VC exception is triggered by user-space, the instruction decoder needs to read the instruction bytes from user addresses. Enhance vc_decode_insn() to safely fetch kernel and user instructions. Hygon-SIG: commit 5e3427a7bc43 upstream x86/sev-es: Handle instruction fetches from user-space Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-49-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 8f3f6302faed..0ff328d8bc1b 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -232,17 +232,30 @@ static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) enum es_result ret; int res; - res = vc_fetch_insn_kernel(ctxt, buffer); - if (unlikely(res == -EFAULT)) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = 0; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; + if (user_mode(ctxt->regs)) { + res = insn_fetch_from_user(ctxt->regs, buffer); + if (!res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + if (!insn_decode(&ctxt->insn, ctxt->regs, buffer, res)) + return ES_DECODE_FAILED; + } else { + res = vc_fetch_insn_kernel(ctxt, buffer); + if (res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); + insn_get_length(&ctxt->insn); } - insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); - insn_get_length(&ctxt->insn); - ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; return ret; -- Gitee From eebf6d79498ea3b991d1db17d3235288ab220ffc Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:50 +0200 Subject: [PATCH 078/113] x86/sev-es: Handle MMIO events commit 51ee7d6e3d2b70a3f232cceffab5084a2abd6719 upstream. Add a handler for #VC exceptions caused by MMIO intercepts. These intercepts come along as nested page faults on pages with reserved bits set. Hygon-SIG: commit 51ee7d6e3d2b upstream x86/sev-es: Handle MMIO events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-50-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/uapi/asm/svm.h | 5 + arch/x86/kernel/sev-es.c | 222 ++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index c68d1618c9b0..8f36ae021a7f 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -81,6 +81,11 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 +/* SEV-ES software-defined VMGEXIT events */ +#define SVM_VMGEXIT_MMIO_READ 0x80000001 +#define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff + #define SVM_EXIT_ERR -1 #define SVM_EXIT_REASONS \ diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 0ff328d8bc1b..1e598b408b7f 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -359,6 +359,37 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, return ES_EXCEPTION; } +static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned long vaddr, phys_addr_t *paddr) +{ + unsigned long va = (unsigned long)vaddr; + unsigned int level; + phys_addr_t pa; + pgd_t *pgd; + pte_t *pte; + + pgd = __va(read_cr3_pa()); + pgd = &pgd[pgd_index(va)]; + pte = lookup_address_in_pgd(pgd, va, &level); + if (!pte) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.cr2 = vaddr; + ctxt->fi.error_code = 0; + + if (user_mode(ctxt->regs)) + ctxt->fi.error_code |= X86_PF_USER; + + return false; + } + + pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; + pa |= va & ~page_level_mask(level); + + *paddr = pa; + + return true; +} + /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" @@ -447,6 +478,194 @@ static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) do_early_exception(ctxt->regs, trapnr); } +static long *vc_insn_get_reg(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_reg_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} + +static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned int bytes, bool read) +{ + u64 exit_code, exit_info_1, exit_info_2; + unsigned long ghcb_pa = __pa(ghcb); + phys_addr_t paddr; + void __user *ref; + + ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); + if (ref == (void __user *)-1L) + return ES_UNSUPPORTED; + + exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; + + if (!vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr)) { + if (!read) + ctxt->fi.error_code |= X86_PF_WRITE; + + return ES_EXCEPTION; + } + + exit_info_1 = paddr; + /* Can never be greater than 8 */ + exit_info_2 = bytes; + + ghcb->save.sw_scratch = ghcb_pa + offsetof(struct ghcb, shared_buffer); + + return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); +} + +static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + unsigned int bytes = 0; + enum es_result ret; + int sign_byte; + long *reg_data; + + switch (insn->opcode.bytes[1]) { + /* MMIO Read w/ zero-extension */ + case 0xb6: + bytes = 1; + fallthrough; + case 0xb7: + if (!bytes) + bytes = 2; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero extend based on operand size */ + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + memset(reg_data, 0, insn->opnd_bytes); + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + /* MMIO Read w/ sign-extension */ + case 0xbe: + bytes = 1; + fallthrough; + case 0xbf: + if (!bytes) + bytes = 2; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Sign extend based on operand size */ + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + if (bytes == 1) { + u8 *val = (u8 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + u16 *val = (u16 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + memset(reg_data, sign_byte, insn->opnd_bytes); + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + default: + ret = ES_UNSUPPORTED; + } + + return ret; +} + +static enum es_result vc_handle_mmio(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + unsigned int bytes = 0; + enum es_result ret; + long *reg_data; + + switch (insn->opcode.bytes[0]) { + /* MMIO Write */ + case 0x88: + bytes = 1; + fallthrough; + case 0x89: + if (!bytes) + bytes = insn->opnd_bytes; + + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + memcpy(ghcb->shared_buffer, reg_data, bytes); + + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + + case 0xc6: + bytes = 1; + fallthrough; + case 0xc7: + if (!bytes) + bytes = insn->opnd_bytes; + + memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); + + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + + /* MMIO Read */ + case 0x8a: + bytes = 1; + fallthrough; + case 0x8b: + if (!bytes) + bytes = insn->opnd_bytes; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + /* Zero-extend for 32-bit operation */ + if (bytes == 4) + *reg_data = 0; + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + /* Two-Byte Opcodes */ + case 0x0f: + ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt); + break; + default: + ret = ES_UNSUPPORTED; + } + + return ret; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -460,6 +679,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_IOIO: result = vc_handle_ioio(ghcb, ctxt); break; + case SVM_EXIT_NPF: + result = vc_handle_mmio(ghcb, ctxt); + break; default: /* * Unexpected #VC exception -- Gitee From d17c1e2ddc918ba00739e7c38e476894d75ab5f3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:51 +0200 Subject: [PATCH 079/113] x86/sev-es: Handle MMIO String Instructions commit 0118b604c2c94c6e34982015cfa7891af4764786 upstream. Add handling for emulation of the MOVS instruction on MMIO regions, as done by the memcpy_toio() and memcpy_fromio() functions. Hygon-SIG: commit 0118b604c2c9 upstream x86/sev-es: Handle MMIO String Instructions Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-51-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 1e598b408b7f..4b74cc542792 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -594,6 +594,73 @@ static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb, return ret; } +/* + * The MOVS instruction has two memory operands, which raises the + * problem that it is not known whether the access to the source or the + * destination caused the #VC exception (and hence whether an MMIO read + * or write operation needs to be emulated). + * + * Instead of playing games with walking page-tables and trying to guess + * whether the source or destination is an MMIO range, split the move + * into two operations, a read and a write with only one memory operand. + * This will cause a nested #VC exception on the MMIO address which can + * then be handled. + * + * This implementation has the benefit that it also supports MOVS where + * source _and_ destination are MMIO regions. + * + * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a + * rare operation. If it turns out to be a performance problem the split + * operations can be moved to memcpy_fromio() and memcpy_toio(). + */ +static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, + unsigned int bytes) +{ + unsigned long ds_base, es_base; + unsigned char *src, *dst; + unsigned char buffer[8]; + enum es_result ret; + bool rep; + int off; + + ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + if (ds_base == -1L || es_base == -1L) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + src = ds_base + (unsigned char *)ctxt->regs->si; + dst = es_base + (unsigned char *)ctxt->regs->di; + + ret = vc_read_mem(ctxt, src, buffer, bytes); + if (ret != ES_OK) + return ret; + + ret = vc_write_mem(ctxt, dst, buffer, bytes); + if (ret != ES_OK) + return ret; + + if (ctxt->regs->flags & X86_EFLAGS_DF) + off = -bytes; + else + off = bytes; + + ctxt->regs->si += off; + ctxt->regs->di += off; + + rep = insn_has_rep_prefix(&ctxt->insn); + if (rep) + ctxt->regs->cx -= 1; + + if (!rep || ctxt->regs->cx == 0) + return ES_OK; + else + return ES_RETRY; +} + static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { @@ -655,6 +722,16 @@ static enum es_result vc_handle_mmio(struct ghcb *ghcb, memcpy(reg_data, ghcb->shared_buffer, bytes); break; + /* MOVS instruction */ + case 0xa4: + bytes = 1; + fallthrough; + case 0xa5: + if (!bytes) + bytes = insn->opnd_bytes; + + ret = vc_handle_mmio_movs(ctxt, bytes); + break; /* Two-Byte Opcodes */ case 0x0f: ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt); -- Gitee From f461894a7dba0c27e5f4176feb800cc709c9338d Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:52 +0200 Subject: [PATCH 080/113] x86/sev-es: Handle MSR events commit a4afa6081c88701635e1e20090f953a25f9444e0 upstream. Implement a handler for #VC exceptions caused by RDMSR/WRMSR instructions. Hygon-SIG: commit a4afa6081c88 upstream x86/sev-es: Handle MSR events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure. ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-52-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4b74cc542792..80d0eefd0602 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -393,6 +393,31 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + enum es_result ret; + u64 exit_info_1; + + /* Is it a WRMSR? */ + exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0; + + ghcb_set_rcx(ghcb, regs->cx); + if (exit_info_1) { + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rdx(ghcb, regs->dx); + } + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0); + + if ((ret == ES_OK) && (!exit_info_1)) { + regs->ax = ghcb->save.rax; + regs->dx = ghcb->save.rdx; + } + + return ret; +} + /* * This function runs on the first #VC exception after the kernel * switched to virtual addresses. @@ -756,6 +781,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_IOIO: result = vc_handle_ioio(ghcb, ctxt); break; + case SVM_EXIT_MSR: + result = vc_handle_msr(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; -- Gitee From 3c42a9da17a61a6c131d37d11db6ae10f6315bc0 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:53 +0200 Subject: [PATCH 081/113] x86/sev-es: Handle DR7 read/write events commit 479a7bf5c1f7f14eb37c6d6aeb401562efc1fcad upstream. Add code to handle #VC exceptions on DR7 register reads and writes. This is needed early because show_regs() reads DR7 to print it out. Under SEV-ES, there is currently no support for saving/restoring the DRx registers but software expects to be able to write to the DR7 register. For now, cache the value written to DR7 and return it on read attempts, but do not touch the real hardware DR7. Hygon-SIG: commit 479a7bf5c1f7 upstream x86/sev-es: Handle DR7 read/write events Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to #VC handling framework - Support early usage ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-53-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 80d0eefd0602..4cd2943c0822 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -29,6 +29,8 @@ #include #include +#define DR7_RESET_VALUE 0x400 + /* For early boot hypervisor communication in SEV-ES enabled guests */ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); @@ -76,6 +78,13 @@ struct sev_es_runtime_data { */ bool ghcb_active; bool backup_ghcb_active; + + /* + * Cached DR7 value - write it on DR7 writes and return it on reads. + * That value will never make it to the real hardware DR7 as debugging + * is currently unsupported in SEV-ES guests. + */ + unsigned long dr7; }; struct ghcb_state { @@ -519,6 +528,21 @@ static long *vc_insn_get_reg(struct es_em_ctxt *ctxt) return reg_array + offset; } +static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, unsigned int bytes, bool read) { @@ -768,6 +792,61 @@ static enum es_result vc_handle_mmio(struct ghcb *ghcb, return ret; } +static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long val, *reg = vc_insn_get_rm(ctxt); + enum es_result ret; + + if (!reg) + return ES_DECODE_FAILED; + + val = *reg; + + /* Upper 32 bits must be written as zeroes */ + if (val >> 32) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + /* Clear out other reserved bits and set bit 10 */ + val = (val & 0xffff23ffL) | BIT(10); + + /* Early non-zero writes to DR7 are not supported */ + if (!data && (val & ~DR7_RESET_VALUE)) + return ES_UNSUPPORTED; + + /* Using a value of 0 for ExitInfo1 means RAX holds the value */ + ghcb_set_rax(ghcb, val); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); + if (ret != ES_OK) + return ret; + + if (data) + data->dr7 = val; + + return ES_OK; +} + +static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long *reg = vc_insn_get_rm(ctxt); + + if (!reg) + return ES_DECODE_FAILED; + + if (data) + *reg = data->dr7; + else + *reg = DR7_RESET_VALUE; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -775,6 +854,12 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, enum es_result result; switch (exit_code) { + case SVM_EXIT_READ_DR7: + result = vc_handle_dr7_read(ghcb, ctxt); + break; + case SVM_EXIT_WRITE_DR7: + result = vc_handle_dr7_write(ghcb, ctxt); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; -- Gitee From dec01911920a62d8dbe0b4f6c224adb314f6145b Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:54 +0200 Subject: [PATCH 082/113] x86/sev-es: Handle WBINVD Events commit a14a92fc4b420ae6713800a8ae8c3686c1a1fa20 upstream. Implement a handler for #VC exceptions caused by WBINVD instructions. Hygon-SIG: commit a14a92fc4b42 upstream x86/sev-es: Handle WBINVD Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-54-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4cd2943c0822..eb1eb3e2d1ba 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -847,6 +847,12 @@ static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -869,6 +875,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MSR: result = vc_handle_msr(ghcb, ctxt); break; + case SVM_EXIT_WBINVD: + result = vc_handle_wbinvd(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; -- Gitee From a5dde8e03301dbb52bf1f939aeb65361542c267c Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:55 +0200 Subject: [PATCH 083/113] x86/sev-es: Handle RDTSC(P) Events commit 4711e7acaa125d8cc242f06e1f4d6c74e177454b upstream. Implement a handler for #VC exceptions caused by RDTSC and RDTSCP instructions. Also make it available in the pre-decompression stage because the KASLR code uses RDTSC/RDTSCP to gather entropy and some hypervisors intercept these instructions. Hygon-SIG: commit 4711e7acaa12 upstream x86/sev-es: Handle RDTSC(P) Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to #VC handling infrastructure - Make it available early ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-55-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/sev-es.c | 4 ++++ arch/x86/kernel/sev-es-shared.c | 23 +++++++++++++++++++++++ arch/x86/kernel/sev-es.c | 4 ++++ 3 files changed, 31 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index b1790f487456..5f15e5864e0c 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -181,6 +181,10 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) goto finish; switch (exit_code) { + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code); + break; case SVM_EXIT_IOIO: result = vc_handle_ioio(boot_ghcb, &ctxt); break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 491b557bdfba..4be8af2f9c57 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -467,3 +467,26 @@ static enum es_result vc_handle_cpuid(struct ghcb *ghcb, return ES_OK; } + +static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + unsigned long exit_code) +{ + bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); + enum es_result ret; + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && + (!rdtscp || ghcb_rcx_is_valid(ghcb)))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + if (rdtscp) + ctxt->regs->cx = ghcb->save.rcx; + + return ES_OK; +} diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index eb1eb3e2d1ba..498dd058407f 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -866,6 +866,10 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WRITE_DR7: result = vc_handle_dr7_write(ghcb, ctxt); break; + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(ghcb, ctxt, exit_code); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; -- Gitee From c1a32b98e54f4a13289af679c7938448021d8031 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:56 +0200 Subject: [PATCH 084/113] x86/sev-es: Handle RDPMC Events commit 5d55cf78a8785a7496919462f3d1a4689f62fb1d upstream. Implement a handler for #VC exceptions caused by RDPMC instructions. Hygon-SIG: commit 5d55cf78a878 upstream x86/sev-es: Handle RDPMC Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-56-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 498dd058407f..14da836e426b 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -853,6 +853,25 @@ static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); } +static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rcx(ghcb, ctxt->regs->cx); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -870,6 +889,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_RDTSCP: result = vc_handle_rdtsc(ghcb, ctxt, exit_code); break; + case SVM_EXIT_RDPMC: + result = vc_handle_rdpmc(ghcb, ctxt); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; -- Gitee From 74766e9e3f70aa6142466859fac9755990dc5b5c Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:57 +0200 Subject: [PATCH 085/113] x86/sev-es: Handle INVD Events commit 8b4ce83707cb2cff5f6d175ea38f751ac4456096 upstream. Implement a handler for #VC exceptions caused by INVD instructions. Since Linux should never use INVD, just mark it as unsupported. Hygon-SIG: commit 8b4ce83707cb upstream x86/sev-es: Handle INVD Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-57-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 14da836e426b..e4a315a9d64f 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -892,6 +892,10 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_RDPMC: result = vc_handle_rdpmc(ghcb, ctxt); break; + case SVM_EXIT_INVD: + pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); + result = ES_UNSUPPORTED; + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; -- Gitee From c3777f186f9712f911edfdb5298effc0dcaef082 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:58 +0200 Subject: [PATCH 086/113] x86/sev-es: Handle MONITOR/MONITORX Events commit 0c2fd2ef64ef1a91d81c2f61309735ac438b68a4 upstream. Implement a handler for #VC exceptions caused by MONITOR and MONITORX instructions. Hygon-SIG: commit 0c2fd2ef64ef upstream x86/sev-es: Handle MONITOR/MONITORX Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-58-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index e4a315a9d64f..8f221828d5e9 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -872,6 +872,16 @@ static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt return ES_OK; } +static enum es_result vc_handle_monitor(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Treat it as a NOP and do not leak a physical address to the + * hypervisor. + */ + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -908,6 +918,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WBINVD: result = vc_handle_wbinvd(ghcb, ctxt); break; + case SVM_EXIT_MONITOR: + result = vc_handle_monitor(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; -- Gitee From 7158cc4961e4d2df9d675489308d839a0e2168f0 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:59 +0200 Subject: [PATCH 087/113] x86/sev-es: Handle MWAIT/MWAITX Events commit ded476bbe203e59cb571d6c576f680efe4a0ddff upstream. Implement a handler for #VC exceptions caused by MWAIT and MWAITX instructions. Hygon-SIG: commit ded476bbe203 upstream x86/sev-es: Handle MWAIT/MWAITX Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-59-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 8f221828d5e9..910d47a510b0 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -882,6 +882,13 @@ static enum es_result vc_handle_monitor(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_mwait(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* Treat the same as MONITOR/MONITORX */ + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -921,6 +928,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MONITOR: result = vc_handle_monitor(ghcb, ctxt); break; + case SVM_EXIT_MWAIT: + result = vc_handle_mwait(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; -- Gitee From e1e12f12100645f1548eef4496505f432f22bfb7 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:00 +0200 Subject: [PATCH 088/113] x86/sev-es: Handle VMMCALL Events commit 2eb7dcf0ccc40ad3f39b000becf16661abf98102 upstream. Implement a handler for #VC exceptions caused by VMMCALL instructions. This is only a starting point, VMMCALL emulation under SEV-ES needs further hypervisor-specific changes to provide additional state. [ bp: Drop "this patch". ] Hygon-SIG: commit 2eb7dcf0ccc4 upstream x86/sev-es: Handle VMMCALL Events Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-60-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 910d47a510b0..0350b8f11b42 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -889,6 +889,26 @@ static enum es_result vc_handle_mwait(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rax(ghcb, ctxt->regs->ax); + ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); + if (ret != ES_OK) + return ret; + + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -922,6 +942,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MSR: result = vc_handle_msr(ghcb, ctxt); break; + case SVM_EXIT_VMMCALL: + result = vc_handle_vmmcall(ghcb, ctxt); + break; case SVM_EXIT_WBINVD: result = vc_handle_wbinvd(ghcb, ctxt); break; -- Gitee From 8a87dcbc0434b79c726671a7e93f33c91d5efb3d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:01 +0200 Subject: [PATCH 089/113] x86/sev-es: Handle #AC Events commit a2d0171a9cf59637411281a929900fde80e6c1cb upstream. Implement a handler for #VC exceptions caused by #AC exceptions. The #AC exception is just forwarded to do_alignment_check() and not pushed down to the hypervisor, as requested by the SEV-ES GHCB Standardization Specification. Hygon-SIG: commit a2d0171a9cf5 upstream x86/sev-es: Handle #AC Events Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-61-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 0350b8f11b42..fe0e5548d4f3 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -909,6 +909,19 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Calling ecx_alignment_check() directly does not work, because it + * enables IRQs and the GHCB is active. Forward the exception and call + * it later from vc_forward_exception(). + */ + ctxt->fi.vector = X86_TRAP_AC; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -922,6 +935,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WRITE_DR7: result = vc_handle_dr7_write(ghcb, ctxt); break; + case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: + result = vc_handle_trap_ac(ghcb, ctxt); + break; case SVM_EXIT_RDTSC: case SVM_EXIT_RDTSCP: result = vc_handle_rdtsc(ghcb, ctxt, exit_code); @@ -981,6 +997,9 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) case X86_TRAP_UD: do_invalid_op(ctxt->regs, error_code); break; + case X86_TRAP_AC: + do_alignment_check(ctxt->regs, error_code); + break; default: pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); BUG(); -- Gitee From 6eea2857e3634b224e915ea50e429df065cebc5a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:02 +0200 Subject: [PATCH 090/113] x86/sev-es: Handle #DB Events commit cb1ad3ecea959593400dfac4f027dbc005e62c39 upstream. Handle #VC exceptions caused by #DB exceptions in the guest. Those must be handled outside of instrumentation_begin()/end() so that the handler will not be raised recursively. Handle them by calling the kernel's debug exception handler. Hygon-SIG: commit cb1ad3ecea95 upstream x86/sev-es: Handle #DB Events Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-62-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index fe0e5548d4f3..a6aea64a7fc5 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -922,6 +922,24 @@ static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, return ES_EXCEPTION; } +static __always_inline void vc_handle_trap_db(struct pt_regs *regs) +{ + /* + * The #VC handled here is caused by a #DB in the guest. Upstream + * cb1ad3ecea95 handles it by calling the kernel's debug exception + * handler - noist_exc_debug() or exc_debug(), both noinstr - because + * those "must be handled outside of instrumentation_begin()/end() so + * that the handler will not be raised recursively". + * + * This tree has no noinstr #DB handler yet, only the monolithic + * do_debug(). It is plain dotraplinkage (i.e. __visible), neither + * noinstr nor notrace, and objtool does not validate noinstr at + * all. So the guarantee upstream relies on - that the #DB handler + * is not instrumented - does not hold here. + */ + do_debug(regs, 0); +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -1033,6 +1051,15 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) struct ghcb *ghcb; lockdep_assert_irqs_disabled(); + + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB) { + vc_handle_trap_db(regs); + return; + } + instrumentation_begin(); /* -- Gitee From 8314a8ddb53dca66f03e118d578b18c80d0b54c1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:03 +0200 Subject: [PATCH 091/113] x86/paravirt: Allow hypervisor-specific VMMCALL handling under SEV-ES commit f6a9f8a45810d2914ea422ff39bfe2e0251c50f2 upstream. Add two new paravirt callbacks to provide hypervisor-specific processor state in the GHCB and to copy state from the hypervisor back to the processor. Hygon-SIG: commit f6a9f8a45810 upstream x86/paravirt: Allow hypervisor-specific VMMCALL handling under SEV-ES Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-63-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/x86_init.h | 16 +++++++++++++++- arch/x86/kernel/sev-es.c | 12 ++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 96d9cd208610..c4790ec279cc 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -4,8 +4,10 @@ #include +struct ghcb; struct mpc_bus; struct mpc_cpu; +struct pt_regs; struct mpc_table; struct cpuinfo_x86; @@ -238,10 +240,22 @@ struct x86_legacy_features { /** * struct x86_hyper_runtime - x86 hypervisor specific runtime callbacks * - * @pin_vcpu: pin current vcpu to specified physical cpu (run rarely) + * @pin_vcpu: pin current vcpu to specified physical + * cpu (run rarely) + * @sev_es_hcall_prepare: Load additional hypervisor-specific + * state into the GHCB when doing a VMMCALL under + * SEV-ES. Called from the #VC exception handler. + * @sev_es_hcall_finish: Copies state from the GHCB back into the + * processor (or pt_regs). Also runs checks on the + * state returned from the hypervisor after a + * VMMCALL under SEV-ES. Needs to return 'false' + * if the checks fail. Called from the #VC + * exception handler. */ struct x86_hyper_runtime { void (*pin_vcpu)(int cpu); + void (*sev_es_hcall_prepare)(struct ghcb *ghcb, struct pt_regs *regs); + bool (*sev_es_hcall_finish)(struct ghcb *ghcb, struct pt_regs *regs); }; /** diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index a6aea64a7fc5..4c9123bf5adb 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -897,6 +897,9 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, ghcb_set_rax(ghcb, ctxt->regs->ax); ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + if (x86_platform.hyper.sev_es_hcall_prepare) + x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); if (ret != ES_OK) return ret; @@ -906,6 +909,15 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, ctxt->regs->ax = ghcb->save.rax; + /* + * Call sev_es_hcall_finish() after regs->ax is already set. + * This allows the hypervisor handler to overwrite it again if + * necessary. + */ + if (x86_platform.hyper.sev_es_hcall_finish && + !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) + return ES_VMM_ERROR; + return ES_OK; } -- Gitee From 48614d1345793d5c5e182abdff3b77fe738bd35f Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:04 +0200 Subject: [PATCH 092/113] x86/kvm: Add KVM-specific VMMCALL handling under SEV-ES commit 99419b251e5427b89dbfae103d8a2f469efaa4b2 upstream. Implement the callbacks to copy the processor state required by KVM to the GHCB. Hygon-SIG: commit 99419b251e54 upstream x86/kvm: Add KVM-specific VMMCALL handling under SEV-ES Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Split out of a larger patch - Adapt to different callback functions ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-64-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/kvm.c] Signed-off-by: hanliyang --- arch/x86/kernel/kvm.c | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6e9a5abaafb4..74869b4be778 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -37,6 +37,8 @@ #include #include #include +#include +#include static int kvmapf = 1; @@ -880,13 +882,34 @@ static void __init kvm_init_platform(void) x86_platform.apic_post_init = kvm_apic_init; } +#if defined(CONFIG_AMD_MEM_ENCRYPT) +static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs) +{ + /* RAX and CPL are already in the GHCB */ + ghcb_set_rbx(ghcb, regs->bx); + ghcb_set_rcx(ghcb, regs->cx); + ghcb_set_rdx(ghcb, regs->dx); + ghcb_set_rsi(ghcb, regs->si); +} + +static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) +{ + /* No checking of the return state needed */ + return true; +} +#endif + const __initconst struct hypervisor_x86 x86_hyper_kvm = { - .name = "KVM", - .detect = kvm_detect, - .type = X86_HYPER_KVM, - .init.guest_late_init = kvm_guest_init, - .init.x2apic_available = kvm_para_available, - .init.init_platform = kvm_init_platform, + .name = "KVM", + .detect = kvm_detect, + .type = X86_HYPER_KVM, + .init.guest_late_init = kvm_guest_init, + .init.x2apic_available = kvm_para_available, + .init.init_platform = kvm_init_platform, +#if defined(CONFIG_AMD_MEM_ENCRYPT) + .runtime.sev_es_hcall_prepare = kvm_sev_es_hcall_prepare, + .runtime.sev_es_hcall_finish = kvm_sev_es_hcall_finish, +#endif }; static __init int activate_jump_labels(void) -- Gitee From 1bcb94c6b0c6429d7cc94b83965537d50dceed92 Mon Sep 17 00:00:00 2001 From: Doug Covelli Date: Mon, 7 Sep 2020 15:16:05 +0200 Subject: [PATCH 093/113] x86/vmware: Add VMware-specific handling for VMMCALL under SEV-ES commit 1a222de8dcfb903d039810b0823570ee0be4e6c6 upstream. Add VMware-specific handling for #VC faults caused by VMMCALL instructions. Hygon-SIG: commit 1a222de8dcfb upstream x86/vmware: Add VMware-specific handling for VMMCALL under SEV-ES Signed-off-by: Doug Covelli Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to different paravirt interface ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-65-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/cpu/vmware.c | 50 ++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 46d732696c1c..32cd52758d2d 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -31,6 +31,7 @@ #include #include #include +#include #undef pr_fmt #define pr_fmt(fmt) "vmware: " fmt @@ -263,10 +264,49 @@ static bool __init vmware_legacy_x2apic_available(void) (eax & (1 << VMWARE_CMD_LEGACY_X2APIC)) != 0; } +#ifdef CONFIG_AMD_MEM_ENCRYPT +static void vmware_sev_es_hcall_prepare(struct ghcb *ghcb, + struct pt_regs *regs) +{ + /* Copy VMWARE specific Hypercall parameters to the GHCB */ + ghcb_set_rip(ghcb, regs->ip); + ghcb_set_rbx(ghcb, regs->bx); + ghcb_set_rcx(ghcb, regs->cx); + ghcb_set_rdx(ghcb, regs->dx); + ghcb_set_rsi(ghcb, regs->si); + ghcb_set_rdi(ghcb, regs->di); + ghcb_set_rbp(ghcb, regs->bp); +} + +static bool vmware_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) +{ + if (!(ghcb_rbx_is_valid(ghcb) && + ghcb_rcx_is_valid(ghcb) && + ghcb_rdx_is_valid(ghcb) && + ghcb_rsi_is_valid(ghcb) && + ghcb_rdi_is_valid(ghcb) && + ghcb_rbp_is_valid(ghcb))) + return false; + + regs->bx = ghcb->save.rbx; + regs->cx = ghcb->save.rcx; + regs->dx = ghcb->save.rdx; + regs->si = ghcb->save.rsi; + regs->di = ghcb->save.rdi; + regs->bp = ghcb->save.rbp; + + return true; +} +#endif + const __initconst struct hypervisor_x86 x86_hyper_vmware = { - .name = "VMware", - .detect = vmware_platform, - .type = X86_HYPER_VMWARE, - .init.init_platform = vmware_platform_setup, - .init.x2apic_available = vmware_legacy_x2apic_available, + .name = "VMware", + .detect = vmware_platform, + .type = X86_HYPER_VMWARE, + .init.init_platform = vmware_platform_setup, + .init.x2apic_available = vmware_legacy_x2apic_available, +#ifdef CONFIG_AMD_MEM_ENCRYPT + .runtime.sev_es_hcall_prepare = vmware_sev_es_hcall_prepare, + .runtime.sev_es_hcall_finish = vmware_sev_es_hcall_finish, +#endif }; -- Gitee From 53e2b5169d54a360f9f6118ef442057e8c62889c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:06 +0200 Subject: [PATCH 094/113] x86/realmode: Add SEV-ES specific trampoline entry point commit bf5ff276448f64f1f9ef9ffc9e231026e3887d3d upstream. The code at the trampoline entry point is executed in real-mode. In real-mode, #VC exceptions can't be handled so anything that might cause such an exception must be avoided. In the standard trampoline entry code this is the WBINVD instruction and the call to verify_cpu(), which are both not needed anyway when running as an SEV-ES guest. Hygon-SIG: commit bf5ff276448f upstream x86/realmode: Add SEV-ES specific trampoline entry point Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-66-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/realmode.h, arch/x86/realmode/rm/header.S, arch/x86/realmode/rm/trampoline_64.S] Signed-off-by: hanliyang --- arch/x86/include/asm/realmode.h | 3 +++ arch/x86/realmode/rm/header.S | 3 +++ arch/x86/realmode/rm/trampoline_64.S | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index a55231c104c7..c07b067f7bab 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -21,6 +21,9 @@ struct real_mode_header { /* SMP trampoline */ u32 trampoline_start; u32 trampoline_header; +#ifdef CONFIG_AMD_MEM_ENCRYPT + u32 sev_es_trampoline_start; +#endif #ifdef CONFIG_X86_64 u32 trampoline_pgd; #endif diff --git a/arch/x86/realmode/rm/header.S b/arch/x86/realmode/rm/header.S index 6363761cc74c..6a7e2b183f62 100644 --- a/arch/x86/realmode/rm/header.S +++ b/arch/x86/realmode/rm/header.S @@ -20,6 +20,9 @@ GLOBAL(real_mode_header) /* SMP trampoline */ .long pa_trampoline_start .long pa_trampoline_header +#ifdef CONFIG_AMD_MEM_ENCRYPT + .long pa_sev_es_trampoline_start +#endif #ifdef CONFIG_X86_64 .long pa_trampoline_pgd; #endif diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S index aee2b45d83b8..ee5e9d2f265b 100644 --- a/arch/x86/realmode/rm/trampoline_64.S +++ b/arch/x86/realmode/rm/trampoline_64.S @@ -56,6 +56,7 @@ ENTRY(trampoline_start) testl %eax, %eax # Check for return code jnz no_longmode +.Lswitch_to_protected: /* * GDT tables in non default location kernel can be beyond 16MB and * lgdt will not be able to load the address as in real mode default @@ -78,6 +79,26 @@ ENTRY(trampoline_start) no_longmode: hlt jmp no_longmode + +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* SEV-ES supports non-zero IP for entry points - no alignment needed */ +SYM_CODE_START(sev_es_trampoline_start) + cli # We should be safe anyway + + LJMPW_RM(1f) +1: + mov %cs, %ax # Code and data in the same place + mov %ax, %ds + mov %ax, %es + mov %ax, %ss + + # Setup stack + movl $rm_stack_end, %esp + + jmp .Lswitch_to_protected +SYM_CODE_END(sev_es_trampoline_start) +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + #include "../kernel/verify_cpu.S" .section ".text32","ax" -- Gitee From 199f2fa05f499f07aa5ee1d999c015cce27b0199 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:07 +0200 Subject: [PATCH 095/113] x86/realmode: Setup AP jump table commit 8940ac9ced8bc1c48c4e28b0784e3234c9d14469 upstream. As part of the GHCB specification, the booting of APs under SEV-ES requires an AP jump table when transitioning from one layer of code to another (e.g. when going from UEFI to the OS). As a result, each layer that parks an AP must provide the physical address of an AP jump table to the next layer via the hypervisor. Upon booting of the kernel, read the AP jump table address from the hypervisor. Under SEV-ES, APs are started using the INIT-SIPI-SIPI sequence. Before issuing the first SIPI request for an AP, the start CS and IP is programmed into the AP jump table. Upon issuing the SIPI request, the AP will awaken and jump to that start CS:IP address. Hygon-SIG: commit 8940ac9ced8b upstream x86/realmode: Setup AP jump table Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapted to different code base - Moved AP table setup from SIPI sending path to real-mode setup code - Fix sparse warnings ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-67-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/sev-es.h, arch/x86/include/uapi/asm/svm.h, arch/x86/kernel/sev-es.c, arch/x86/realmode/init.c] Signed-off-by: hanliyang --- arch/x86/include/asm/sev-es.h | 5 +++ arch/x86/include/uapi/asm/svm.h | 3 ++ arch/x86/kernel/sev-es.c | 69 +++++++++++++++++++++++++++++++++ arch/x86/realmode/init.c | 18 ++++++++- 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 59176e8c6b81..db88e1c3442d 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -73,6 +73,9 @@ static inline u64 lower_bits(u64 val, unsigned int bits) return (val & mask); } +struct real_mode_header; +enum stack_type; + /* Early IDT entry points for #VC handler */ extern void vc_no_ghcb(void); extern void vc_boot_ghcb(void); @@ -92,9 +95,11 @@ static __always_inline void sev_es_ist_exit(void) if (static_branch_unlikely(&sev_es_enable_key)) __sev_es_ist_exit(); } +extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } +static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } #endif #endif diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 8f36ae021a7f..346b8a7155e8 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,9 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 +#define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 +#define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 #define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff #define SVM_EXIT_ERR -1 diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4c9123bf5adb..4ab265122b09 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -214,6 +215,9 @@ static __always_inline void sev_es_put_ghcb(struct ghcb_state *state) } } +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static inline u64 sev_es_rd_ghcb_msr(void) { return __rdmsr(MSR_AMD64_SEV_ES_GHCB); @@ -402,6 +406,71 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +static u64 get_jump_table_addr(void) +{ + struct ghcb_state state; + unsigned long flags; + struct ghcb *ghcb; + u64 ret = 0; + + local_irq_save(flags); + + ghcb = sev_es_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); + ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + if (ghcb_sw_exit_info_1_is_valid(ghcb) && + ghcb_sw_exit_info_2_is_valid(ghcb)) + ret = ghcb->save.sw_exit_info_2; + + sev_es_put_ghcb(&state); + + local_irq_restore(flags); + + return ret; +} + +int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) +{ + u16 startup_cs, startup_ip; + phys_addr_t jump_table_pa; + u64 jump_table_addr; + u16 __iomem *jump_table; + + jump_table_addr = get_jump_table_addr(); + + /* On UP guests there is no jump table so this is not a failure */ + if (!jump_table_addr) + return 0; + + /* Check if AP Jump Table is page-aligned */ + if (jump_table_addr & ~PAGE_MASK) + return -EINVAL; + + jump_table_pa = jump_table_addr & PAGE_MASK; + + startup_cs = (u16)(rmh->trampoline_start >> 4); + startup_ip = (u16)(rmh->sev_es_trampoline_start - + rmh->trampoline_start); + + jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE); + if (!jump_table) + return -EIO; + + writew(startup_ip, &jump_table[0]); + writew(startup_cs, &jump_table[1]); + + iounmap(jump_table); + + return 0; +} + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index fac50ebb122b..43c98ff221bb 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -9,6 +9,7 @@ #include #include #include +#include struct real_mode_header *real_mode_header; u32 *trampoline_cr4_features; @@ -64,6 +65,19 @@ void __init reserve_real_mode(void) crash_reserve_low_1M(); } +static void sme_sev_setup_real_mode(struct trampoline_header *th) +{ +#ifdef CONFIG_AMD_MEM_ENCRYPT + if (sme_active()) + th->flags |= TH_FLAGS_SME_ACTIVE; + + if (sev_es_active()) { + if (sev_es_setup_ap_jump_table(real_mode_header)) + panic("Failed to get/update SEV-ES AP Jump Table"); + } +#endif +} + static void __init setup_real_mode(void) { u16 real_mode_seg; @@ -131,8 +145,6 @@ static void __init setup_real_mode(void) *trampoline_cr4_features = mmu_cr4_features; trampoline_header->flags = 0; - if (sme_active()) - trampoline_header->flags |= TH_FLAGS_SME_ACTIVE; trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd); @@ -147,6 +159,8 @@ static void __init setup_real_mode(void) for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++) trampoline_pgd[i] = init_top_pgt[i].pgd; #endif + + sme_sev_setup_real_mode(trampoline_header); } /* -- Gitee From 910760ac2ce98769e996416a55dfd1ef9ebc46f0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:08 +0200 Subject: [PATCH 096/113] x86/smpboot: Load TSS and getcpu GDT entry before loading IDT commit 520d030852b4c9babfce9a79d8b5320b6b5545e6 upstream. The IDT on 64-bit contains vectors which use paranoid_entry() and/or IST stacks. To make these vectors work, the TSS and the getcpu GDT entry need to be set up before the IDT is loaded. Hygon-SIG: commit 520d030852b4 upstream x86/smpboot: Load TSS and getcpu GDT entry before loading IDT Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-68-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/processor.h, arch/x86/kernel/cpu/common.c, arch/x86/kernel/smpboot.c] Signed-off-by: hanliyang --- arch/x86/include/asm/processor.h | 1 + arch/x86/kernel/cpu/common.c | 40 ++++++++++++++++++++++++++++++++ arch/x86/kernel/smpboot.c | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 258e54676f1a..a7c796fc3cca 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -731,6 +731,7 @@ extern void load_direct_gdt(int); extern void load_fixmap_gdt(int); extern void load_percpu_segment(int); extern void cpu_init(void); +extern void cpu_init_exception_handling(void); extern void cr4_init(void); static inline unsigned long get_debugctlmsr(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 631f95d1fe8a..97be80d4049c 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1975,6 +1975,46 @@ static void setup_getcpu(int cpu) } #endif +/* + * Setup everything needed to handle exceptions from the IDT, including the IST + * exceptions which use paranoid_entry(). + */ +void cpu_init_exception_handling(void) +{ + struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); + int cpu = raw_smp_processor_id(); + int i; + +#ifdef CONFIG_X86_64 + /* paranoid_entry() gets the CPU number from the GDT */ + setup_getcpu(cpu); + + /* IST vectors need TSS to be set up. */ + tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF); + tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI); + tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB); + tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE); + /* Only mapped when SEV-ES is active */ + tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC); +#endif + + tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET; + + /* + * <= is required because the CPU will access up to + * 8 bits beyond the end of the IO permission bitmap. + */ + for (i = 0; i <= IO_BITMAP_LONGS; i++) + tss->io_bitmap[i] = ~0UL; + + set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss); + + load_TR_desc(); + + /* Finally load the IDT */ + load_current_idt(); +} + /* * cpu_init() initializes state that is per-CPU. Some data is already * initialized (naturally) in the bootstrap process, such as the GDT diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 9c939380df88..060cbbbcc9e5 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -230,7 +230,7 @@ static void notrace start_secondary(void *unused) load_cr3(swapper_pg_dir); __flush_tlb_all(); #endif - load_current_idt(); + cpu_init_exception_handling(); cpu_init(); rcu_cpu_starting(raw_smp_processor_id()); x86_cpuinit.early_percpu_clock_init(); -- Gitee From b65c5ec2f1f90d37d184887eb7e8ff224d858a1f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:09 +0200 Subject: [PATCH 097/113] x86/head/64: Don't call verify_cpu() on starting APs commit 3ecacdbd23956a549d93023f86adc87b4a9d6520 upstream. The APs are not ready to handle exceptions when verify_cpu() is called in secondary_startup_64(). Hygon-SIG: commit 3ecacdbd2395 upstream x86/head/64: Don't call verify_cpu() on starting APs Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-69-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/asm/realmode.h | 1 + arch/x86/kernel/head_64.S | 12 ++++++++++++ arch/x86/realmode/init.c | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index c07b067f7bab..4c23d6b6eb4e 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -72,6 +72,7 @@ extern unsigned char startup_32_smp[]; extern unsigned char boot_gdt[]; #else extern unsigned char secondary_startup_64[]; +extern unsigned char secondary_startup_64_no_verify[]; #endif static inline size_t real_mode_size_needed(void) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index d1d2517ec804..aa8999f5cbe0 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -123,6 +123,18 @@ ENTRY(secondary_startup_64) /* Sanitize CPU configuration */ call verify_cpu + /* + * The secondary_startup_64_no_verify entry point is only used by + * SEV-ES guests. In those guests the call to verify_cpu() would cause + * #VC exceptions which can not be handled at this stage of secondary + * CPU bringup. + * + * All non SEV-ES systems, especially Intel systems, need to execute + * verify_cpu() above to make sure NX is enabled. + */ +SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) + UNWIND_HINT_EMPTY + /* * Retrieve the modifier (SME encryption mask if SME is active) to be * added to the initial pgdir entry that will be programmed into CR3. diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 43c98ff221bb..35267f71c277 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -72,6 +72,12 @@ static void sme_sev_setup_real_mode(struct trampoline_header *th) th->flags |= TH_FLAGS_SME_ACTIVE; if (sev_es_active()) { + /* + * Skip the call to verify_cpu() in secondary_startup_64 as it + * will cause #VC exceptions when the AP can't handle them yet. + */ + th->start = (u64) secondary_startup_64_no_verify; + if (sev_es_setup_ap_jump_table(real_mode_header)) panic("Failed to get/update SEV-ES AP Jump Table"); } -- Gitee From ed292a564fea23ede07b9d0d70809a1b32275974 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:10 +0200 Subject: [PATCH 098/113] x86/sev-es: Support CPU offline/online commit 094794f59720d7e877a1eeb372ecedeed6b441ab upstream. Add a play_dead handler when running under SEV-ES. This is needed because the hypervisor can't deliver an SIPI request to restart the AP. Instead, the kernel has to issue a VMGEXIT to halt the VCPU until the hypervisor wakes it up again. Hygon-SIG: commit 094794f59720 upstream x86/sev-es: Support CPU offline/online Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-70-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/sev-es.c | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 346b8a7155e8..c1dcf3e114e4 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,7 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_AP_HLT_LOOP 0x80000004 #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 #define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4ab265122b09..3e377d439770 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #define DR7_RESET_VALUE 0x400 @@ -518,6 +520,65 @@ static bool __init sev_es_setup_ghcb(void) return true; } +#ifdef CONFIG_HOTPLUG_CPU +static void sev_es_ap_hlt_loop(void) +{ + struct ghcb_state state; + struct ghcb *ghcb; + + ghcb = sev_es_get_ghcb(&state); + + while (true) { + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + /* Wakeup signal? */ + if (ghcb_sw_exit_info_2_is_valid(ghcb) && + ghcb->save.sw_exit_info_2) + break; + } + + sev_es_put_ghcb(&state); +} + +/* + * Play_dead handler when running under SEV-ES. This is needed because + * the hypervisor can't deliver an SIPI request to restart the AP. + * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the + * hypervisor wakes it up again. + */ +static void sev_es_play_dead(void) +{ + play_dead_common(); + + /* IRQs now disabled */ + + sev_es_ap_hlt_loop(); + + /* + * If we get here, the VCPU was woken up again. Jump to CPU + * startup code to get it back online. + */ + start_cpu0(); +} +#else /* CONFIG_HOTPLUG_CPU */ +#define sev_es_play_dead native_play_dead +#endif /* CONFIG_HOTPLUG_CPU */ + +#ifdef CONFIG_SMP +static void __init sev_es_setup_play_dead(void) +{ + smp_ops.play_dead = sev_es_play_dead; +} +#else +static inline void sev_es_setup_play_dead(void) { } +#endif + static void __init alloc_runtime_data(int cpu) { struct sev_es_runtime_data *data; @@ -566,6 +627,8 @@ void __init sev_es_init_vc_handling(void) setup_vc_stacks(cpu); } + sev_es_setup_play_dead(); + /* Secondary CPUs use the runtime #VC handler */ initial_vc_handler = (unsigned long)safe_stack_exc_vmm_communication; } -- Gitee From 5b53d74815372df5861bbc16cb172d8f165090e6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:11 +0200 Subject: [PATCH 099/113] x86/sev-es: Handle NMI State commit 4ca68e023b11e4d5908bf9ee326fab01111d77d5 upstream. When running under SEV-ES, the kernel has to tell the hypervisor when to open the NMI window again after an NMI was injected. This is done with an NMI-complete message to the hypervisor. Add code to the kernel's NMI handler to send this message right at the beginning of do_nmi(). This always allows nesting NMIs. [ bp: Mark __sev_es_nmi_complete() noinstr: vmlinux.o: warning: objtool: exc_nmi()+0x17: call to __sev_es_nmi_complete() leaves .noinstr.text section While at it, use __pa_nodebug() for the same reason due to CONFIG_DEBUG_VIRTUAL=y: vmlinux.o: warning: objtool: __sev_es_nmi_complete()+0xd9: call to __phys_addr() leaves .noinstr.text section ] Hygon-SIG: commit 4ca68e023b11 upstream x86/sev-es: Handle NMI State Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-71-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/include/asm/sev-es.h, arch/x86/include/uapi/asm/svm.h, arch/x86/kernel/nmi.c, arch/x86/kernel/sev-es.c] Signed-off-by: hanliyang --- arch/x86/include/asm/sev-es.h | 7 +++++++ arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/nmi.c | 6 ++++++ arch/x86/kernel/sev-es.c | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index db88e1c3442d..e919f09ae33c 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -96,10 +96,17 @@ static __always_inline void sev_es_ist_exit(void) __sev_es_ist_exit(); } extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); +extern void __sev_es_nmi_complete(void); +static __always_inline void sev_es_nmi_complete(void) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_nmi_complete(); +} #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } +static inline void sev_es_nmi_complete(void) { } #endif #endif diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index c1dcf3e114e4..a7a3403645e5 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,7 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_NMI_COMPLETE 0x80000003 #define SVM_VMGEXIT_AP_HLT_LOOP 0x80000004 #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 3ac2ac1488b5..c80a710f465b 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -510,6 +510,12 @@ NOKPROBE_SYMBOL(is_debug_stack); dotraplinkage notrace void do_nmi(struct pt_regs *regs, long error_code) { + /* + * Re-enable NMIs right here when running as an SEV-ES guest. This might + * cause nested NMIs, but those can be handled safely. + */ + sev_es_nmi_complete(); + if (IS_ENABLED(CONFIG_SMP) && cpu_is_offline(smp_processor_id())) return; diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 3e377d439770..41e570730428 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -408,6 +408,24 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +void noinstr __sev_es_nmi_complete(void) +{ + struct ghcb_state state; + struct ghcb *ghcb; + + ghcb = sev_es_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); + VMGEXIT(); + + sev_es_put_ghcb(&state); +} + static u64 get_jump_table_addr(void) { struct ghcb_state state; -- Gitee From d6272be81b69fab371da9d2a856105b62146c2e5 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:12 +0200 Subject: [PATCH 100/113] x86/efi: Add GHCB mappings when SEV-ES is active commit 39336f4ffb2478ad384075cf4ba7ef2e5db2bbd7 upstream. Calling down to EFI runtime services can result in the firmware performing VMGEXIT calls. The firmware is likely to use the GHCB of the OS (e.g., for setting EFI variables), so each GHCB in the system needs to be identity-mapped in the EFI page tables, as unencrypted, to avoid page faults. Hygon-SIG: commit 39336f4ffb24 upstream x86/efi: Add GHCB mappings when SEV-ES is active Signed-off-by: Tom Lendacky [ jroedel@suse.de: Moved GHCB mapping loop to sev-es.c ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Ard Biesheuvel Link: https://lkml.kernel.org/r/20200907131613.12703-72-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/sev-es.c | 1 + arch/x86/include/asm/sev-es.h | 2 ++ arch/x86/kernel/sev-es.c | 30 ++++++++++++++++++++++++++++++ arch/x86/platform/efi/efi_64.c | 10 ++++++++++ 4 files changed, 43 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 5f15e5864e0c..2a6c7c375244 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -12,6 +12,7 @@ */ #include "misc.h" +#include #include #include #include diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index e919f09ae33c..cf1d957c7091 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -102,11 +102,13 @@ static __always_inline void sev_es_nmi_complete(void) if (static_branch_unlikely(&sev_es_enable_key)) __sev_es_nmi_complete(); } +extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } static inline void sev_es_nmi_complete(void) { } +static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; } #endif #endif diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 41e570730428..db0b5cf2b2a4 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -491,6 +491,36 @@ int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) return 0; } +/* + * This is needed by the OVMF UEFI firmware which will use whatever it finds in + * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu + * runtime GHCBs used by the kernel are also mapped in the EFI page-table. + */ +int __init sev_es_efi_map_ghcbs(pgd_t *pgd) +{ + struct sev_es_runtime_data *data; + unsigned long address, pflags; + int cpu; + u64 pfn; + + if (!sev_es_active()) + return 0; + + pflags = _PAGE_NX | _PAGE_RW; + + for_each_possible_cpu(cpu) { + data = per_cpu(runtime_data, cpu); + + address = __pa(&data->ghcb_page); + pfn = address >> PAGE_SHIFT; + + if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags)) + return 1; + } + + return 0; +} + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 528397818110..b1b94e5dedff 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -48,6 +48,7 @@ #include #include #include +#include /* * We allocate runtime services regions top-down, starting from -4G, i.e. @@ -381,6 +382,15 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) return 1; } + /* + * When SEV-ES is active, the GHCB as set by the kernel will be used + * by firmware. Create a 1:1 unencrypted mapping for each GHCB. + */ + if (sev_es_efi_map_ghcbs(pgd)) { + pr_err("Failed to create 1:1 mapping for the GHCBs!\n"); + return 1; + } + /* * When making calls to the firmware everything needs to be 1:1 * mapped and addressable with 32-bit pointers. Map the kernel -- Gitee From c51090821f01b0109300cf77d9db18eb2ffa1fd2 Mon Sep 17 00:00:00 2001 From: Martin Radev Date: Mon, 7 Sep 2020 15:16:13 +0200 Subject: [PATCH 101/113] x86/sev-es: Check required CPU features for SEV-ES commit f5ed777586e08e09c4b6f1e87161a145ee1431cf upstream. Make sure the machine supports RDRAND, otherwise there is no trusted source of randomness in the system. To also check this in the pre-decompression stage, make has_cpuflag() not depend on CONFIG_RANDOMIZE_BASE anymore. Hygon-SIG: commit f5ed777586e0 upstream x86/sev-es: Check required CPU features for SEV-ES Signed-off-by: Martin Radev Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-73-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/cpuflags.c | 4 ---- arch/x86/boot/compressed/misc.h | 5 +++-- arch/x86/boot/compressed/sev-es.c | 3 +++ arch/x86/kernel/sev-es-shared.c | 15 +++++++++++++++ arch/x86/kernel/sev-es.c | 3 +++ 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/arch/x86/boot/compressed/cpuflags.c b/arch/x86/boot/compressed/cpuflags.c index 6448a8196d32..0cc1323896d1 100644 --- a/arch/x86/boot/compressed/cpuflags.c +++ b/arch/x86/boot/compressed/cpuflags.c @@ -1,6 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#ifdef CONFIG_RANDOMIZE_BASE - #include "../cpuflags.c" bool has_cpuflag(int flag) @@ -9,5 +7,3 @@ bool has_cpuflag(int flag) return test_bit(flag, cpu.flags); } - -#endif diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 6fd41171bcfc..c0f1b6932030 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -85,8 +85,6 @@ void choose_random_location(unsigned long input, unsigned long *output, unsigned long output_size, unsigned long *virt_addr); -/* cpuflags.c */ -bool has_cpuflag(int flag); #else static inline void choose_random_location(unsigned long input, unsigned long input_size, @@ -97,6 +95,9 @@ static inline void choose_random_location(unsigned long input, } #endif +/* cpuflags.c */ +bool has_cpuflag(int flag); + #ifdef CONFIG_X86_64 extern int set_page_decrypted(unsigned long address); extern int set_page_encrypted(unsigned long address); diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 2a6c7c375244..954cb2702e23 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -145,6 +145,9 @@ void sev_es_shutdown_ghcb(void) if (!boot_ghcb) return; + if (!sev_es_check_cpu_features()) + error("SEV-ES CPU Features missing."); + /* * GHCB Page must be flushed from the cache and mapped encrypted again. * Otherwise the running kernel will see strange cache effects when diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 4be8af2f9c57..5f83ccaab877 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,6 +9,21 @@ * and is included directly into both code-bases. */ +#ifndef __BOOT_COMPRESSED +#define error(v) pr_err(v) +#define has_cpuflag(f) boot_cpu_has(f) +#endif + +static bool __init sev_es_check_cpu_features(void) +{ + if (!has_cpuflag(X86_FEATURE_RDRAND)) { + error("RDRAND instruction not supported - no trusted source of randomness available\n"); + return false; + } + + return true; +} + static void sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index db0b5cf2b2a4..79aac421789c 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -665,6 +665,9 @@ void __init sev_es_init_vc_handling(void) if (!sev_es_active()) return; + if (!sev_es_check_cpu_features()) + panic("SEV-ES CPU Features missing"); + /* Enable SEV-ES special handling */ static_branch_enable(&sev_es_enable_key); -- Gitee From 0883458ed5839a501cc40be9e0f79213fad09eab Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Fri, 25 Sep 2020 08:38:26 -0500 Subject: [PATCH 102/113] x86/sev-es: Use GHCB accessor for setting the MMIO scratch buffer commit 0ddfb1cf3b6b07c97cff16ea69931d986f9622ee upstream. Use ghcb_set_sw_scratch() to set the GHCB scratch field, which will also set the corresponding bit in the GHCB valid_bitmap field to denote that sw_scratch is actually valid. Hygon-SIG: commit 0ddfb1cf3b6b upstream x86/sev-es: Use GHCB accessor for setting the MMIO scratch buffer Signed-off-by: Tom Lendacky Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Link: https://lkml.kernel.org/r/ba84deabdf44a7a880454fb351d189c6ad79d4ba.1601041106.git.thomas.lendacky@amd.com Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 79aac421789c..9fbb39bd3a50 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -751,7 +751,7 @@ static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Can never be greater than 8 */ exit_info_2 = bytes; - ghcb->save.sw_scratch = ghcb_pa + offsetof(struct ghcb, shared_buffer); + ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); } -- Gitee From c02eeb99f16d88e35aee6c1e4607bb8be6b3e093 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Thu, 8 Oct 2020 15:16:23 -0400 Subject: [PATCH 103/113] x86/head/64: Disable stack protection for head$(BITS).o commit 103a4908ad4da9decdf9bc7216ec5a4861edf703 upstream. On 64-bit, the startup_64_setup_env() function added in 866b556efa12 ("x86/head/64: Install startup GDT") has stack protection enabled because of set_bringup_idt_handler(). This happens when CONFIG_STACKPROTECTOR_STRONG is enabled. It also currently needs CONFIG_AMD_MEM_ENCRYPT enabled because then set_bringup_idt_handler() is not an empty stub but that might change in the future, when the other vendor adds their similar technology. At this point, %gs is not yet initialized, and this doesn't cause a crash only because the #PF handler from the decompressor stub is still installed and handles the page fault. Disable stack protection for the whole file, and do it on 32-bit as well to avoid surprises. [ bp: Extend commit message with the exact explanation how it happens. ] Hygon-SIG: commit 103a4908ad4d upstream x86/head/64: Disable stack protection for head$(BITS).o Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Link: https://lkml.kernel.org/r/20201008191623.2881677-6-nivedita@alum.mit.edu Signed-off-by: hanliyang --- arch/x86/kernel/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 313aec582bec..a86cf5e7c576 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -44,6 +44,8 @@ endif # non-deterministic coverage. KCOV_INSTRUMENT := n +CFLAGS_head$(BITS).o += -fno-stack-protector + CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace obj-y := process_$(BITS).o signal.o -- Gitee From 94a2e55cf858408e57f49bdb3afdae963bfd4a16 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Fri, 16 Oct 2020 16:04:01 -0400 Subject: [PATCH 104/113] x86/boot/64: Explicitly map boot_params and command line commit b17a45b6e53f6613118b2e5cfc4a992cc50deb2c upstream. Commits ca0e22d4f011 ("x86/boot/compressed/64: Always switch to own page table") 8570978ea030 ("x86/boot/compressed/64: Don't pre-map memory in KASLR code") set up a new page table in the decompressor stub, but without explicit mappings for boot_params and the kernel command line, relying on the #PF handler instead. This is fragile, as boot_params and the command line mappings are required for the main kernel. If EARLY_PRINTK and RANDOMIZE_BASE are disabled, a QEMU/OVMF boot never accesses the command line in the decompressor stub, and so it never gets mapped. The main kernel accesses it from the identity mapping if AMD_MEM_ENCRYPT is enabled, and will crash. Fix this by adding back the explicit mapping of boot_params and the command line. Note: the changes also removed the explicit mapping of the main kernel, with the result that .bss and .brk may not be in the identity mapping, but those don't get accessed by the main kernel before it switches to its own page tables. [ bp: Pass boot_params with a MOV %rsp... instead of PUSH/POP. Use block formatting for the comment. ] Hygon-SIG: commit b17a45b6e53f upstream x86/boot/64: Explicitly map boot_params and command line Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Link: https://lkml.kernel.org/r/20201016200404.1615994-1-nivedita@alum.mit.edu Signed-off-by: hanliyang --- arch/x86/boot/compressed/head_64.S | 3 +++ arch/x86/boot/compressed/ident_map_64.c | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index ef521541e0af..835b28b673ab 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -527,6 +527,9 @@ ENDPROC(efi64_stub_entry) pushq %rsi call set_sev_encryption_mask call load_stage2_idt + + /* Pass boot_params to initialize_identity_maps() */ + movq (%rsp), %rdi call initialize_identity_maps popq %rsi diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 207101a06f63..395564ebf0c3 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -33,6 +33,12 @@ #define __PAGE_OFFSET __PAGE_OFFSET_BASE #include "../../mm/ident_map.c" +#define _SETUP +#include /* For COMMAND_LINE_SIZE */ +#undef _SETUP + +extern unsigned long get_cmd_line_ptr(void); + /* Used by PAGE_KERN* macros: */ pteval_t __default_kernel_pte_mask __read_mostly = ~0; @@ -101,8 +107,10 @@ static void add_identity_map(unsigned long start, unsigned long end) } /* Locates and clears a region for a new top level page table. */ -void initialize_identity_maps(void) +void initialize_identity_maps(void *rmode) { + unsigned long cmdline; + /* Exclude the encryption mask from __PHYSICAL_MASK */ physical_mask &= ~sme_me_mask; @@ -143,10 +151,19 @@ void initialize_identity_maps(void) } /* - * New page-table is set up - map the kernel image and load it - * into cr3. + * New page-table is set up - map the kernel image, boot_params and the + * command line. The uncompressed kernel requires boot_params and the + * command line to be mapped in the identity mapping. Map them + * explicitly here in case the compressed kernel does not touch them, + * or does not touch all the pages covering them. */ add_identity_map((unsigned long)_head, (unsigned long)_end); + boot_params = rmode; + add_identity_map((unsigned long)boot_params, (unsigned long)(boot_params + 1)); + cmdline = get_cmd_line_ptr(); + add_identity_map(cmdline, cmdline + COMMAND_LINE_SIZE); + + /* Load the new page-table. */ write_cr3(top_level_pgt); } -- Gitee From 18b01f6742326c9cd38d37d4e19a4c9200efc748 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Oct 2020 17:46:55 +0100 Subject: [PATCH 105/113] x86/boot/compressed/64: Introduce sev_status commit 3ad84246a4097010f3ae3d6944120c0be00e9e7a upstream. Introduce sev_status and initialize it together with sme_me_mask to have an indicator which SEV features are enabled. Hygon-SIG: commit 3ad84246a409 upstream x86/boot/compressed/64: Introduce sev_status Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Tom Lendacky Link: https://lkml.kernel.org/r/20201028164659.27002-2-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/boot/compressed/mem_encrypt.S | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S index 28d703cad310..124fd69a5d7a 100644 --- a/arch/x86/boot/compressed/mem_encrypt.S +++ b/arch/x86/boot/compressed/mem_encrypt.S @@ -81,6 +81,19 @@ ENTRY(set_sev_encryption_mask) bts %rax, sme_me_mask(%rip) /* Create the encryption mask */ + /* + * Read MSR_AMD64_SEV again and store it to sev_status. Can't do this in + * get_sev_encryption_bit() because this function is 32-bit code and + * shared between 64-bit and 32-bit boot path. + */ + movl $MSR_AMD64_SEV, %ecx /* Read the SEV MSR */ + rdmsr + + /* Store MSR value in sev_status */ + shlq $32, %rdx + orq %rdx, %rax + movq %rax, sev_status(%rip) + .Lno_sev_mask: movq %rbp, %rsp /* Restore original stack pointer */ @@ -96,5 +109,6 @@ ENDPROC(set_sev_encryption_mask) #ifdef CONFIG_AMD_MEM_ENCRYPT .balign 8 -SYM_DATA(sme_me_mask, .quad 0) +SYM_DATA(sme_me_mask, .quad 0) +SYM_DATA(sev_status, .quad 0) #endif -- Gitee From 3157a016e6c3fac410171a9b209df28187156dcb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Oct 2020 17:46:56 +0100 Subject: [PATCH 106/113] x86/boot/compressed/64: Sanity-check CPUID results in the early #VC handler commit ed7b895f3efb5df184722f5a30f8164fcaffceb1 upstream. The early #VC handler which doesn't have a GHCB can only handle CPUID exit codes. It is needed by the early boot code to handle #VC exceptions raised in verify_cpu() and to get the position of the C-bit. But the CPUID information comes from the hypervisor which is untrusted and might return results which trick the guest into the no-SEV boot path with no C-bit set in the page-tables. All data written to memory would then be unencrypted and could leak sensitive data to the hypervisor. Add sanity checks to the early #VC handler to make sure the hypervisor can not pretend that SEV is disabled. [ bp: Massage a bit. ] Hygon-SIG: commit ed7b895f3efb upstream x86/boot/compressed/64: Sanity-check CPUID results in the early #VC handler Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Tom Lendacky Link: https://lkml.kernel.org/r/20201028164659.27002-3-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es-shared.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 5f83ccaab877..7d04b356d44d 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -178,6 +178,32 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) goto fail; regs->dx = val >> 32; + /* + * This is a VC handler and the #VC is only raised when SEV-ES is + * active, which means SEV must be active too. Do sanity checks on the + * CPUID results to make sure the hypervisor does not trick the kernel + * into the no-sev path. This could map sensitive data unencrypted and + * make it accessible to the hypervisor. + * + * In particular, check for: + * - Hypervisor CPUID bit + * - Availability of CPUID leaf 0x8000001f + * - SEV CPUID bit. + * + * The hypervisor might still report the wrong C-bit position, but this + * can't be checked here. + */ + + if ((fn == 1 && !(regs->cx & BIT(31)))) + /* Hypervisor bit */ + goto fail; + else if (fn == 0x80000000 && (regs->ax < 0x8000001f)) + /* SEV leaf check */ + goto fail; + else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) + /* SEV bit */ + goto fail; + /* Skip over the CPUID two-byte opcode */ regs->ip += 2; -- Gitee From 502b6f7be5342b42cd24f4bf0dcecd153218da1c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Oct 2020 17:46:57 +0100 Subject: [PATCH 107/113] x86/boot/compressed/64: Check SEV encryption in 64-bit boot-path commit 86ce43f7dde81562f58b24b426cef068bd9f7595 upstream. Check whether the hypervisor reported the correct C-bit when running as an SEV guest. Using a wrong C-bit position could be used to leak sensitive data from the guest to the hypervisor. The check function is in a separate file: arch/x86/kernel/sev_verify_cbit.S so that it can be re-used in the running kernel image. [ bp: Massage. ] Hygon-SIG: commit 86ce43f7dde8 upstream x86/boot/compressed/64: Check SEV encryption in 64-bit boot-path Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Tom Lendacky Link: https://lkml.kernel.org/r/20201028164659.27002-4-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/boot/compressed/ident_map_64.c, arch/x86/boot/compressed/mem_encrypt.S, arch/x86/boot/compressed/misc.h, arch/x86/kernel/sev_verify_cbit.S] Signed-off-by: hanliyang --- arch/x86/boot/compressed/ident_map_64.c | 1 + arch/x86/boot/compressed/mem_encrypt.S | 4 ++ arch/x86/boot/compressed/misc.h | 2 + arch/x86/kernel/sev_verify_cbit.S | 89 +++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 arch/x86/kernel/sev_verify_cbit.S diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 395564ebf0c3..e8851d437d75 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -164,6 +164,7 @@ void initialize_identity_maps(void *rmode) add_identity_map(cmdline, cmdline + COMMAND_LINE_SIZE); /* Load the new page-table. */ + sev_verify_cbit(top_level_pgt); write_cr3(top_level_pgt); } diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S index 124fd69a5d7a..47256ca5f003 100644 --- a/arch/x86/boot/compressed/mem_encrypt.S +++ b/arch/x86/boot/compressed/mem_encrypt.S @@ -68,6 +68,9 @@ ENTRY(get_sev_encryption_bit) ENDPROC(get_sev_encryption_bit) .code64 + +#include "../../kernel/sev_verify_cbit.S" + ENTRY(set_sev_encryption_mask) #ifdef CONFIG_AMD_MEM_ENCRYPT push %rbp @@ -111,4 +114,5 @@ ENDPROC(set_sev_encryption_mask) .balign 8 SYM_DATA(sme_me_mask, .quad 0) SYM_DATA(sev_status, .quad 0) +SYM_DATA(sev_check_data, .quad 0) #endif diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index c0f1b6932030..60694f177056 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -159,4 +159,6 @@ void boot_page_fault(void); void boot_stage1_vc(void); void boot_stage2_vc(void); +unsigned long sev_verify_cbit(unsigned long cr3); + #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/kernel/sev_verify_cbit.S b/arch/x86/kernel/sev_verify_cbit.S new file mode 100644 index 000000000000..ee04941a6546 --- /dev/null +++ b/arch/x86/kernel/sev_verify_cbit.S @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * sev_verify_cbit.S - Code for verification of the C-bit position reported + * by the Hypervisor when running with SEV enabled. + * + * Copyright (c) 2020 Joerg Roedel (jroedel@suse.de) + * + * sev_verify_cbit() is called before switching to a new long-mode page-table + * at boot. + * + * Verify that the C-bit position is correct by writing a random value to + * an encrypted memory location while on the current page-table. Then it + * switches to the new page-table to verify the memory content is still the + * same. After that it switches back to the current page-table and when the + * check succeeded it returns. If the check failed the code invalidates the + * stack pointer and goes into a hlt loop. The stack-pointer is invalidated to + * make sure no interrupt or exception can get the CPU out of the hlt loop. + * + * New page-table pointer is expected in %rdi (first parameter) + * + */ +SYM_FUNC_START(sev_verify_cbit) +#ifdef CONFIG_AMD_MEM_ENCRYPT + /* First check if a C-bit was detected */ + movq sme_me_mask(%rip), %rsi + testq %rsi, %rsi + jz 3f + + /* sme_me_mask != 0 could mean SME or SEV - Check also for SEV */ + movq sev_status(%rip), %rsi + testq %rsi, %rsi + jz 3f + + /* Save CR4 in %rsi */ + movq %cr4, %rsi + + /* Disable Global Pages */ + movq %rsi, %rdx + andq $(~X86_CR4_PGE), %rdx + movq %rdx, %cr4 + + /* + * Verified that running under SEV - now get a random value using + * RDRAND. This instruction is mandatory when running as an SEV guest. + * + * Don't bail out of the loop if RDRAND returns errors. It is better to + * prevent forward progress than to work with a non-random value here. + */ +1: rdrand %rdx + jnc 1b + + /* Store value to memory and keep it in %rdx */ + movq %rdx, sev_check_data(%rip) + + /* Backup current %cr3 value to restore it later */ + movq %cr3, %rcx + + /* Switch to new %cr3 - This might unmap the stack */ + movq %rdi, %cr3 + + /* + * Compare value in %rdx with memory location. If C-bit is incorrect + * this would read the encrypted data and make the check fail. + */ + cmpq %rdx, sev_check_data(%rip) + + /* Restore old %cr3 */ + movq %rcx, %cr3 + + /* Restore previous CR4 */ + movq %rsi, %cr4 + + /* Check CMPQ result */ + je 3f + + /* + * The check failed, prevent any forward progress to prevent ROP + * attacks, invalidate the stack and go into a hlt loop. + */ + xorq %rsp, %rsp + subq $0x1000, %rsp +2: hlt + jmp 2b +3: +#endif + /* Return page-table pointer */ + movq %rdi, %rax + ret +SYM_FUNC_END(sev_verify_cbit) -- Gitee From 08ccf65ebebf7932f13a6b8d6f7df55611ae60b8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Oct 2020 17:46:58 +0100 Subject: [PATCH 108/113] x86/head/64: Check SEV encryption before switching to kernel page-table commit c9f09539e16e281f92a27760fdfae71e8af036f6 upstream. When SEV is enabled, the kernel requests the C-bit position again from the hypervisor to build its own page-table. Since the hypervisor is an untrusted source, the C-bit position needs to be verified before the kernel page-table is used. Call sev_verify_cbit() before writing the CR3. [ bp: Massage. ] Hygon-SIG: commit c9f09539e16e upstream x86/head/64: Check SEV encryption before switching to kernel page-table Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Tom Lendacky Link: https://lkml.kernel.org/r/20201028164659.27002-5-joro@8bytes.org [hanliyang: Fix conflict on arch/x86/kernel/head_64.S, arch/x86/mm/mem_encrypt.c] Signed-off-by: hanliyang --- arch/x86/kernel/head_64.S | 16 ++++++++++++++++ arch/x86/mm/mem_encrypt.c | 1 + 2 files changed, 17 insertions(+) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index aa8999f5cbe0..d800c125e88d 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -159,6 +159,21 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) /* Setup early boot stage 4-/5-level pagetables. */ addq phys_base(%rip), %rax + + /* + * For SEV guests: Verify that the C-bit is correct. A malicious + * hypervisor could lie about the C-bit position to perform a ROP + * attack on the guest by writing to the unencrypted stack and wait for + * the next RET instruction. + * %rsi carries pointer to realmode data and is callee-clobbered. Save + * and restore it. + */ + pushq %rsi + movq %rax, %rdi + call sev_verify_cbit + popq %rsi + + /* Switch to new page-table */ movq %rax, %cr3 /* Ensure I am executing from virtual addresses */ @@ -277,6 +292,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) END(secondary_startup_64) #include "verify_cpu.S" +#include "sev_verify_cbit.S" #ifdef CONFIG_HOTPLUG_CPU /* diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index c3ce11d715bf..6e911b788f80 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -41,6 +41,7 @@ */ u64 sme_me_mask __section(.data) = 0; u64 sev_status __section(.data) = 0; +u64 sev_check_data __section(.data) = 0; EXPORT_SYMBOL(sme_me_mask); DEFINE_STATIC_KEY_FALSE(sev_enable_key); EXPORT_SYMBOL_GPL(sev_enable_key); -- Gitee From a20ff55b886b9452f9f0cb446efc064a68d8fbeb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Oct 2020 17:46:59 +0100 Subject: [PATCH 109/113] x86/sev-es: Do not support MMIO to/from encrypted memory commit 2411cd82112397bfb9d8f0f19cd46c3d71e0ce67 upstream. MMIO memory is usually not mapped encrypted, so there is no reason to support emulated MMIO when it is mapped encrypted. Prevent a possible hypervisor attack where a RAM page is mapped as an MMIO page in the nested page-table, so that any guest access to it will trigger a #VC exception and leak the data on that page to the hypervisor via the GHCB (like with valid MMIO). On the read side this attack would allow the HV to inject data into the guest. Hygon-SIG: commit 2411cd821123 upstream x86/sev-es: Do not support MMIO to/from encrypted memory Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Tom Lendacky Link: https://lkml.kernel.org/r/20201028164659.27002-6-joro@8bytes.org Signed-off-by: hanliyang --- arch/x86/kernel/sev-es.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 9fbb39bd3a50..d74bb2ba4b7d 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -374,8 +374,8 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, return ES_EXCEPTION; } -static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, - unsigned long vaddr, phys_addr_t *paddr) +static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned long vaddr, phys_addr_t *paddr) { unsigned long va = (unsigned long)vaddr; unsigned int level; @@ -394,15 +394,19 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, if (user_mode(ctxt->regs)) ctxt->fi.error_code |= X86_PF_USER; - return false; + return ES_EXCEPTION; } + if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) + /* Emulated MMIO to/from encrypted memory not supported */ + return ES_UNSUPPORTED; + pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; pa |= va & ~page_level_mask(level); *paddr = pa; - return true; + return ES_OK; } /* Include code shared with pre-decompression boot stage */ @@ -731,6 +735,7 @@ static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, { u64 exit_code, exit_info_1, exit_info_2; unsigned long ghcb_pa = __pa(ghcb); + enum es_result res; phys_addr_t paddr; void __user *ref; @@ -740,11 +745,12 @@ static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; - if (!vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr)) { - if (!read) + res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); + if (res != ES_OK) { + if (res == ES_EXCEPTION && !read) ctxt->fi.error_code |= X86_PF_WRITE; - return ES_EXCEPTION; + return res; } exit_info_1 = paddr; -- Gitee From 31ed62c8b5ac43450246b4cf4352703933b2bddd Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 3 Dec 2020 13:50:50 +0900 Subject: [PATCH 110/113] x86/insn-eval: Use new for_each_insn_prefix() macro to loop over prefixes bytes commit 12cb908a11b2544b5f53e9af856e6b6a90ed5533 upstream. Since insn.prefixes.nbytes can be bigger than the size of insn.prefixes.bytes[] when a prefix is repeated, the proper check must be insn.prefixes.bytes[i] != 0 and i < 4 instead of using insn.prefixes.nbytes. Use the new for_each_insn_prefix() macro which does it correctly. Debugged by Kees Cook . [ bp: Massage commit message. ] Hygon-SIG: commit 12cb908a11b2 upstream x86/insn-eval: Use new for_each_insn_prefix() macro to loop over prefixes bytes Fixes: 32d0b95300db ("x86/insn-eval: Add utility functions to get segment selector") Reported-by: syzbot+9b64b619f10f19d19a7c@syzkaller.appspotmail.com Signed-off-by: Masami Hiramatsu Signed-off-by: Borislav Petkov Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/160697104969.3146288.16329307586428270032.stgit@devnote2 [hanliyang: Fix conflict on arch/x86/lib/insn-eval.c] Signed-off-by: hanliyang --- arch/x86/lib/insn-eval.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 748063c410c4..9475b26363ec 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -63,13 +63,12 @@ static bool is_string_insn(struct insn *insn) */ bool insn_has_rep_prefix(struct insn *insn) { + insn_byte_t p; int i; insn_get_prefixes(insn); - for (i = 0; i < insn->prefixes.nbytes; i++) { - insn_byte_t p = insn->prefixes.bytes[i]; - + for_each_insn_prefix(insn, i, p) { if (p == 0xf2 || p == 0xf3) return true; } -- Gitee From 7ac70eb84920fdefce0428d73c5293c8f4f97621 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 3 Dec 2020 13:51:01 +0900 Subject: [PATCH 111/113] x86/sev-es: Use new for_each_insn_prefix() macro to loop over prefixes bytes commit 84da009f06e60cf59d5e861f8e2101d2d3885517 upstream. Since insn.prefixes.nbytes can be bigger than the size of insn.prefixes.bytes[] when a prefix is repeated, the proper check must be: insn.prefixes.bytes[i] != 0 and i < 4 instead of using insn.prefixes.nbytes. Use the new for_each_insn_prefix() macro which does it correctly. Debugged by Kees Cook . [ bp: Massage commit message. ] Hygon-SIG: commit 84da009f06e6 upstream x86/sev-es: Use new for_each_insn_prefix() macro to loop over prefixes bytes Fixes: 25189d08e516 ("x86/sev-es: Add support for handling IOIO exceptions") Reported-by: syzbot+9b64b619f10f19d19a7c@syzkaller.appspotmail.com Signed-off-by: Masami Hiramatsu Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/160697106089.3146288.2052422845039649176.stgit@devnote2 Signed-off-by: hanliyang --- arch/x86/boot/compressed/sev-es.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 954cb2702e23..27826c265aab 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -32,13 +32,12 @@ struct ghcb *boot_ghcb; */ static bool insn_has_rep_prefix(struct insn *insn) { + insn_byte_t p; int i; insn_get_prefixes(insn); - for (i = 0; i < insn->prefixes.nbytes; i++) { - insn_byte_t p = insn->prefixes.bytes[i]; - + for_each_insn_prefix(insn, i, p) { if (p == 0xf2 || p == 0xf3) return true; } -- Gitee From 2583eb32a81db8b22c06c96799ac3c8ba090cdad Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Thu, 10 Dec 2020 01:25:15 +0000 Subject: [PATCH 112/113] x86,swiotlb: Adjust SWIOTLB bounce buffer size for SEV guests commit e998879d4fb7991856916972168cf27c0d86ed12 upstream. For SEV, all DMA to and from guest has to use shared (un-encrypted) pages. SEV uses SWIOTLB to make this happen without requiring changes to device drivers. However, depending on the workload being run, the default 64MB of it might not be enough and it may run out of buffers to use for DMA, resulting in I/O errors and/or performance degradation for high I/O workloads. Adjust the default size of SWIOTLB for SEV guests using a percentage of the total memory available to guest for the SWIOTLB buffers. Adds a new sev_setup_arch() function which is invoked from setup_arch() and it calls into a new swiotlb generic code function swiotlb_adjust_size() to do the SWIOTLB buffer adjustment. v5 fixed build errors and warnings as Hygon-SIG: commit e998879d4fb7 upstream x86,swiotlb: Adjust SWIOTLB bounce buffer size for SEV guests Reported-by: kbuild test robot Signed-off-by: Ashish Kalra Co-developed-by: Borislav Petkov Signed-off-by: Borislav Petkov Signed-off-by: Konrad Rzeszutek Wilk [hanliyang: Fix conflict on arch/x86/include/asm/mem_encrypt.h, arch/x86/kernel/setup.c, arch/x86/mm/mem_encrypt.c, include/linux/swiotlb.h, kernel/dma/swiotlb.c] Signed-off-by: hanliyang --- arch/x86/include/asm/mem_encrypt.h | 2 ++ arch/x86/kernel/setup.c | 6 ++++++ arch/x86/mm/mem_encrypt.c | 31 ++++++++++++++++++++++++++++++ include/linux/swiotlb.h | 8 ++++++++ kernel/dma/swiotlb.c | 20 +++++++++++++++++-- 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 239e67e0cda4..03b6fdc474a5 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -37,6 +37,7 @@ void __init sme_map_bootdata(char *real_mode_data); void __init sme_unmap_bootdata(char *real_mode_data); void __init sme_early_init(void); +void __init sev_setup_arch(void); void __init sme_encrypt_kernel(struct boot_params *bp); void __init sme_enable(struct boot_params *bp); @@ -70,6 +71,7 @@ static inline void __init sme_map_bootdata(char *real_mode_data) { } static inline void __init sme_unmap_bootdata(char *real_mode_data) { } static inline void __init sme_early_init(void) { } +static inline void __init sev_setup_arch(void) { } static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } static inline void __init sme_enable(struct boot_params *bp) { } diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index c28587deda8d..13e6b47c6ebf 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -964,6 +964,12 @@ void __init setup_arch(char **cmdline_p) memblock_set_current_limit(ISA_END_ADDRESS); e820__memblock_setup(); + /* + * Needs to run after memblock setup because it needs the physical + * memory size. + */ + sev_setup_arch(); + reserve_bios_regions(); if (efi_enabled(EFI_MEMMAP)) { diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 6e911b788f80..a23d1c62e7fb 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -200,6 +200,37 @@ void __init sme_early_init(void) swiotlb_force = SWIOTLB_FORCE; } +void __init sev_setup_arch(void) +{ + phys_addr_t total_mem = memblock_phys_mem_size(); + unsigned long size; + + if (!sev_active()) + return; + + /* + * For SEV, all DMA has to occur via shared/unencrypted pages. + * SEV uses SWIOTLB to make this happen without changing device + * drivers. However, depending on the workload being run, the + * default 64MB of SWIOTLB may not be enough and SWIOTLB may + * run out of buffers for DMA, resulting in I/O errors and/or + * performance degradation especially with high I/O workloads. + * + * Adjust the default size of SWIOTLB for SEV guests using + * a percentage of guest memory for SWIOTLB buffers. + * Also, as the SWIOTLB bounce buffer memory is allocated + * from low memory, ensure that the adjusted size is within + * the limits of low available memory. + * + * The percentage of guest memory used here for SWIOTLB buffers + * is more of an approximation of the static adjustment which + * 64MB for <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6% + */ + size = total_mem * 6 / 100; + size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G); + swiotlb_adjust_size(size); +} + static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot) { unsigned long pfn = 0; diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 0a8fced6aaec..3e154b8a48aa 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -24,6 +24,9 @@ enum swiotlb_force { */ #define IO_TLB_SEGSIZE 128 +/* default to 64MB */ +#define IO_TLB_DEFAULT_SIZE (64UL<<20) + /* * log of the size of each IO TLB slab. The number of slabs is command line * controllable. @@ -80,6 +83,7 @@ void __init swiotlb_exit(void); unsigned int swiotlb_max_segment(void); size_t swiotlb_max_mapping_size(struct device *dev); bool is_swiotlb_active(void); +void __init swiotlb_adjust_size(unsigned long new_size); #else #define swiotlb_force SWIOTLB_NO_FORCE static inline bool is_swiotlb_buffer(phys_addr_t paddr) @@ -108,6 +112,10 @@ static inline bool is_swiotlb_active(void) { return false; } + +static inline void swiotlb_adjust_size(unsigned long new_size) +{ +} #endif /* CONFIG_SWIOTLB */ extern void swiotlb_print_info(void); diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 913cb71198af..4a1fd00c4534 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -151,8 +151,6 @@ void swiotlb_set_max_segment(unsigned int val) max_segment = rounddown(val, PAGE_SIZE); } -/* default to 64MB */ -#define IO_TLB_DEFAULT_SIZE (64UL<<20) unsigned long swiotlb_size_or_default(void) { unsigned long size; @@ -162,6 +160,24 @@ unsigned long swiotlb_size_or_default(void) return size ? size : (IO_TLB_DEFAULT_SIZE); } +void __init swiotlb_adjust_size(unsigned long new_size) +{ + unsigned long size; + + /* + * If swiotlb parameter has not been specified, give a chance to + * architectures such as those supporting memory encryption to + * adjust/expand SWIOTLB size for their use. + */ + if (!io_tlb_nslabs) { + size = ALIGN(new_size, 1 << IO_TLB_SHIFT); + io_tlb_nslabs = size >> IO_TLB_SHIFT; + io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE); + + pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20); + } +} + void swiotlb_print_info(void) { unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT; -- Gitee From 62a4d60f059df4e9c7085f4fc6c062c2ea38d3a1 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Thu, 10 Dec 2020 11:09:36 -0600 Subject: [PATCH 113/113] x86/cpu: Add VM page flush MSR availablility as a CPUID feature commit 69372cf01290b9587d2cee8fbe161d75d55c3adc upstream. On systems that do not have hardware enforced cache coherency between encrypted and unencrypted mappings of the same physical page, the hypervisor can use the VM page flush MSR (0xc001011e) to flush the cache contents of an SEV guest page. When a small number of pages are being flushed, this can be used in place of issuing a WBINVD across all CPUs. CPUID 0x8000001f_eax[2] is used to determine if the VM page flush MSR is available. Add a CPUID feature to indicate it is supported and define the MSR. Hygon-SIG: commit 69372cf01290 upstream x86/cpu: Add VM page flush MSR availablility as a CPUID feature Signed-off-by: Tom Lendacky Message-Id: Signed-off-by: Paolo Bonzini [hanliyang: Fix conflict on arch/x86/include/asm/msr-index.h, arch/x86/kernel/cpu/scattered.c] Signed-off-by: hanliyang --- arch/x86/include/asm/msr-index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 3f90bd07451c..3277e1f9af52 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -499,6 +499,7 @@ #define MSR_AMD64_ICIBSEXTDCTL 0xc001103c #define MSR_AMD64_IBSOPDATA4 0xc001103d #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ +#define MSR_AMD64_VM_PAGE_FLUSH 0xc001011e #define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 -- Gitee