|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v6.5 11/26] x86: Support indirect thunks from assembly code
Introduce CALL_THUNK and JMP_THUNK which either degrade to a normal indirect
branch, or dispatch to the __x86.indirect_thunk.* symbols.
Update all the manual indirect branches in to use the new thunks. The
indirect branches in the early boot and kexec path are left intact as we can't
use the compiled-in thunks at those points.
Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
---
v4:
* New
v6:
* Thunk more assembly
* Fix check_wakeup_from_wait() to not corrupt its stack
v7:
* Protect the jmp from the trampoline into the high mappings on the AP boot
path, and the hand-crafted indirect jump in the PV IO emulation stubs.
---
xen/arch/x86/boot/trampoline.S | 24 +++++++++++++++++--
xen/arch/x86/extable.c | 4 ++--
xen/arch/x86/pv/emul-priv-op.c | 41 ++++++++++++++++++++++++--------
xen/arch/x86/x86_64/entry.S | 6 +++--
xen/arch/x86/x86_emulate/x86_emulate.c | 4 ++--
xen/common/wait.c | 8 ++++---
xen/include/asm-x86/asm_defns.h | 8 +++++++
xen/include/asm-x86/indirect_thunk_asm.h | 41 ++++++++++++++++++++++++++++++++
8 files changed, 115 insertions(+), 21 deletions(-)
create mode 100644 xen/include/asm-x86/indirect_thunk_asm.h
diff --git a/xen/arch/x86/boot/trampoline.S b/xen/arch/x86/boot/trampoline.S
index 4d640f3..abf40f3 100644
--- a/xen/arch/x86/boot/trampoline.S
+++ b/xen/arch/x86/boot/trampoline.S
@@ -153,8 +153,28 @@ trampoline_protmode_entry:
.code64
start64:
/* Jump to high mappings. */
- movabs $__high_start,%rax
- jmpq *%rax
+ movabs $__high_start, %rdi
+
+#ifdef CONFIG_INDIRECT_THUNK
+ /*
+ * If booting virtualised, or hot-onlining a CPU, sibling threads can
+ * attempt Branch Target Injection against this jmp.
+ *
+ * We've got no usable stack so can't use a RETPOLINE thunk, and are
+ * further than +- 2G from the high mappings so couldn't use JUMP_THUNK
+ * even if was a non-RETPOLINE thunk. Futhermore, an LFENCE isn't
+ * necesserily safe to use at this point.
+ *
+ * As this isn't a hotpath, use a fully serialising event to reduce
+ * the speculation window as much as possible. %ebx needs preserving
+ * for __high_start.
+ */
+ mov %ebx, %esi
+ cpuid
+ mov %esi, %ebx
+#endif
+
+ jmpq *%rdi
#include "wakeup.S"
diff --git a/xen/arch/x86/extable.c b/xen/arch/x86/extable.c
index 6fffe05..bf5822d 100644
--- a/xen/arch/x86/extable.c
+++ b/xen/arch/x86/extable.c
@@ -158,7 +158,7 @@ static int __init stub_selftest(void)
memcpy(ptr, tests[i].opc, ARRAY_SIZE(tests[i].opc));
unmap_domain_page(ptr);
- asm volatile ( "call *%[stb]\n"
+ asm volatile ( "CALL_THUNK %[stb]\n"
".Lret%=:\n\t"
".pushsection .fixup,\"ax\"\n"
".Lfix%=:\n\t"
@@ -167,7 +167,7 @@ static int __init stub_selftest(void)
".popsection\n\t"
_ASM_EXTABLE(.Lret%=, .Lfix%=)
: [exn] "+m" (res)
- : [stb] "rm" (addr), "a" (tests[i].rax));
+ : [stb] "r" (addr), "a" (tests[i].rax));
ASSERT(res == tests[i].res.raw);
}
diff --git a/xen/arch/x86/pv/emul-priv-op.c b/xen/arch/x86/pv/emul-priv-op.c
index 1041a4c..6da9b46 100644
--- a/xen/arch/x86/pv/emul-priv-op.c
+++ b/xen/arch/x86/pv/emul-priv-op.c
@@ -73,37 +73,58 @@ void (*pv_post_outb_hook)(unsigned int port, u8 value);
typedef void io_emul_stub_t(struct cpu_user_regs *);
+#ifdef CONFIG_INDIRECT_THUNK
+extern char ind_thunk_rcx[] asm ("__x86.indirect_thunk.rcx");
+#endif
+
static io_emul_stub_t *io_emul_stub_setup(struct priv_op_ctxt *ctxt, u8 opcode,
unsigned int port, unsigned int
bytes)
{
+ struct stubs *this_stubs = &this_cpu(stubs);
+ unsigned long stub_va = this_stubs->addr + STUB_BUF_SIZE / 2;
+
if ( !ctxt->io_emul_stub )
- ctxt->io_emul_stub = map_domain_page(_mfn(this_cpu(stubs.mfn))) +
- (this_cpu(stubs.addr) &
- ~PAGE_MASK) +
- STUB_BUF_SIZE / 2;
+ ctxt->io_emul_stub =
+ map_domain_page(_mfn(this_stubs->mfn)) + (stub_va & ~PAGE_MASK);
/* movq $host_to_guest_gpr_switch,%rcx */
ctxt->io_emul_stub[0] = 0x48;
ctxt->io_emul_stub[1] = 0xb9;
*(void **)&ctxt->io_emul_stub[2] = (void *)host_to_guest_gpr_switch;
+
+#ifdef CONFIG_INDIRECT_THUNK
+ /* callq __x86.indirect_thunk.rcx */
+ ctxt->io_emul_stub[10] = 0xe8;
+ *(int32_t *)&ctxt->io_emul_stub[11] =
+ _p(ind_thunk_rcx) - _p(stub_va + 11 + 4);
+
+#else
/* callq *%rcx */
ctxt->io_emul_stub[10] = 0xff;
ctxt->io_emul_stub[11] = 0xd1;
+
+ /*
+ * 3 bytes of P6_NOPS.
+ * TODO: untangle ideal_nops from init/livepatch Kconfig options.
+ */
+ memcpy(&ctxt->io_emul_stub[12], "\x0f\x1f\x00", 3);
+#endif
+
/* data16 or nop */
- ctxt->io_emul_stub[12] = (bytes != 2) ? 0x90 : 0x66;
+ ctxt->io_emul_stub[15] = (bytes != 2) ? 0x90 : 0x66;
/* <io-access opcode> */
- ctxt->io_emul_stub[13] = opcode;
+ ctxt->io_emul_stub[16] = opcode;
/* imm8 or nop */
- ctxt->io_emul_stub[14] = !(opcode & 8) ? port : 0x90;
+ ctxt->io_emul_stub[17] = !(opcode & 8) ? port : 0x90;
/* ret (jumps to guest_to_host_gpr_switch) */
- ctxt->io_emul_stub[15] = 0xc3;
- BUILD_BUG_ON(STUB_BUF_SIZE / 2 < 16);
+ ctxt->io_emul_stub[18] = 0xc3;
+ BUILD_BUG_ON(STUB_BUF_SIZE / 2 < 19);
if ( ioemul_handle_quirk )
ioemul_handle_quirk(opcode, &ctxt->io_emul_stub[12], ctxt->ctxt.regs);
/* Handy function-typed pointer to the stub. */
- return (void *)(this_cpu(stubs.addr) + STUB_BUF_SIZE / 2);
+ return (void *)stub_va;
}
diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S
index 1dd9ccf..b49d62b 100644
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -474,7 +474,8 @@ handle_exception_saved:
movzbl UREGS_entry_vector(%rsp),%eax
leaq exception_table(%rip),%rdx
PERFC_INCR(exceptions, %rax, %rbx)
- callq *(%rdx,%rax,8)
+ mov (%rdx, %rax, 8), %rdx
+ CALL_THUNK %rdx
testb $3,UREGS_cs(%rsp)
jz restore_all_xen
leaq VCPU_trap_bounce(%rbx),%rdx
@@ -615,7 +616,8 @@ handle_ist_exception:
1: movq %rsp,%rdi
movzbl UREGS_entry_vector(%rsp),%eax
leaq exception_table(%rip),%rdx
- callq *(%rdx,%rax,8)
+ mov (%rdx, %rax, 8), %rdx
+ CALL_THUNK %rdx
cmpb $TRAP_nmi,UREGS_entry_vector(%rsp)
jne ret_from_intr
diff --git a/xen/arch/x86/x86_emulate/x86_emulate.c
b/xen/arch/x86/x86_emulate/x86_emulate.c
index 820495f..3d56b81 100644
--- a/xen/arch/x86/x86_emulate/x86_emulate.c
+++ b/xen/arch/x86/x86_emulate/x86_emulate.c
@@ -867,7 +867,7 @@ static inline int mkec(uint8_t e, int32_t ec, ...)
#ifdef __XEN__
# define invoke_stub(pre, post, constraints...) do { \
union stub_exception_token res_ = { .raw = ~0 }; \
- asm volatile ( pre "\n\tcall *%[stub]\n\t" post "\n" \
+ asm volatile ( pre "\n\tCALL_THUNK %[stub]\n\t" post "\n" \
".Lret%=:\n\t" \
".pushsection .fixup,\"ax\"\n" \
".Lfix%=:\n\t" \
@@ -876,7 +876,7 @@ static inline int mkec(uint8_t e, int32_t ec, ...)
".popsection\n\t" \
_ASM_EXTABLE(.Lret%=, .Lfix%=) \
: [exn] "+g" (res_), constraints, \
- [stub] "rm" (stub.func), \
+ [stub] "r" (stub.func), \
"m" (*(uint8_t(*)[MAX_INST_LEN + 1])stub.ptr) ); \
if ( unlikely(~res_.raw) ) \
{ \
diff --git a/xen/common/wait.c b/xen/common/wait.c
index 3d3d9fe..4b5a8bf 100644
--- a/xen/common/wait.c
+++ b/xen/common/wait.c
@@ -203,12 +203,14 @@ void check_wakeup_from_wait(void)
/*
* Hand-rolled longjmp(). Returns to the pointer on the top of
- * wqv->stack, and lands on a `rep movs` instruction.
+ * wqv->stack, and lands on a `rep movs` instruction. All other GPRs are
+ * restored from the stack, so are available for use here.
*/
asm volatile (
- "mov %1,%%"__OP"sp; jmp *(%0)"
+ "mov %1,%%"__OP"sp; JMP_THUNK %[ip]"
: : "S" (wqv->stack), "D" (wqv->esp),
- "c" ((char *)get_cpu_info() - (char *)wqv->esp)
+ "c" ((char *)get_cpu_info() - (char *)wqv->esp),
+ [ip] "r" (*(unsigned long *)wqv->stack)
: "memory" );
unreachable();
}
diff --git a/xen/include/asm-x86/asm_defns.h b/xen/include/asm-x86/asm_defns.h
index 7e8838e..40b0250 100644
--- a/xen/include/asm-x86/asm_defns.h
+++ b/xen/include/asm-x86/asm_defns.h
@@ -13,6 +13,14 @@
#include <asm/cpufeature.h>
#include <asm/alternative.h>
+#ifdef __ASSEMBLY__
+# include <asm/indirect_thunk_asm.h>
+#else
+asm ( "\t.equ CONFIG_INDIRECT_THUNK, "
+ __stringify(IS_ENABLED(CONFIG_INDIRECT_THUNK)) );
+asm ( "\t.include \"asm/indirect_thunk_asm.h\"" );
+#endif
+
#ifndef __ASSEMBLY__
void ret_from_intr(void);
#endif
diff --git a/xen/include/asm-x86/indirect_thunk_asm.h
b/xen/include/asm-x86/indirect_thunk_asm.h
new file mode 100644
index 0000000..554dd7e
--- /dev/null
+++ b/xen/include/asm-x86/indirect_thunk_asm.h
@@ -0,0 +1,41 @@
+/*
+ * Warning! This file is included at an assembler level for .c files, causing
+ * usual #ifdef'ary to turn into comments.
+ */
+
+.macro IND_THUNK insn:req arg:req
+/*
+ * Create an indirect branch. insn is one of call/jmp, arg is a single
+ * register.
+ *
+ * With no compiler support, this degrated into a plain indirect call/jmp.
+ * With compiler support, dispatch to the correct __x86.indirect_thunk.*
+ */
+ .if CONFIG_INDIRECT_THUNK == 1
+
+ $done = 0
+ .irp reg, rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9, r10, r11, r12,
r13, r14, r15
+ .ifeqs "\arg", "%\reg"
+ \insn __x86.indirect_thunk.\reg
+ $done = 1
+ .exitm
+ .endif
+ .endr
+
+ .if $done != 1
+ .error "Bad register arg \arg"
+ .endif
+
+ .else
+ \insn *\arg
+ .endif
+.endm
+
+/* Convenience wrappers. */
+.macro CALL_THUNK arg:req
+ IND_THUNK call, \arg
+.endm
+
+.macro JMP_THUNK arg:req
+ IND_THUNK jmp, \arg
+.endm
--
2.1.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |