|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v3 2/5] x86: use PDEP/PEXT for maddr/direct-map-offset conversion when available
This allows to fold 6 instructions into a single one, reducing code size
quite a bit, especially when not considering the fallback functions
(which won't ever need to be brought into iCache or their mappings into
iTLB on systems supporting BMI2).
Make use of gcc's new V operand modifier, even if that results in a
slightly odd dependency in the sources (but I also didn't want to
introduce yet another manifest constant).
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v3: Move infrastructure pieces here from "x86: use PDEP for PTE flags
insertion when available".
v2: Avoid quoted symbols; use gcc's new V operand modifier instead.
Re-base.
--- a/xen/arch/x86/alternative.c
+++ b/xen/arch/x86/alternative.c
@@ -224,6 +224,12 @@ void init_or_livepatch apply_alternative
/* 0xe8/0xe9 are relative branches; fix the offset. */
if ( a->repl_len >= 5 && (*buf & 0xfe) == 0xe8 )
*(int32_t *)(buf + 1) += repl - orig;
+ /* RIP-relative addressing is easy to check for in VEX-encoded insns.
*/
+ else if ( a->repl_len >= 8 &&
+ (*buf & ~1) == 0xc4 &&
+ a->repl_len >= 9 - (*buf & 1) &&
+ (buf[4 - (*buf & 1)] & ~0x38) == 0x05 )
+ *(int32_t *)(buf + 5 - (*buf & 1)) += repl - orig;
add_nops(buf + a->repl_len, total_len - a->repl_len);
text_poke(orig, buf, total_len);
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -390,6 +390,25 @@ void __init arch_init_memory(void)
#endif
}
+paddr_t __read_mostly ma_real_mask = ~0UL;
+
+#ifndef CONFIG_INDIRECT_THUNK /* V modifier unavailable? */
+
+/* Conversion between machine address and direct map offset. */
+paddr_t do2ma(unsigned long off)
+{
+ return (off & ma_va_bottom_mask) |
+ ((off << pfn_pdx_hole_shift) & ma_top_mask);
+}
+
+unsigned long ma2do(paddr_t ma)
+{
+ return (ma & ma_va_bottom_mask) |
+ ((ma & ma_top_mask) >> pfn_pdx_hole_shift);
+}
+
+#endif
+
int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
{
uint64_t maddr = pfn_to_paddr(mfn);
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -450,6 +450,8 @@ void __init srat_parse_regions(u64 addr)
}
pfn_pdx_hole_setup(mask >> PAGE_SHIFT);
+
+ ma_real_mask = ma_top_mask | ma_va_bottom_mask;
}
/* Use the information discovered above to actually set up the nodes. */
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -66,6 +66,7 @@ SECTIONS
_stext = .; /* Text and read-only data */
*(.text)
*(.text.__x86_indirect_thunk_*)
+ *(.gnu.linkonce.t.*)
*(.text.page_aligned)
. = ALIGN(PAGE_SIZE);
--- a/xen/include/asm-x86/asm_defns.h
+++ b/xen/include/asm-x86/asm_defns.h
@@ -186,6 +186,20 @@ void ret_from_intr(void);
UNLIKELY_END_SECTION "\n" \
".Llikely." #tag ".%=:"
+#define LINKONCE_PROLOGUE(sym) \
+ ".ifndef " sym() "\n\t" \
+ ".pushsection " sym(.gnu.linkonce.t.) "," \
+ "\"ax\",@progbits\n\t" \
+ ".p2align 4\n" \
+ sym() ":"
+
+#define LINKONCE_EPILOGUE(sym) \
+ ".weak " sym() "\n\t" \
+ ".type " sym() ", @function\n\t" \
+ ".size " sym() ", . - " sym() "\n\t" \
+ ".popsection\n\t" \
+ ".endif"
+
#endif
/* "Raw" instruction opcodes */
--- a/xen/include/asm-x86/x86_64/page.h
+++ b/xen/include/asm-x86/x86_64/page.h
@@ -42,11 +42,18 @@ static inline unsigned long canonicalise
return addr & ~CANONICAL_MASK;
}
+#include <asm/alternative.h>
+#include <asm/asm_defns.h>
+#include <asm/cpufeature.h>
#include <asm/types.h>
#include <xen/pdx.h>
extern unsigned long xen_virt_end;
+extern paddr_t ma_real_mask;
+
+paddr_t do2ma(unsigned long);
+unsigned long ma2do(paddr_t);
/*
* Note: These are solely for the use by page_{get,set}_owner(), and
@@ -57,8 +64,10 @@ extern unsigned long xen_virt_end;
#define pdx_to_virt(pdx) ((void *)(DIRECTMAP_VIRT_START + \
((unsigned long)(pdx) << PAGE_SHIFT)))
-static inline unsigned long __virt_to_maddr(unsigned long va)
+static always_inline paddr_t __virt_to_maddr(unsigned long va)
{
+ paddr_t ma;
+
ASSERT(va < DIRECTMAP_VIRT_END);
if ( va >= DIRECTMAP_VIRT_START )
va -= DIRECTMAP_VIRT_START;
@@ -71,16 +80,77 @@ static inline unsigned long __virt_to_ma
va += xen_phys_start - XEN_VIRT_START;
}
- return (va & ma_va_bottom_mask) |
- ((va << pfn_pdx_hole_shift) & ma_top_mask);
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "do2ma_%V[ma]_%V[off]"
+ alternative_io("call " SYMNAME() "\n\t"
+ LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+ "mov %[shift], %%ecx\n\t"
+ "mov %[off], %[ma]\n\t"
+ "and %[bmask], %[ma]\n\t"
+ "shl %%cl, %[off]\n\t"
+ "and %[tmask], %[off]\n\t"
+ "or %[off], %[ma]\n\t"
+ "ret\n\t"
+ LINKONCE_EPILOGUE(SYMNAME),
+ "pdep %[mask], %[off], %[ma]", X86_FEATURE_BMI2,
+ ASM_OUTPUT2([ma] "=&r" (ma), [off] "+r" (va)),
+ [mask] "m" (ma_real_mask),
+ [shift] "m" (pfn_pdx_hole_shift),
+ [bmask] "m" (ma_va_bottom_mask),
+ [tmask] "m" (ma_top_mask)
+ : "ecx");
+#undef SYMNAME
+#else
+ alternative_io("call do2ma",
+ /* pdep ma_real_mask(%rip), %rdi, %rax */
+ ".byte 0xc4, 0xe2, 0xc3, 0xf5, 0x05\n\t"
+ ".long ma_real_mask - 4 - .",
+ X86_FEATURE_BMI2,
+ ASM_OUTPUT2("=a" (ma), "+D" (va)), "m" (ma_real_mask)
+ : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+ return ma;
}
-static inline void *__maddr_to_virt(unsigned long ma)
+static always_inline void *__maddr_to_virt(paddr_t ma)
{
+ unsigned long off;
+
ASSERT(pfn_to_pdx(ma >> PAGE_SHIFT) < (DIRECTMAP_SIZE >> PAGE_SHIFT));
- return (void *)(DIRECTMAP_VIRT_START +
- ((ma & ma_va_bottom_mask) |
- ((ma & ma_top_mask) >> pfn_pdx_hole_shift)));
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "ma2do_%V[off]_%V[ma]"
+ alternative_io("call " SYMNAME() "\n\t"
+ LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+ "mov %[tmask], %[off]\n\t"
+ "mov %[shift], %%ecx\n\t"
+ "and %[ma], %[off]\n\t"
+ "and %[bmask], %[ma]\n\t"
+ "shr %%cl, %[off]\n\t"
+ "or %[ma], %[off]\n\t"
+ "ret\n\t"
+ LINKONCE_EPILOGUE(SYMNAME),
+ "pext %[mask], %[ma], %[off]", X86_FEATURE_BMI2,
+ ASM_OUTPUT2([off] "=&r" (off), [ma] "+r" (ma)),
+ [mask] "m" (ma_real_mask),
+ [shift] "m" (pfn_pdx_hole_shift),
+ [bmask] "m" (ma_va_bottom_mask),
+ [tmask] "m" (ma_top_mask)
+ : "ecx");
+#undef SYMNAME
+#else
+ alternative_io("call ma2do",
+ /* pext ma_real_mask(%rip), %rdi, %rax */
+ ".byte 0xc4, 0xe2, 0xc2, 0xf5, 0x05\n\t"
+ ".long ma_real_mask - 4 - .",
+ X86_FEATURE_BMI2,
+ ASM_OUTPUT2("=a" (off), "+D" (ma)), "m" (ma_real_mask)
+ : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+ return (void *)DIRECTMAP_VIRT_START + off;
}
/* read access (should only be used for debug printk's) */
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |