|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH 17/22] x86/mm: introduce support to populate a per-CPU page-table region
Add logic in map_pages_to_xen() and modify_xen_mappings() so that TLB flushes
are only performed locally when dealing with entries in the per-CPU area of the
page-tables.
No functional change intended, as there are no callers added that create or
modify per-CPU mappings, nor is the per-CPU area still properly setup in
the page-tables yet.
Note that the removed flush_area() ended up calling flush_area_mask() through
the flush_area_all() alias.
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
xen/arch/x86/include/asm/config.h | 4 ++
xen/arch/x86/include/asm/flushtlb.h | 1 -
xen/arch/x86/mm.c | 64 +++++++++++++++++++----------
3 files changed, 47 insertions(+), 22 deletions(-)
diff --git a/xen/arch/x86/include/asm/config.h
b/xen/arch/x86/include/asm/config.h
index 2a260a2581fd..c24d735a0cee 100644
--- a/xen/arch/x86/include/asm/config.h
+++ b/xen/arch/x86/include/asm/config.h
@@ -204,6 +204,10 @@ extern unsigned char boot_edid_info[128];
#define PERDOMAIN_SLOTS 3
#define PERDOMAIN_VIRT_SLOT(s) (PERDOMAIN_VIRT_START + (s) * \
(PERDOMAIN_SLOT_MBYTES << 20))
+#define PERCPU_VIRT_START PERDOMAIN_VIRT_SLOT(PERDOMAIN_SLOTS)
+#define PERCPU_SLOTS 1
+#define PERCPU_VIRT_SLOT(s) (PERCPU_VIRT_START + (s) * \
+ (PERDOMAIN_SLOT_MBYTES << 20))
/* Slot 4: mirror of per-domain mappings (for compat xlat area accesses). */
#define PERDOMAIN_ALT_VIRT_START PML4_ADDR(4)
/* Slot 261: machine-to-phys conversion table (256GB). */
diff --git a/xen/arch/x86/include/asm/flushtlb.h
b/xen/arch/x86/include/asm/flushtlb.h
index 1b98d03decdc..affe944d1a5b 100644
--- a/xen/arch/x86/include/asm/flushtlb.h
+++ b/xen/arch/x86/include/asm/flushtlb.h
@@ -146,7 +146,6 @@ void flush_area_mask(const cpumask_t *mask, const void *va,
#define flush_mask(mask, flags) flush_area_mask(mask, NULL, flags)
/* Flush all CPUs' TLBs/caches */
-#define flush_area_all(va, flags) flush_area_mask(&cpu_online_map, va, flags)
#define flush_all(flags) flush_mask(&cpu_online_map, flags)
/* Flush local TLBs */
diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 1367f3361ffe..c468b46a9d1b 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -5023,9 +5023,13 @@ static DEFINE_SPINLOCK(map_pgdir_lock);
*/
static l3_pgentry_t *virt_to_xen_l3e(unsigned long v)
{
+ unsigned int cpu = smp_processor_id();
+ /* Called before idle_vcpu is populated, fallback to idle_pg_table. */
+ root_pgentry_t *root_pgt = idle_vcpu[cpu] ?
+ maddr_to_virt(idle_vcpu[cpu]->arch.cr3) : idle_pg_table;
l4_pgentry_t *pl4e;
- pl4e = &idle_pg_table[l4_table_offset(v)];
+ pl4e = &root_pgt[l4_table_offset(v)];
if ( !(l4e_get_flags(*pl4e) & _PAGE_PRESENT) )
{
bool locking = system_state > SYS_STATE_boot;
@@ -5138,8 +5142,8 @@ static l1_pgentry_t *virt_to_xen_l1e(unsigned long v)
#define l1f_to_lNf(f) (((f) & _PAGE_PRESENT) ? ((f) | _PAGE_PSE) : (f))
#define lNf_to_l1f(f) (((f) & _PAGE_PRESENT) ? ((f) & ~_PAGE_PSE) : (f))
-/* flush_area_all() can be used prior to any other CPU being online. */
-#define flush_area(v, f) flush_area_all((const void *)(v), f)
+/* flush_area_mask() can be used prior to any other CPU being online. */
+#define flush_area_mask(m, v, f) flush_area_mask(m, (const void *)(v), f)
#define L3T_INIT(page) (page) = ZERO_BLOCK_PTR
@@ -5222,7 +5226,11 @@ int map_pages_to_xen(
unsigned long nr_mfns,
unsigned int flags)
{
- bool locking = system_state > SYS_STATE_boot;
+ bool global = virt < PERCPU_VIRT_START ||
+ virt >= PERCPU_VIRT_SLOT(PERCPU_SLOTS);
+ bool locking = system_state > SYS_STATE_boot && global;
+ const cpumask_t *flush_mask = global ? &cpu_online_map
+ : cpumask_of(smp_processor_id());
l3_pgentry_t *pl3e = NULL, ol3e;
l2_pgentry_t *pl2e = NULL, ol2e;
l1_pgentry_t *pl1e, ol1e;
@@ -5244,6 +5252,11 @@ int map_pages_to_xen(
} \
} while (0)
+ /* Ensure it's a global mapping or it's only modifying the per-CPU area. */
+ ASSERT(global ||
+ (virt + nr_mfns * PAGE_SIZE >= PERCPU_VIRT_START &&
+ virt + nr_mfns * PAGE_SIZE < PERCPU_VIRT_SLOT(PERCPU_SLOTS)));
+
L3T_INIT(current_l3page);
while ( nr_mfns != 0 )
@@ -5278,7 +5291,7 @@ int map_pages_to_xen(
if ( l3e_get_flags(ol3e) & _PAGE_PSE )
{
flush_flags(lNf_to_l1f(l3e_get_flags(ol3e)));
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
}
else
{
@@ -5301,7 +5314,7 @@ int map_pages_to_xen(
unmap_domain_page(l1t);
}
}
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
{
ol2e = l2t[i];
@@ -5373,7 +5386,7 @@ int map_pages_to_xen(
}
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
free_xen_pagetable(l2mfn);
}
@@ -5399,7 +5412,7 @@ int map_pages_to_xen(
if ( l2e_get_flags(ol2e) & _PAGE_PSE )
{
flush_flags(lNf_to_l1f(l2e_get_flags(ol2e)));
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
}
else
{
@@ -5407,7 +5420,7 @@ int map_pages_to_xen(
for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
flush_flags(l1e_get_flags(l1t[i]));
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
unmap_domain_page(l1t);
free_xen_pagetable(l2e_get_mfn(ol2e));
}
@@ -5476,7 +5489,7 @@ int map_pages_to_xen(
}
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
free_xen_pagetable(l1mfn);
}
@@ -5491,7 +5504,7 @@ int map_pages_to_xen(
unsigned int flush_flags = FLUSH_TLB | FLUSH_ORDER(0);
flush_flags(l1e_get_flags(ol1e));
- flush_area(virt, flush_flags);
+ flush_area_mask(flush_mask, virt, flush_flags);
}
virt += 1UL << L1_PAGETABLE_SHIFT;
@@ -5540,9 +5553,9 @@ int map_pages_to_xen(
l2e_write(pl2e, l2e_from_pfn(base_mfn, l1f_to_lNf(flags)));
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(virt - PAGE_SIZE,
- FLUSH_TLB_GLOBAL |
- FLUSH_ORDER(PAGETABLE_ORDER));
+ flush_area_mask(flush_mask, virt - PAGE_SIZE,
+ FLUSH_TLB_GLOBAL |
+ FLUSH_ORDER(PAGETABLE_ORDER));
free_xen_pagetable(l2e_get_mfn(ol2e));
}
else if ( locking )
@@ -5589,9 +5602,9 @@ int map_pages_to_xen(
l3e_write(pl3e, l3e_from_pfn(base_mfn, l1f_to_lNf(flags)));
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(virt - PAGE_SIZE,
- FLUSH_TLB_GLOBAL |
- FLUSH_ORDER(2*PAGETABLE_ORDER));
+ flush_area_mask(flush_mask, virt - PAGE_SIZE,
+ FLUSH_TLB_GLOBAL |
+ FLUSH_ORDER(2*PAGETABLE_ORDER));
free_xen_pagetable(l3e_get_mfn(ol3e));
}
else if ( locking )
@@ -5629,7 +5642,11 @@ int __init populate_pt_range(unsigned long virt,
unsigned long nr_mfns)
*/
int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
{
- bool locking = system_state > SYS_STATE_boot;
+ bool global = s < PERCPU_VIRT_START ||
+ s >= PERCPU_VIRT_SLOT(PERCPU_SLOTS);
+ bool locking = system_state > SYS_STATE_boot && global;
+ const cpumask_t *flush_mask = global ? &cpu_online_map
+ : cpumask_of(smp_processor_id());
l3_pgentry_t *pl3e = NULL;
l2_pgentry_t *pl2e = NULL;
l1_pgentry_t *pl1e;
@@ -5638,6 +5655,9 @@ int modify_xen_mappings(unsigned long s, unsigned long e,
unsigned int nf)
int rc = -ENOMEM;
struct page_info *current_l3page;
+ ASSERT(global ||
+ (e >= PERCPU_VIRT_START && e < PERCPU_VIRT_SLOT(PERCPU_SLOTS)));
+
/* Set of valid PTE bits which may be altered. */
#define FLAGS_MASK (_PAGE_NX|_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW|_PAGE_PRESENT)
nf &= FLAGS_MASK;
@@ -5836,7 +5856,8 @@ int modify_xen_mappings(unsigned long s, unsigned long e,
unsigned int nf)
l2e_write(pl2e, l2e_empty());
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(NULL, FLUSH_TLB_GLOBAL); /* flush before free */
+ /* flush before free */
+ flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
free_xen_pagetable(l1mfn);
}
else if ( locking )
@@ -5880,7 +5901,8 @@ int modify_xen_mappings(unsigned long s, unsigned long e,
unsigned int nf)
l3e_write(pl3e, l3e_empty());
if ( locking )
spin_unlock(&map_pgdir_lock);
- flush_area(NULL, FLUSH_TLB_GLOBAL); /* flush before free */
+ /* flush before free */
+ flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
free_xen_pagetable(l2mfn);
}
else if ( locking )
@@ -5888,7 +5910,7 @@ int modify_xen_mappings(unsigned long s, unsigned long e,
unsigned int nf)
}
}
- flush_area(NULL, FLUSH_TLB_GLOBAL);
+ flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
#undef FLAGS_MASK
rc = 0;
--
2.45.2
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |