ChangeSet 1.1236.32.14, 2005/03/21 12:01:36+00:00, mafetter@xxxxxxxxxxxxxxxx
Added prediction of where to find the last writable PTE for a given
page;
greatly speeds up promotion of a page to be used as a page table.
Removed some broken concepts of write protecting PDEs and higher level
entries. To write protect a page, all we need to do is write protect
all
L1 entries that point at it.
Fixed a bug with translated IO pages; gotta check that MFNs are really
backed
by RAM before we go looking in the frame_table for them...
Signed-off-by: michael.fetterman@xxxxxxxxxxxx
arch/x86/audit.c | 12 +-
arch/x86/mm.c | 2
arch/x86/shadow.c | 237 +++++++++++++++++++++++++++++++----------------
include/asm-x86/mm.h | 9 +
include/asm-x86/shadow.h | 94 ++++++++++++------
include/xen/perfc.h | 1
include/xen/perfc_defn.h | 7 +
7 files changed, 248 insertions(+), 114 deletions(-)
diff -Nru a/xen/arch/x86/audit.c b/xen/arch/x86/audit.c
--- a/xen/arch/x86/audit.c 2005-04-05 12:14:01 -04:00
+++ b/xen/arch/x86/audit.c 2005-04-05 12:14:01 -04:00
@@ -333,22 +333,26 @@
smfn = a->smfn;
page = &frame_table[smfn];
- adjust(pfn_to_page(gmfn), 0);
-
switch ( a->gpfn_and_flags & PGT_type_mask ) {
+ case PGT_writable_pred:
+ break;
case PGT_snapshot:
+ adjust(pfn_to_page(gmfn), 0);
break;
case PGT_l1_shadow:
+ adjust(pfn_to_page(gmfn), 0);
adjust_l1_page(smfn);
if ( page->u.inuse.type_info & PGT_pinned )
adjust(page, 0);
break;
case PGT_hl2_shadow:
+ adjust(pfn_to_page(gmfn), 0);
adjust_hl2_page(smfn);
if ( page->u.inuse.type_info & PGT_pinned )
adjust(page, 0);
break;
case PGT_l2_shadow:
+ adjust(pfn_to_page(gmfn), 0);
adjust_l2_page(smfn);
if ( page->u.inuse.type_info & PGT_pinned )
adjust(page, 0);
@@ -619,6 +623,7 @@
scan_for_pfn_in_mfn(d, xmfn, a->smfn);
break;
case PGT_snapshot:
+ case PGT_writable_pred:
break;
default:
BUG();
@@ -834,6 +839,9 @@
page->count_info);
errors++;
}
+ break;
+ case PGT_writable_pred:
+ // XXX - nothing to check?
break;
default:
diff -Nru a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
--- a/xen/arch/x86/mm.c 2005-04-05 12:14:01 -04:00
+++ b/xen/arch/x86/mm.c 2005-04-05 12:14:01 -04:00
@@ -268,7 +268,7 @@
if ( unlikely(shadow_mode_enabled(d)) )
{
shadow_lock(d);
- shadow_remove_all_write_access(d, PGT_l1_shadow, PGT_l1_shadow, gpfn,
gmfn);
+ shadow_remove_all_write_access(d, gpfn, gmfn);
}
res = get_page_and_type(&frame_table[gmfn], d, PGT_ldt_page);
diff -Nru a/xen/arch/x86/shadow.c b/xen/arch/x86/shadow.c
--- a/xen/arch/x86/shadow.c 2005-04-05 12:14:00 -04:00
+++ b/xen/arch/x86/shadow.c 2005-04-05 12:14:00 -04:00
@@ -48,7 +48,6 @@
shadow_promote(struct domain *d, unsigned long gpfn, unsigned long gmfn,
unsigned long new_type)
{
- unsigned long min_type, max_type;
struct pfn_info *page = pfn_to_page(gmfn);
int pinned = 0, okay = 1;
@@ -61,20 +60,11 @@
}
if ( unlikely(page_is_page_table(page)) )
- {
- min_type = shadow_max_pgtable_type(d, gpfn) + PGT_l1_shadow;
- max_type = new_type;
- }
- else
- {
- min_type = PGT_l1_shadow;
- max_type = PGT_l1_shadow;
- }
- FSH_LOG("shadow_promote gpfn=%p gmfn=%p nt=%p min=%p max=%p",
- gpfn, gmfn, new_type, min_type, max_type);
+ return 1;
- if ( (min_type <= max_type) &&
- !shadow_remove_all_write_access(d, min_type, max_type, gpfn, gmfn) )
+ FSH_LOG("shadow_promote gpfn=%p gmfn=%p nt=%p", gpfn, gmfn, new_type);
+
+ if ( !shadow_remove_all_write_access(d, gpfn, gmfn) )
return 0;
// To convert this page to use as a page table, the writable count
@@ -1737,114 +1727,192 @@
return 0;
}
+#define GPFN_TO_GPTEPAGE(_gpfn) ((_gpfn) / (PAGE_SIZE / sizeof(l1_pgentry_t)))
+static inline unsigned long
+predict_writable_pte_page(struct domain *d, unsigned long gpfn)
+{
+ return __shadow_status(d, GPFN_TO_GPTEPAGE(gpfn), PGT_writable_pred);
+}
+
+static inline void
+increase_writable_pte_prediction(struct domain *d, unsigned long gpfn,
unsigned long prediction)
+{
+ unsigned long score = prediction & PGT_score_mask;
+ int create = (score == 0);
+
+ // saturating addition
+ score = (score + (1u << PGT_score_shift)) & PGT_score_mask;
+ score = score ? score : PGT_score_mask;
+
+ prediction = (prediction & PGT_mfn_mask) | score;
+
+ //printk("increase gpfn=%p pred=%p create=%d\n", gpfn, prediction, create);
+ set_shadow_status(d, GPFN_TO_GPTEPAGE(gpfn), 0, prediction,
PGT_writable_pred);
+
+ if ( create )
+ perfc_incr(writable_pte_predictions);
+}
+
+static inline void
+decrease_writable_pte_prediction(struct domain *d, unsigned long gpfn,
unsigned long prediction)
+{
+ unsigned long score = prediction & PGT_score_mask;
+ ASSERT(score);
+
+ // divide score by 2... We don't like bad predictions.
+ //
+ score = (score >> 1) & PGT_score_mask;
+
+ prediction = (prediction & PGT_mfn_mask) | score;
+
+ //printk("decrease gpfn=%p pred=%p score=%p\n", gpfn, prediction, score);
+
+ if ( score )
+ set_shadow_status(d, GPFN_TO_GPTEPAGE(gpfn), 0, prediction,
PGT_writable_pred);
+ else
+ {
+ delete_shadow_status(d, GPFN_TO_GPTEPAGE(gpfn), 0, PGT_writable_pred);
+ perfc_decr(writable_pte_predictions);
+ }
+}
+
static u32 remove_all_write_access_in_ptpage(
- struct domain *d, unsigned long pt_mfn, unsigned long readonly_mfn)
+ struct domain *d, unsigned long pt_pfn, unsigned long pt_mfn,
+ unsigned long readonly_gpfn, unsigned long readonly_gmfn,
+ u32 max_refs_to_find, unsigned long prediction)
{
unsigned long *pt = map_domain_mem(pt_mfn << PAGE_SHIFT);
unsigned long match =
- (readonly_mfn << PAGE_SHIFT) | _PAGE_RW | _PAGE_PRESENT;
+ (readonly_gmfn << PAGE_SHIFT) | _PAGE_RW | _PAGE_PRESENT;
unsigned long mask = PAGE_MASK | _PAGE_RW | _PAGE_PRESENT;
int i;
- u32 count = 0;
+ u32 found = 0;
int is_l1_shadow =
((frame_table[pt_mfn].u.inuse.type_info & PGT_type_mask) ==
PGT_l1_shadow);
- for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
+#define MATCH_ENTRY(_i) (((pt[_i] ^ match) & mask) == 0)
+
+ // returns true if all refs have been found and fixed.
+ //
+ int fix_entry(int i)
{
- if ( unlikely(((pt[i] ^ match) & mask) == 0) )
- {
- unsigned long old = pt[i];
- unsigned long new = old & ~_PAGE_RW;
+ unsigned long old = pt[i];
+ unsigned long new = old & ~_PAGE_RW;
- if ( is_l1_shadow &&
- !shadow_get_page_from_l1e(mk_l1_pgentry(new), d) )
- BUG();
+ if ( is_l1_shadow && !shadow_get_page_from_l1e(mk_l1_pgentry(new), d) )
+ BUG();
+ found++;
+ pt[i] = new;
+ if ( is_l1_shadow )
+ put_page_from_l1e(mk_l1_pgentry(old), d);
- count++;
- pt[i] = new;
+#if 0
+ printk("removed write access to pfn=%p mfn=%p in smfn=%p entry %x "
+ "is_l1_shadow=%d\n",
+ readonly_gpfn, readonly_gmfn, pt_mfn, i, is_l1_shadow);
+#endif
- if ( is_l1_shadow )
- put_page_from_l1e(mk_l1_pgentry(old), d);
+ return (found == max_refs_to_find);
+ }
- FSH_LOG("removed write access to mfn=%p in smfn=%p entry %x "
- "is_l1_shadow=%d",
- readonly_mfn, pt_mfn, i, is_l1_shadow);
- }
+ if ( MATCH_ENTRY(readonly_gpfn & (L1_PAGETABLE_ENTRIES - 1)) &&
+ fix_entry(readonly_gpfn & (L1_PAGETABLE_ENTRIES - 1)) )
+ {
+ perfc_incrc(remove_write_fast_exit);
+ increase_writable_pte_prediction(d, readonly_gpfn, prediction);
+ unmap_domain_mem(pt);
+ return found;
+ }
+
+ for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
+ {
+ if ( unlikely(MATCH_ENTRY(i)) && fix_entry(i) )
+ break;
}
unmap_domain_mem(pt);
- return count;
+ return found;
+#undef MATCH_ENTRY
}
int shadow_remove_all_write_access(
- struct domain *d, unsigned min_type, unsigned max_type,
- unsigned long gpfn, unsigned long gmfn)
+ struct domain *d, unsigned long readonly_gpfn, unsigned long readonly_gmfn)
{
int i;
struct shadow_status *a;
- unsigned long sl1mfn = __shadow_status(d, gpfn, PGT_l1_shadow);
- u32 count = 0;
- u32 write_refs;
+ u32 found = 0, fixups, write_refs;
+ unsigned long prediction, predicted_gpfn, predicted_smfn;
ASSERT(spin_is_locked(&d->arch.shadow_lock));
- ASSERT(gmfn);
+ ASSERT(VALID_MFN(readonly_gmfn));
perfc_incrc(remove_write_access);
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home