.. to avoid using hypercalls for clearing dead (un-pinned) page tables.
This eliminates well over a quarter of a million hypercalls during
kernel builds (up to about a million depending on the configuration).
This is against 2.6.25.4, it does not apply to the 2.6.18 tree (I'd
make this effort only if the patch is desired into that tree). The
primary goal of sending this is to find out whether the required change
to include/asm-generic/pgtable.h and mm/memory.c is considered
worthwhile (in which case I'd make an attempt at getting this accepted
upstream).
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>
Index: head-2008-05-19/include/asm-generic/pgtable.h
===================================================================
--- head-2008-05-19.orig/include/asm-generic/pgtable.h 2008-05-19
10:48:15.000000000 +0200
+++ head-2008-05-19/include/asm-generic/pgtable.h 2008-05-19
10:57:25.000000000 +0200
@@ -133,6 +133,18 @@ static inline void ptep_set_wrprotect(st
#define move_pte(pte, prot, old_addr, new_addr) (pte)
#endif
+#ifndef __HAVE_ARCH_PMD_CLEAR_FULL
+#define pmd_clear_full(mm, addr, pmd, full) pmd_clear(pmd)
+#endif
+
+#ifndef __HAVE_ARCH_PUD_CLEAR_FULL
+#define pud_clear_full(mm, addr, pud, full) pud_clear(pud)
+#endif
+
+#ifndef __HAVE_ARCH_PGD_CLEAR_FULL
+#define pgd_clear_full(mm, addr, pgd, full) pgd_clear(pgd)
+#endif
+
/*
* When walking page tables, get the address of the next boundary,
* or the end address of the range if that comes earlier. Although no
Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable.h
===================================================================
--- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable.h 2008-05-19
10:57:24.000000000 +0200
+++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable.h 2008-05-19
10:57:25.000000000 +0200
@@ -397,6 +397,22 @@ static inline pte_t ptep_get_and_clear(s
pte_t xen_ptep_get_and_clear_full(struct vm_area_struct *, unsigned long,
pte_t *, int);
+#define __HAVE_ARCH_PMD_CLEAR_FULL
+#define pmd_clear_full(mm, addr, pmd, full) \
+ (!mm_is_pinned(mm) ? __xen_pmd_clear(pmd) : xen_pmd_clear(pmd))
+
+#ifndef __PAGETABLE_PMD_FOLDED
+#define __HAVE_ARCH_PUD_CLEAR_FULL
+#define pud_clear_full(mm, addr, pud, full) \
+ (!mm_is_pinned(mm) ? __xen_pud_clear(pud) : xen_pud_clear(pud))
+#endif
+
+#ifndef __PAGETABLE_PUD_FOLDED
+#define __HAVE_ARCH_PGD_CLEAR_FULL
+#define pgd_clear_full(mm, addr, pgd, full) \
+ (!mm_is_pinned(mm) ? __xen_pgd_clear(pgd) : xen_pgd_clear(pgd))
+#endif
+
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long
addr, pte_t *ptep)
{
Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-2level.h
===================================================================
--- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable-2level.h
2008-05-19 10:49:58.000000000 +0200
+++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-2level.h
2008-05-19 10:57:25.000000000 +0200
@@ -35,6 +35,11 @@ static inline void xen_pmd_clear(pmd_t *
#define __xen_pte_clear(ptep) xen_set_pte(ptep, __pte(0))
+static inline void __xen_pmd_clear(pmd_t *pmdp)
+{
+ *pmdp = __pmd(0);
+}
+
#ifdef CONFIG_SMP
static inline pte_t xen_ptep_get_and_clear(pte_t *xp, pte_t res)
{
Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-3level.h
===================================================================
--- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable-3level.h
2008-05-19 10:49:58.000000000 +0200
+++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-3level.h
2008-05-19 10:57:25.000000000 +0200
@@ -76,6 +76,11 @@ static inline void xen_pmd_clear(pmd_t *
xen_l2_entry_update(pmd, __pmd(0));
}
+static inline void __xen_pmd_clear(pmd_t *pmd)
+{
+ *pmd = __pmd(0);
+}
+
static inline void pud_clear(pud_t *pudp)
{
pgdval_t pgd;
@@ -96,6 +101,11 @@ static inline void pud_clear(pud_t *pudp
xen_tlb_flush();
}
+static inline void __xen_pud_clear(pud_t *pudp)
+{
+ *pudp = __pud(0);
+}
+
#define pud_page(pud) \
((struct page *) __va(pud_val(pud) & PAGE_MASK))
Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable_64.h
===================================================================
--- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable_64.h
2008-05-19 10:57:24.000000000 +0200
+++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable_64.h 2008-05-19
10:57:25.000000000 +0200
@@ -109,6 +109,11 @@ static inline void xen_pmd_clear(pmd_t *
xen_set_pmd(pmd, xen_make_pmd(0));
}
+static inline void __xen_pmd_clear(pmd_t *pmd)
+{
+ *pmd = xen_make_pmd(0);
+}
+
static inline void xen_set_pud(pud_t *pudp, pud_t pud)
{
xen_l3_entry_update(pudp, pud);
@@ -119,6 +124,11 @@ static inline void xen_pud_clear(pud_t *
xen_set_pud(pud, xen_make_pud(0));
}
+static inline void __xen_pud_clear(pud_t *pud)
+{
+ *pud = xen_make_pud(0);
+}
+
#define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
static inline void xen_set_pgd(pgd_t *pgdp, pgd_t pgd)
@@ -132,6 +142,12 @@ static inline void xen_pgd_clear(pgd_t *
xen_set_pgd(__user_pgd(pgd), xen_make_pgd(0));
}
+static inline void __xen_pgd_clear(pgd_t *pgd)
+{
+ *pgd = xen_make_pgd(0);
+ *__user_pgd(pgd) = xen_make_pgd(0);
+}
+
#define pte_same(a, b) ((a).pte == (b).pte)
#endif /* !__ASSEMBLY__ */
Index: head-2008-05-19/mm/memory.c
===================================================================
--- head-2008-05-19.orig/mm/memory.c 2008-05-19 10:49:46.000000000 +0200
+++ head-2008-05-19/mm/memory.c 2008-05-19 10:57:25.000000000 +0200
@@ -132,10 +132,11 @@ void pmd_clear_bad(pmd_t *pmd)
* Note: this doesn't free the actual pages themselves. That
* has been handled earlier when unmapping all the memory regions.
*/
-static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
+static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
+ unsigned long addr)
{
pgtable_t token = pmd_pgtable(*pmd);
- pmd_clear(pmd);
+ pmd_clear_full(tlb->mm, addr, pmd, tlb->fullmm);
pte_free_tlb(tlb, token);
tlb->mm->nr_ptes--;
}
@@ -154,7 +155,7 @@ static inline void free_pmd_range(struct
next = pmd_addr_end(addr, end);
if (pmd_none_or_clear_bad(pmd))
continue;
- free_pte_range(tlb, pmd);
+ free_pte_range(tlb, pmd, addr);
} while (pmd++, addr = next, addr != end);
start &= PUD_MASK;
@@ -169,7 +170,7 @@ static inline void free_pmd_range(struct
return;
pmd = pmd_offset(pud, start);
- pud_clear(pud);
+ pud_clear_full(tlb->mm, start, pud, tlb->fullmm);
pmd_free_tlb(tlb, pmd);
}
@@ -202,7 +203,7 @@ static inline void free_pud_range(struct
return;
pud = pud_offset(pgd, start);
- pgd_clear(pgd);
+ pgd_clear_full(tlb->mm, start, pgd, tlb->fullmm);
pud_free_tlb(tlb, pud);
}
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home