[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] linux: allocate and free cold pages



To reduce the performance side effects of ballooning, use and return
cold pages. To limit the impact scrubbing of these (and other) pages
has on the cache, also implement a dedicated scrubbing function on x86
which uses non-temporal stores (when available).

As usual, written and tested on 2.6.24 and made apply to the 2.6.18
tree without further testing.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>

Index: head-2008-01-28/arch/i386/lib/Makefile
===================================================================
--- head-2008-01-28.orig/arch/i386/lib/Makefile 2008-01-24 23:58:37.000000000 
+0100
+++ head-2008-01-28/arch/i386/lib/Makefile      2008-02-08 10:46:49.000000000 
+0100
@@ -7,3 +7,4 @@ include ${srctree}/arch/x86/lib/Makefile
        bitops.o
 
 lib-$(CONFIG_X86_USE_3DNOW) += mmx.o
+lib-$(CONFIG_XEN_SCRUB_PAGES) += scrub.o
Index: head-2008-01-28/arch/i386/lib/scrub.c
===================================================================
--- /dev/null   1970-01-01 00:00:00.000000000 +0000
+++ head-2008-01-28/arch/i386/lib/scrub.c       2008-02-08 12:30:51.000000000 
+0100
@@ -0,0 +1,21 @@
+#include <asm/cpufeature.h>
+#include <asm/page.h>
+#include <asm/processor.h>
+
+void scrub_pages(void *v, unsigned int count)
+{
+       if (likely(cpu_has_xmm2)) {
+               unsigned long n = count * (PAGE_SIZE / sizeof(long) / 4);
+
+               for (; n--; v += sizeof(long) * 4)
+                       asm("movnti %1,(%0)\n\t"
+                           "movnti %1,%c2(%0)\n\t"
+                           "movnti %1,2*%c2(%0)\n\t"
+                           "movnti %1,3*%c2(%0)\n\t"
+                           : : "r" (v), "r" (0L), "i" (sizeof(long))
+                           : "memory");
+               asm volatile("sfence" : : : "memory");
+       } else
+               for (; count--; v += PAGE_SIZE)
+                       clear_page(v);
+}
Index: head-2008-01-28/arch/i386/mm/hypervisor.c
===================================================================
--- head-2008-01-28.orig/arch/i386/mm/hypervisor.c      2008-02-08 
14:29:22.000000000 +0100
+++ head-2008-01-28/arch/i386/mm/hypervisor.c   2008-02-08 11:30:34.000000000 
+0100
@@ -281,7 +281,7 @@ int xen_create_contiguous_region(
        set_xen_guest_handle(exchange.in.extent_start, in_frames);
        set_xen_guest_handle(exchange.out.extent_start, &out_frame);
 
-       scrub_pages(vstart, 1 << order);
+       scrub_pages((void *)vstart, 1 << order);
 
        balloon_lock(flags);
 
@@ -374,7 +374,7 @@ void xen_destroy_contiguous_region(unsig
        set_xen_guest_handle(exchange.in.extent_start, &in_frame);
        set_xen_guest_handle(exchange.out.extent_start, out_frames);
 
-       scrub_pages(vstart, 1 << order);
+       scrub_pages((void *)vstart, 1 << order);
 
        balloon_lock(flags);
 
Index: head-2008-01-28/arch/x86_64/lib/Makefile
===================================================================
--- head-2008-01-28.orig/arch/x86_64/lib/Makefile       2008-01-24 
23:58:37.000000000 +0100
+++ head-2008-01-28/arch/x86_64/lib/Makefile    2008-02-08 10:46:49.000000000 
+0100
@@ -10,3 +10,4 @@ include ${srctree}/arch/x86/lib/Makefile
        usercopy.o getuser.o putuser.o  \
        thunk.o clear_page.o copy_page.o bitstr.o bitops.o
 lib-y += memcpy.o memmove.o memset.o copy_user.o
+lib-$(CONFIG_XEN_SCRUB_PAGES) += scrub.o
Index: head-2008-01-28/arch/x86_64/lib/scrub.c
===================================================================
--- /dev/null   1970-01-01 00:00:00.000000000 +0000
+++ head-2008-01-28/arch/x86_64/lib/scrub.c     2008-02-08 12:30:51.000000000 
+0100
@@ -0,0 +1,1 @@
+#include "../../i386/lib/scrub.c"
Index: head-2008-01-28/drivers/xen/balloon/balloon.c
===================================================================
--- head-2008-01-28.orig/drivers/xen/balloon/balloon.c  2008-02-08 
14:29:22.000000000 +0100
+++ head-2008-01-28/drivers/xen/balloon/balloon.c       2008-02-08 
14:29:36.000000000 +0100
@@ -105,7 +105,7 @@ static struct timer_list balloon_timer;
 /* When ballooning out (allocating memory to return to Xen) we don't really 
    want the kernel to try too hard since that can trigger the oom killer. */
 #define GFP_BALLOON \
-       (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
+       (GFP_HIGHUSER|__GFP_NOWARN|__GFP_NORETRY|__GFP_NOMEMALLOC|__GFP_COLD)
 
 #define PAGE_TO_LIST(p) (&(p)->lru)
 #define LIST_TO_PAGE(l) list_entry((l), struct page, lru)
@@ -175,6 +175,17 @@ static struct page *balloon_next_page(st
        return LIST_TO_PAGE(next);
 }
 
+static inline void balloon_free_page(struct page *page)
+{
+#ifndef MODULE
+       if (put_page_testzero(page))
+               free_cold_page(page);
+#else
+       /* free_cold_page() is not being exported. */
+       __free_page(page);
+#endif
+}
+
 static void balloon_alarm(unsigned long unused)
 {
        schedule_work(&balloon_worker);
@@ -292,7 +303,7 @@ static int increase_reservation(unsigned
                /* Relinquish the page back to the allocator. */
                ClearPageReserved(page);
                init_page_count(page);
-               __free_page(page);
+               balloon_free_page(page);
        }
 
        bs.current_pages += nr_pages;
@@ -623,7 +634,8 @@ static int dealloc_pte_fn(
 
 struct page **alloc_empty_pages_and_pagevec(int nr_pages)
 {
-       unsigned long vaddr, flags;
+       unsigned long flags;
+       void *v;
        struct page *page, **pagevec;
        int i, ret;
 
@@ -632,13 +644,12 @@ struct page **alloc_empty_pages_and_page
                return NULL;
 
        for (i = 0; i < nr_pages; i++) {
-               page = pagevec[i] = alloc_page(GFP_KERNEL);
+               page = pagevec[i] = alloc_page(GFP_KERNEL|__GFP_COLD);
                if (page == NULL)
                        goto err;
 
-               vaddr = (unsigned long)page_address(page);
-
-               scrub_pages(vaddr, 1);
+               v = page_address(page);
+               scrub_pages(v, 1);
 
                balloon_lock(flags);
 
@@ -656,8 +667,9 @@ struct page **alloc_empty_pages_and_page
                                ret = 0; /* success */
                } else {
 #ifdef CONFIG_XEN
-                       ret = apply_to_page_range(&init_mm, vaddr, PAGE_SIZE,
-                                                 dealloc_pte_fn, NULL);
+                       ret = apply_to_page_range(&init_mm, (unsigned long)v,
+                                                 PAGE_SIZE, dealloc_pte_fn,
+                                                 NULL);
 #else
                        /* Cannot handle non-auto translate mode. */
                        ret = 1;
@@ -668,7 +679,7 @@ struct page **alloc_empty_pages_and_page
 
                if (ret != 0) {
                        balloon_unlock(flags);
-                       __free_page(page);
+                       balloon_free_page(page);
                        goto err;
                }
 
Index: head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h
===================================================================
--- head-2008-01-28.orig/include/asm-i386/mach-xen/asm/hypervisor.h     
2008-02-08 14:29:22.000000000 +0100
+++ head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h  2008-02-08 
10:42:13.000000000 +0100
@@ -121,7 +121,7 @@ int xen_limit_pages_to_max_mfn(
 u64 jiffies_to_st(unsigned long jiffies);
 
 #ifdef CONFIG_XEN_SCRUB_PAGES
-#define scrub_pages(_p,_n) memset((void *)(_p), 0, (_n) << PAGE_SHIFT)
+void scrub_pages(void *, unsigned int);
 #else
 #define scrub_pages(_p,_n) ((void)0)
 #endif



_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.