[Xen-devel] [PATCH] 7/7 xen: Add basic NUMA support - Trace support

[Ryan Harper <ryanh@xxxxxxxxxx>]

This patch adds a new trace class, TRC_NUMA, and several NUMA events.
This patch is useful for watching for NUMA memory allocations and whether
certain operations (domain creation, network traffic) result in NUMA hit
or misses when allocating memory in the hypervisor.

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@xxxxxxxxxx


diffstat output:
 tools/xentrace/formats     |    6 ++++++
 xen/common/page_alloc.c    |   22 ++++++++++++++--------
 xen/include/public/trace.h |    7 +++++++
 3 files changed, 27 insertions(+), 8 deletions(-)

Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
Signed-off-by: Ryan Grimm <grimm@xxxxxxxxxx>
---
diff -r cec10c1bb620 tools/xentrace/formats
--- a/tools/xentrace/formats    Wed Dec 14 21:47:32 2005
+++ b/tools/xentrace/formats    Wed Dec 14 15:48:27 2005
@@ -19,3 +19,9 @@
 0x00090001      CPU%(cpu)d      %(tsc)d         VMENTRY                 
0x%(1)08x 0x%(2)08x 0x%(3)08x 0x%(4)08x 0x%(5)08x
 0x00090002      CPU%(cpu)d      %(tsc)d         VMEXIT                  
0x%(1)08x 0x%(2)08x 0x%(3)08x 
 
+0x000cf001     CPU%(cpu)d      %(tsc)d NUMA_HIT         on CPU%(1)d, %(2)d 
pages in chunk 0x%(3)08x-0x%(4)08x
+0x000cf002     CPU%(cpu)d      %(tsc)d NUMA_MISS on CPU%(1)d, request 
0x%(2)x-0x%(3)x not in cpu range 0x%(4)08x-0x%(5)08x)
+0x000cf003     CPU%(cpu)d      %(tsc)d NUMA_COUNT_HIT  DOM%(1)d CPU%(2)d hits  
=%(3)d 
+0x000cf004     CPU%(cpu)d      %(tsc)d NUMA_COUNT_MISS DOM%(1)d CPU%(2)d 
misses=%(3)d 
+0x000cf005     CPU%(cpu)d      %(tsc)d NUMA_NONLOCAL_ALLOC 
+
diff -r cec10c1bb620 xen/common/page_alloc.c
--- a/xen/common/page_alloc.c   Wed Dec 14 21:47:32 2005
+++ b/xen/common/page_alloc.c   Wed Dec 14 15:48:27 2005
@@ -36,6 +36,7 @@
 #include <asm/page.h>
 #ifdef CONFIG_NUMA
 #include <xen/numa.h>
+#include <xen/trace.h>
 #endif
 
 #define MEMZONE_XEN 0
@@ -333,18 +334,15 @@
                c =  &node_memory_chunk[i];
                if ((c->nid == nid) && (c->start_paddr <= start) && 
                      (end <= c->end_paddr)) {
-                  DPRINTK("NUMA hit: (%"PRIx64",%"PRIx64") is in "
-                          "chunk(%"PRIx64",%"PRIx64")\n",
-                          start,end,c->start_paddr,c->end_paddr);
+                  TRACE_4D(TRC_NUMA_HIT, 
+                           cpu, list_order, c->start_paddr, c->end_paddr);
                   goto out;
                }
            }
        }
+       TRACE_5D(TRC_NUMA_MISS, cpu, start, end, cpu_start, cpu_end);
     }
     /* make sure if none are found we return NULL */
-    DPRINTK("NUMA miss: (%"PRIx64",%"PRIx64") not in CPU%d chunk "
-            "max range(%"PRIx64",%"PRIx64")\n", 
-            start, end, cpu, cpu_start, cpu_end);
     pg = NULL;
 
  out:
@@ -428,12 +426,20 @@
             if ( num_memory_chunks > 1 ) {
                 pg = __find_local_page(&heap[zone][i], i, cpu);
                 /* increment counters based on pg */
-                (pg) ?  numa_hit[zone]++ : numa_miss[zone]++;
-                (pg) ?  d->numa_hit++ : d->numa_miss++;
+                if (pg) {
+                    numa_hit[zone]++;
+                    d->numa_hit++;
+                    TRACE_3D(TRC_NUMA_COUNT_HIT, d->domain_id, cpu, 
d->numa_hit);
+                } else {
+                    numa_miss[zone]++;
+                    d->numa_miss++;
+                    TRACE_3D(TRC_NUMA_COUNT_MISS, d->domain_id, cpu, 
d->numa_miss);
+                }
             }
             if ( pg == NULL ) {
                 /* No local pages, just take the first */
                 pg = list_entry(heap[zone][i].next, struct pfn_info, list);
+                TRACE_0D(TRC_NUMA_NONLOCAL_ALLOC);
             }
             if ( pg != NULL) {
                 d->page_alloc++;
diff -r cec10c1bb620 xen/include/public/trace.h
--- a/xen/include/public/trace.h        Wed Dec 14 21:47:32 2005
+++ b/xen/include/public/trace.h        Wed Dec 14 15:48:27 2005
@@ -15,6 +15,7 @@
 #define TRC_DOM0OP  0x0004f000    /* Xen DOM0 operation trace */
 #define TRC_VMX     0x0008f000    /* Xen VMX trace            */
 #define TRC_MEM     0x000af000    /* Xen memory trace         */
+#define TRC_NUMA    0x000cf000    /* Xen NUMA trace           */
 #define TRC_ALL     0xfffff000
 
 /* Trace subclasses */
@@ -47,6 +48,12 @@
 #define TRC_MEM_PAGE_GRANT_MAP      (TRC_MEM + 1)
 #define TRC_MEM_PAGE_GRANT_UNMAP    (TRC_MEM + 2)
 #define TRC_MEM_PAGE_GRANT_TRANSFER (TRC_MEM + 3)
+
+#define TRC_NUMA_HIT            (TRC_NUMA + 1)
+#define TRC_NUMA_MISS           (TRC_NUMA + 2)
+#define TRC_NUMA_COUNT_HIT      (TRC_NUMA + 3)
+#define TRC_NUMA_COUNT_MISS     (TRC_NUMA + 4)
+#define TRC_NUMA_NONLOCAL_ALLOC (TRC_NUMA + 5)
 
 /* trace events per subclass */
 #define TRC_VMX_VMEXIT          (TRC_VMXEXIT + 1)

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel