ChangeSet 1.1387, 2005/03/29 15:35:22+01:00, kaf24@xxxxxxxxxxxxxxxxxxxx
Remove per-cpu batch queues for mmu updates and multicalls. Instead
batching is done locally within functions that most benefit.
Signed-off-by: Keir Fraser <keir@xxxxxxxxxxxxx>
b/linux-2.4.29-xen-sparse/arch/xen/kernel/ldt.c | 8
b/linux-2.4.29-xen-sparse/arch/xen/kernel/process.c | 55 +
b/linux-2.4.29-xen-sparse/arch/xen/kernel/setup.c | 4
b/linux-2.4.29-xen-sparse/arch/xen/mm/fault.c | 1
b/linux-2.4.29-xen-sparse/include/asm-xen/desc.h | 5
b/linux-2.4.29-xen-sparse/include/asm-xen/mmu_context.h | 47 -
b/linux-2.4.29-xen-sparse/mkbuildtree | 1
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/cpu/common.c | 2
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/ldt.c | 8
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c | 12
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c | 48 -
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/setup.c | 3
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 1
b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/traps.c | 1
b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c | 375
+---------
b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/init.c | 1
b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c | 36
b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/desc.h | 4
b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/mmu_context.h | 1
b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgalloc.h | 4
b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h | 1
b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/processor.h | 2
b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h | 20
linux-2.6.11-xen-sparse/include/asm-xen/multicall.h | 115 ---
24 files changed, 158 insertions(+), 597 deletions(-)
diff -Nru a/linux-2.4.29-xen-sparse/arch/xen/kernel/ldt.c
b/linux-2.4.29-xen-sparse/arch/xen/kernel/ldt.c
--- a/linux-2.4.29-xen-sparse/arch/xen/kernel/ldt.c 2005-03-29 10:02:22
-05:00
+++ b/linux-2.4.29-xen-sparse/arch/xen/kernel/ldt.c 2005-03-29 10:02:22
-05:00
@@ -14,6 +14,7 @@
#include <linux/vmalloc.h>
#include <linux/slab.h>
+#include <asm/mmu_context.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <asm/ldt.h>
@@ -58,7 +59,6 @@
pc->ldt,
(pc->size*LDT_ENTRY_SIZE)/PAGE_SIZE);
load_LDT(pc);
- flush_page_update_queue();
#ifdef CONFIG_SMP
if (current->mm->cpu_vm_mask != (1<<smp_processor_id()))
smp_call_function(flush_ldt, 0, 1, 1);
@@ -66,6 +66,8 @@
}
wmb();
if (oldsize) {
+ make_pages_writable(
+ oldldt, (oldsize*LDT_ENTRY_SIZE)/PAGE_SIZE);
if (oldsize*LDT_ENTRY_SIZE > PAGE_SIZE)
vfree(oldldt);
else
@@ -84,7 +86,6 @@
}
memcpy(new->ldt, old->ldt, old->size*LDT_ENTRY_SIZE);
make_pages_readonly(new->ldt, (new->size*LDT_ENTRY_SIZE)/PAGE_SIZE);
- flush_page_update_queue();
return 0;
}
@@ -116,10 +117,11 @@
void destroy_context(struct mm_struct *mm)
{
if (mm->context.size) {
+ if (mm_state_sync & STATE_SYNC_LDT)
+ clear_LDT();
make_pages_writable(
mm->context.ldt,
(mm->context.size*LDT_ENTRY_SIZE)/PAGE_SIZE);
- flush_page_update_queue();
if (mm->context.size*LDT_ENTRY_SIZE > PAGE_SIZE)
vfree(mm->context.ldt);
else
diff -Nru a/linux-2.4.29-xen-sparse/arch/xen/kernel/process.c
b/linux-2.4.29-xen-sparse/arch/xen/kernel/process.c
--- a/linux-2.4.29-xen-sparse/arch/xen/kernel/process.c 2005-03-29 10:02:22
-05:00
+++ b/linux-2.4.29-xen-sparse/arch/xen/kernel/process.c 2005-03-29 10:02:22
-05:00
@@ -43,7 +43,6 @@
#include <asm/i387.h>
#include <asm/desc.h>
#include <asm/mmu_context.h>
-#include <asm/multicall.h>
#include <asm-xen/xen-public/physdev.h>
#include <linux/irq.h>
@@ -305,19 +304,36 @@
{
struct thread_struct *next = &next_p->thread;
physdev_op_t op;
+ multicall_entry_t _mcl[8], *mcl = _mcl;
+ mmu_update_t _mmu[2], *mmu = _mmu;
- __cli();
+ if ( mm_state_sync & STATE_SYNC_PT )
+ {
+ mmu->ptr = virt_to_machine(cur_pgd) | MMU_EXTENDED_COMMAND;
+ mmu->val = MMUEXT_NEW_BASEPTR;
+ mmu++;
+ }
- /*
- * We clobber FS and GS here so that we avoid a GPF when restoring previous
- * task's FS/GS values in Xen when the LDT is switched. If we don't do this
- * then we can end up erroneously re-flushing the page-update queue when
- * we 'execute_multicall_list'.
- */
- __asm__ __volatile__ (
- "xorl %%eax,%%eax; movl %%eax,%%fs; movl %%eax,%%gs" : : : "eax" );
+ if ( mm_state_sync & STATE_SYNC_LDT )
+ {
+ __asm__ __volatile__ (
+ "xorl %%eax,%%eax; movl %%eax,%%fs; movl %%eax,%%gs" : : : "eax" );
+ mmu->ptr = (unsigned long)next_p->mm->context.ldt |
+ MMU_EXTENDED_COMMAND;
+ mmu->val = (next_p->mm->context.size << MMUEXT_CMD_SHIFT) |
+ MMUEXT_SET_LDT;
+ mmu++;
+ }
- MULTICALL_flush_page_update_queue();
+ if ( mm_state_sync != 0 )
+ {
+ mcl->op = __HYPERVISOR_mmu_update;
+ mcl->args[0] = (unsigned long)_mmu;
+ mcl->args[1] = mmu - _mmu;
+ mcl->args[2] = 0;
+ mcl++;
+ mm_state_sync = 0;
+ }
/*
* This is basically 'unlazy_fpu', except that we queue a multicall to
@@ -332,21 +348,26 @@
asm volatile( "fnsave %0 ; fwait"
: "=m" (prev_p->thread.i387.fsave) );
prev_p->flags &= ~PF_USEDFPU;
- queue_multicall1(__HYPERVISOR_fpu_taskswitch, 1);
+ mcl->op = __HYPERVISOR_fpu_taskswitch;
+ mcl->args[0] = 1;
+ mcl++;
}
- queue_multicall2(__HYPERVISOR_stack_switch, __KERNEL_DS, next->esp0);
+ mcl->op = __HYPERVISOR_stack_switch;
+ mcl->args[0] = __KERNEL_DS;
+ mcl->args[1] = next->esp0;
+ mcl++;
if ( prev_p->thread.io_pl != next->io_pl )
{
op.cmd = PHYSDEVOP_SET_IOPL;
op.u.set_iopl.iopl = next->io_pl;
- queue_multicall1(__HYPERVISOR_physdev_op, (unsigned long)&op);
+ mcl->op = __HYPERVISOR_physdev_op;
+ mcl->args[0] = (unsigned long)&op;
+ mcl++;
}
- /* EXECUTE ALL TASK SWITCH XEN SYSCALLS AT THIS POINT. */
- execute_multicall_list();
- __sti();
+ (void)HYPERVISOR_multicall(_mcl, mcl - _mcl);
/*
* Restore %fs and %gs.
diff -Nru a/linux-2.4.29-xen-sparse/arch/xen/kernel/setup.c
b/linux-2.4.29-xen-sparse/arch/xen/kernel/setup.c
--- a/linux-2.4.29-xen-sparse/arch/xen/kernel/setup.c 2005-03-29 10:02:22
-05:00
+++ b/linux-2.4.29-xen-sparse/arch/xen/kernel/setup.c 2005-03-29 10:02:22
-05:00
@@ -62,9 +62,6 @@
unsigned int *phys_to_machine_mapping, *pfn_to_mfn_frame_list;
-DEFINE_PER_CPU(multicall_entry_t, multicall_list[8]);
-DEFINE_PER_CPU(int, nr_multicall_ents);
-
/*
* Machine setup..
*/
@@ -1206,7 +1203,6 @@
HYPERVISOR_stack_switch(__KERNEL_DS, current->thread.esp0);
load_LDT(&init_mm.context);
- flush_page_update_queue();
/* Force FPU initialization. */
current->flags &= ~PF_USEDFPU;
diff -Nru a/linux-2.4.29-xen-sparse/arch/xen/mm/fault.c
b/linux-2.4.29-xen-sparse/arch/xen/mm/fault.c
--- a/linux-2.4.29-xen-sparse/arch/xen/mm/fault.c 2005-03-29 10:02:22
-05:00
+++ b/linux-2.4.29-xen-sparse/arch/xen/mm/fault.c 2005-03-29 10:02:22
-05:00
@@ -28,6 +28,7 @@
extern void die(const char *,struct pt_regs *,long);
pgd_t *cur_pgd;
+int mm_state_sync;
extern spinlock_t timerlist_lock;
diff -Nru a/linux-2.4.29-xen-sparse/include/asm-xen/desc.h
b/linux-2.4.29-xen-sparse/include/asm-xen/desc.h
--- a/linux-2.4.29-xen-sparse/include/asm-xen/desc.h 2005-03-29 10:02:22
-05:00
+++ b/linux-2.4.29-xen-sparse/include/asm-xen/desc.h 2005-03-29 10:02:22
-05:00
@@ -16,6 +16,11 @@
extern struct desc_struct default_ldt[];
+static inline void clear_LDT(void)
+{
+ xen_set_ldt(0, 0);
+}
+
static inline void load_LDT(mm_context_t *pc)
{
void *segments = pc->ldt;
diff -Nru a/linux-2.4.29-xen-sparse/include/asm-xen/mmu_context.h
b/linux-2.4.29-xen-sparse/include/asm-xen/mmu_context.h
--- a/linux-2.4.29-xen-sparse/include/asm-xen/mmu_context.h 2005-03-29
10:02:22 -05:00
+++ b/linux-2.4.29-xen-sparse/include/asm-xen/mmu_context.h 2005-03-29
10:02:22 -05:00
@@ -28,47 +28,34 @@
#endif
extern pgd_t *cur_pgd;
+extern int mm_state_sync;
+#define STATE_SYNC_PT 1
+#define STATE_SYNC_LDT 2
static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk, unsigned cpu)
{
if (prev != next) {
/* stop flush ipis for the previous mm */
clear_bit(cpu, &prev->cpu_vm_mask);
-#ifdef CONFIG_SMP
- cpu_tlbstate[cpu].state = TLBSTATE_OK;
- cpu_tlbstate[cpu].active_mm = next;
-#endif
-
/* Re-load page tables */
cur_pgd = next->pgd;
- queue_pt_switch(__pa(cur_pgd));
- /* load_LDT, if either the previous or next thread
- * has a non-default LDT.
- */
- if (next->context.size+prev->context.size)
- load_LDT(&next->context);
- }
-#ifdef CONFIG_SMP
- else {
- cpu_tlbstate[cpu].state = TLBSTATE_OK;
- if(cpu_tlbstate[cpu].active_mm != next)
- out_of_line_bug();
- if(!test_and_set_bit(cpu, &next->cpu_vm_mask)) {
- /* We were in lazy tlb mode and leave_mm disabled
- * tlb flush IPI delivery. We must reload %cr3.
- */
- cur_pgd = next->pgd;
- queue_pt_switch(__pa(cur_pgd));
- load_LDT(next);
- }
+ mm_state_sync |= STATE_SYNC_PT;
+ /* load_LDT, if either the previous or next thread
+ * has a non-default LDT.
+ */
+ if (next->context.size+prev->context.size)
+ mm_state_sync |= STATE_SYNC_LDT;
}
-#endif
}
-#define activate_mm(prev, next) \
-do { \
- switch_mm((prev),(next),NULL,smp_processor_id()); \
- flush_page_update_queue(); \
+#define activate_mm(prev, next) \
+do { \
+ switch_mm((prev),(next),NULL,smp_processor_id()); \
+ if (mm_state_sync & STATE_SYNC_PT) \
+ xen_pt_switch(__pa(cur_pgd)); \
+ if (mm_state_sync & STATE_SYNC_LDT) \
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home