[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] don't schedule unplugged vcpus



This patch extends the CONFIG_HOTPLUG_CPU behavior down into the
hypervisor.  Currently when a CPU in Linux is moved offline,

echo 0 > /sys/devices/system/cpu/cpuX/online

the offline cpu yields its slice back to the hypervisor.  This patch
adds two SCHEDOPS (vcpu_down/vcpu_up) which set/clear a new VCPU flag,
VCPU_down.  The domain_runnable() check now looks at this flag and
subsequently the vcpu is not scheduled when VCPU_down is set.

The patch was built and tested against 20050606 nightly snapshot.
Testing requires DOMU with CONFIG_SMP and CONFIG_HOTPLUG_CPU.  Please
apply.

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@xxxxxxxxxx


diffstat output:
 linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c       |    7 +
 linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c       |    4 
 linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h |   31 +++++++
 xen/common/schedule.c                                        |   48 +++++++++++
 xen/include/public/xen.h                                     |    3 
 xen/include/xen/sched.h                                      |    5 -
 6 files changed, 96 insertions(+), 2 deletions(-)

Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
---
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 
c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c    2005-06-05 
22:09:07.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c    2005-06-06 
15:15:34.184156730 -0500
@@ -154,8 +154,13 @@
                                cpu_clear(cpu, cpu_idle_map);
                        rmb();
 
-                       if (cpu_is_offline(cpu))
+                       if (cpu_is_offline(cpu)) {
+#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU)
+            /* Tell hypervisor not to schedule dead vcpus */
+            HYPERVISOR_vcpu_down(cpu);
+#endif
                                play_dead();
+         }
 
                        irq_stat[cpu].idle_timestamp = jiffies;
                        xen_idle();
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 
c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c    2005-06-05 
22:09:13.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c    2005-06-06 
15:44:15.988564794 -0500
@@ -1380,6 +1380,10 @@
        }
 
 #ifdef CONFIG_HOTPLUG_CPU
+#ifdef CONFIG_XEN
+   /* Tell hypervisor to bring vcpu up */
+   HYPERVISOR_vcpu_up(cpu);
+#endif
        /* Already up, and in cpu_quiescent now? */
        if (cpu_isset(cpu, smp_commenced_mask)) {
                cpu_enable(cpu);
diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 
c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h
--- b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h      
2005-06-05 22:09:15.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h      
2005-06-06 14:18:17.414798645 -0500
@@ -517,4 +517,35 @@
     return ret;
 }
 
+static inline int
+HYPERVISOR_vcpu_down(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+       : "0" (__HYPERVISOR_sched_op),
+         "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
+
+static inline int
+HYPERVISOR_vcpu_up(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+       : "0" (__HYPERVISOR_sched_op),
+         "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
 #endif /* __HYPERCALL_H__ */
diff -urN b/xen/common/schedule.c c/xen/common/schedule.c
--- b/xen/common/schedule.c     2005-06-05 22:09:14.000000000 -0500
+++ c/xen/common/schedule.c     2005-06-06 15:50:35.549218230 -0500
@@ -261,6 +261,44 @@
     return 0;
 }
 
+/* Mark target vcpu as non-runnable so it is not scheduled */
+static long do_vcpu_down(int vcpu)
+{
+    struct vcpu *target;
+    
+    if (vcpu > MAX_VIRT_CPUS)
+        return -EINVAL;
+
+    target = current->domain->vcpu[vcpu];
+    /* DEBUG
+     * printk("DOM%d VCPU%d going down\n",
+     *     target->domain->domain_id, target->vcpu_id);
+     */
+    set_bit(_VCPUF_down, &target->vcpu_flags);
+
+    return 0;
+}
+
+/* Mark target vcpu as runnable and wake it */
+static long do_vcpu_up(int vcpu)
+{
+    struct vcpu *target;
+   
+    if (vcpu > MAX_VIRT_CPUS)
+        return -EINVAL;
+
+    target = current->domain->vcpu[vcpu];
+    /* DEBUG
+     * printk("DOM%d VCPU%d coming up\n", 
+     *     target->domain->domain_id, target->vcpu_id);
+     */
+    clear_bit(_VCPUF_down, &target->vcpu_flags);
+    /* wake vcpu */
+    domain_wake(target);
+
+    return 0;
+}
+
 /*
  * Demultiplex scheduler-related hypercalls.
  */
@@ -290,6 +328,16 @@
         domain_shutdown((u8)(op >> SCHEDOP_reasonshift));
         break;
     }
+    case SCHEDOP_vcpu_down:
+    {
+        ret = do_vcpu_down((int)(op >> SCHEDOP_vcpushift));
+        break;
+    }
+    case SCHEDOP_vcpu_up:
+    {
+        ret = do_vcpu_up((int)(op >> SCHEDOP_vcpushift));
+        break;
+    }
 
     default:
         ret = -ENOSYS;
diff -urN b/xen/include/public/xen.h c/xen/include/public/xen.h
--- b/xen/include/public/xen.h  2005-06-05 22:09:13.000000000 -0500
+++ c/xen/include/public/xen.h  2005-06-06 14:18:17.427796825 -0500
@@ -200,8 +200,11 @@
 #define SCHEDOP_yield           0   /* Give up the CPU voluntarily.       */
 #define SCHEDOP_block           1   /* Block until an event is received.  */
 #define SCHEDOP_shutdown        2   /* Stop executing this domain.        */
+#define SCHEDOP_vcpu_down       3   /* make target VCPU not-runnable.     */
+#define SCHEDOP_vcpu_up         4   /* make target VCPU runnable.         */
 #define SCHEDOP_cmdmask       255   /* 8-bit command. */
 #define SCHEDOP_reasonshift     8   /* 8-bit reason code. (SCHEDOP_shutdown) */
+#define SCHEDOP_vcpushift       8   /* 8-bit VCPU target. (SCHEDOP_up|down) */
 
 /*
  * Reason codes for SCHEDOP_shutdown. These may be interpreted by control 
diff -urN b/xen/include/xen/sched.h c/xen/include/xen/sched.h
--- b/xen/include/xen/sched.h   2005-06-05 22:09:08.000000000 -0500
+++ c/xen/include/xen/sched.h   2005-06-06 14:18:17.000000000 -0500
@@ -346,6 +346,9 @@
  /* Initialization completed. */
 #define _VCPUF_initialised     8
 #define VCPUF_initialised      (1UL<<_VCPUF_initialised)
+ /* VCPU is not-runnable */
+#define _VCPUF_down            9
+#define VCPUF_down             (1UL<<_VCPUF_down)
 
 /*
  * Per-domain flags (domain_flags).
@@ -375,7 +378,7 @@
 static inline int domain_runnable(struct vcpu *v)
 {
     return ( (atomic_read(&v->pausecnt) == 0) &&
-             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) &&
+             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) &&
              !(v->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) );
 }
 

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.