diff -r 4ea36cce2519 xen/common/cpupool.c
--- a/xen/common/cpupool.c	Mon Feb 14 09:10:22 2011 +0000
+++ b/xen/common/cpupool.c	Wed Feb 16 12:21:37 2011 +0000
@@ -291,49 +291,76 @@
 
     spin_lock(&cpupool_lock);
     ret = -EBUSY;
-    if ( (cpupool_moving_cpu != -1) && (cpu != cpupool_moving_cpu) )
-        goto out;
-    if ( cpu_isset(cpu, cpupool_locked_cpus) )
-        goto out;
+    if ( cpu != cpupool_moving_cpu )
+    {
+        /* Don't start a second operation until the first has completed */
+        if ( (cpupool_moving_cpu != -1) )
+            goto out;
 
-    ret = 0;
-    if ( !cpu_isset(cpu, c->cpu_valid) && (cpu != cpupool_moving_cpu) )
-        goto out;
+        /* Don't start an op on a locked cpu (?)*/
+        if ( cpu_isset(cpu, cpupool_locked_cpus) )
+            goto out;
 
-    if ( (c->n_dom > 0) && (cpus_weight(c->cpu_valid) == 1) &&
-         (cpu != cpupool_moving_cpu) )
+        /* Can't take the last cpu out of cpupool0 */
+        ret = -EINVAL;
+        if ( cpus_weight(c->cpu_valid) == 1
+             && c == cpupool0)
+            goto out;
+
+        ret = 0;
+        if ( !cpu_isset(cpu, c->cpu_valid) )
+            goto out;
+
+        if ( (c->n_dom > 0) && (cpus_weight(c->cpu_valid) == 1) )
+        {
+            for_each_domain(d)
+            {
+                if ( d->cpupool != c )
+                    continue;
+                /* Don't allow the last cpu from a pool to be moved if there's a live
+                 * domain still running on it */
+                if ( !d->is_dying )
+                {
+                    printk("%s: cpu %d pool %p: d%d still present\n",
+                           __func__, cpu, c, d->domain_id);
+                    ret = -EBUSY;
+                    break;
+                }
+                c->n_dom--;
+                ret = sched_move_domain(d, cpupool0);
+                if ( ret )
+                {
+                    c->n_dom++;
+                    break;
+                }
+                cpupool0->n_dom++;
+            }
+            if ( ret )
+                goto out;
+        }
+        cpupool_moving_cpu = cpu;
+        cpupool_cpu_moving = c;
+        cpu_clear(cpu, c->cpu_valid);
+    }
+    else
     {
-        for_each_domain(d)
-        {
-            if ( d->cpupool != c )
-                continue;
-            if ( !d->is_dying )
-            {
-                ret = -EBUSY;
-                break;
-            }
-            c->n_dom--;
-            ret = sched_move_domain(d, cpupool0);
-            if ( ret )
-            {
-                c->n_dom++;
-                break;
-            }
-            cpupool0->n_dom++;
-        }
-        if ( ret )
-            goto out;
+        /* Make sure all the things we did last time still hold */
+        BUG_ON(cpupool_cpu_moving != c);
+        BUG_ON(cpu_isset(cpu, c->cpu_valid));
+        /* compare cpu_valid to 0, since we cleared cpu in cpu_valid above */
+        BUG_ON((c->n_dom > 0) && (cpus_weight(c->cpu_valid) == 0));
     }
-    cpupool_moving_cpu = cpu;
+    /* Increase the refcount both times through, because the return path released
+     * the reference. */
     atomic_inc(&c->refcnt);
-    cpupool_cpu_moving = c;
-    cpu_clear(cpu, c->cpu_valid);
+
     spin_unlock(&cpupool_lock);
 
     work_cpu = smp_processor_id();
     if ( work_cpu == cpu )
     {
         work_cpu = first_cpu(cpupool0->cpu_valid);
+        /* If cpu is in cpupool0, then cpupool0 must contain at least one other cpu */
         if ( work_cpu == cpu )
             work_cpu = next_cpu(cpu, cpupool0->cpu_valid);
     }
diff -r 4ea36cce2519 xen/common/sched_credit.c
--- a/xen/common/sched_credit.c	Mon Feb 14 09:10:22 2011 +0000
+++ b/xen/common/sched_credit.c	Wed Feb 16 12:21:37 2011 +0000
@@ -330,11 +330,14 @@
     prv->ncpus--;
     cpu_clear(cpu, prv->idlers);
     cpu_clear(cpu, prv->cpus);
+#if 0
+    /* This should have been disabled already */
     if ( (prv->master == cpu) && (prv->ncpus > 0) )
     {
         prv->master = first_cpu(prv->cpus);
         migrate_timer(&prv->master_ticker, prv->master);
     }
+#endif
     kill_timer(&spc->ticker);
     if ( prv->ncpus == 0 )
         kill_timer(&prv->master_ticker);
@@ -367,12 +370,16 @@
     {
         prv->master = cpu;
         init_timer(&prv->master_ticker, csched_acct, prv, cpu);
+#if 0
         set_timer(&prv->master_ticker, NOW() +
                   MILLISECS(CSCHED_MSECS_PER_TICK) * CSCHED_TICKS_PER_ACCT);
+#endif
     }
 
     init_timer(&spc->ticker, csched_tick, (void *)(unsigned long)cpu, cpu);
+#if 0
     set_timer(&spc->ticker, NOW() + MILLISECS(CSCHED_MSECS_PER_TICK));
+#endif
 
     INIT_LIST_HEAD(&spc->runq);
     spc->runq_sort_last = prv->runq_sort;
@@ -1531,15 +1538,28 @@
 
 static void csched_tick_suspend(const struct scheduler *ops, unsigned int cpu)
 {
+    struct csched_private *prv = CSCHED_PRIV(ops);
     struct csched_pcpu *spc;
 
     spc = CSCHED_PCPU(cpu);
 
+    if (prv->master == cpu)
+    {
+        if ( (prv->ncpus > 0) )
+        {
+            prv->master = first_cpu(prv->cpus);
+            migrate_timer(&prv->master_ticker, prv->master);
+        }
+        else
+            stop_timer(&prv->master_ticker);
+    }
+
     stop_timer(&spc->ticker);
 }
 
 static void csched_tick_resume(const struct scheduler *ops, unsigned int cpu)
 {
+    struct csched_private *prv = CSCHED_PRIV(ops);
     struct csched_pcpu *spc;
     uint64_t now = NOW();
 
@@ -1547,6 +1567,12 @@
 
     set_timer(&spc->ticker, now + MILLISECS(CSCHED_MSECS_PER_TICK)
             - now % MILLISECS(CSCHED_MSECS_PER_TICK) );
+
+    if (prv->master == cpu)
+    {
+        set_timer(&prv->master_ticker, NOW() +
+                  MILLISECS(CSCHED_MSECS_PER_TICK) * CSCHED_TICKS_PER_ACCT);
+    }
 }
 
 static struct csched_private _csched_priv;
diff -r 4ea36cce2519 xen/common/schedule.c
--- a/xen/common/schedule.c	Mon Feb 14 09:10:22 2011 +0000
+++ b/xen/common/schedule.c	Wed Feb 16 12:21:37 2011 +0000
@@ -469,10 +469,21 @@
     cpumask_t online_affinity;
     int    ret = 0;
     bool_t affinity_broken;
+    struct scheduler *cpu_ops;
 
+    cpu_ops = per_cpu(scheduler, cpu);
     c = per_cpu(cpupool, cpu);
-    if ( c == NULL )
+    if ( c == NULL || cpu_ops == NULL )
+    {
+        printk("%s: no scheduler for cpu %d\n",
+               __func__, cpu);
         return ret;
+    }
+
+    pcpu_schedule_lock_irq(cpu);
+    SCHED_OP(cpu_ops, tick_suspend, cpu);
+    pcpu_schedule_unlock_irq(cpu);
+
 
     for_each_domain ( d )
     {
@@ -1211,6 +1222,8 @@
          ((sd->sched_priv = ops.alloc_pdata(&ops, cpu)) == NULL) )
         return -ENOMEM;
 
+    ops.tick_resume(&ops, cpu);
+
     return 0;
 }
 
@@ -1219,7 +1232,11 @@
     struct schedule_data *sd = &per_cpu(schedule_data, cpu);
 
     if ( sd->sched_priv != NULL )
+    {
+        /* FIXME: What if scheduler has different ops? */
+        SCHED_OP(&ops, tick_suspend, cpu);
         SCHED_OP(&ops, free_pdata, sd->sched_priv, cpu);
+    }
 
     kill_timer(&sd->s_timer);
 }
@@ -1288,6 +1305,8 @@
     if ( ops.alloc_pdata &&
          !(this_cpu(schedule_data).sched_priv = ops.alloc_pdata(&ops, 0)) )
         BUG();
+
+    ops.tick_resume(&ops, 0);
 }
 
 int schedule_cpu_switch(unsigned int cpu, struct cpupool *c)
@@ -1298,31 +1317,51 @@
     struct scheduler *old_ops = per_cpu(scheduler, cpu);
     struct scheduler *new_ops = (c == NULL) ? &ops : c->sched;
 
+    BUG_ON(!is_idle_vcpu(per_cpu(schedule_data, cpu).curr));
+
     if ( old_ops == new_ops )
+    {
+        //printk("%s: cpu %d pool %p: no change\n",
+        //     __func__,  cpu, c);
         return 0;
+    }
 
     idle = idle_vcpu[cpu];
     ppriv = SCHED_OP(new_ops, alloc_pdata, cpu);
     if ( ppriv == NULL )
+    {
+        printk("%s: cpu %d pool %p: alloc_pdata failed\n",
+               __func__, cpu, c);
         return -ENOMEM;
+    }
     vpriv = SCHED_OP(new_ops, alloc_vdata, idle, idle->domain->sched_priv);
     if ( vpriv == NULL )
     {
+        printk("%s: cpu %d pool %p: alloc_vdata(idle) failed\n",
+               __func__, cpu, c);
         SCHED_OP(new_ops, free_pdata, ppriv, cpu);
         return -ENOMEM;
     }
 
     pcpu_schedule_lock_irqsave(cpu, flags);
-
-    SCHED_OP(old_ops, tick_suspend, cpu);
+    //SCHED_OP(old_ops, tick_suspend, cpu);
+    /* Switch idle private */
     vpriv_old = idle->sched_priv;
     idle->sched_priv = vpriv;
+
+    /* Switch ops */
     per_cpu(scheduler, cpu) = new_ops;
+
+    /* Switch pcpu private */
     ppriv_old = per_cpu(schedule_data, cpu).sched_priv;
     per_cpu(schedule_data, cpu).sched_priv = ppriv;
-    SCHED_OP(new_ops, tick_resume, cpu);
+
     SCHED_OP(new_ops, insert_vcpu, idle);
 
+    /* Enabling ticks should be the last thing done,
+     * and only if moving it to a cpu pool */
+    if ( c != NULL )
+        SCHED_OP(new_ops, tick_resume, cpu);
     pcpu_schedule_unlock_irqrestore(cpu, flags);
 
     SCHED_OP(old_ops, free_vdata, vpriv_old);