diff --git a/xen/common/rcupdate.c b/xen/common/rcupdate.c index d6dc4b48db..42ab9dbbd6 100644 --- a/xen/common/rcupdate.c +++ b/xen/common/rcupdate.c @@ -52,8 +52,8 @@ static struct rcu_ctrlblk { int next_pending; /* Is the next batch already waiting? */ spinlock_t lock __cacheline_aligned; - cpumask_t cpumask; /* CPUs that need to switch in order ... */ - cpumask_t idle_cpumask; /* ... unless they are already idle */ + cpumask_t cpumask; /* CPUs that need to switch in order ... */ + cpumask_t ignore_cpumask; /* ... unless they are already idle */ /* for current batch to proceed. */ } __cacheline_aligned rcu_ctrlblk = { .cur = -300, @@ -86,8 +86,8 @@ struct rcu_data { long last_rs_qlen; /* qlen during the last resched */ /* 3) idle CPUs handling */ - struct timer idle_timer; - bool idle_timer_active; + struct timer cb_timer; + bool cb_timer_active; }; /* @@ -116,22 +116,22 @@ struct rcu_data { * CPU that is going idle. The user can change this, via a boot time * parameter, but only up to 100ms. */ -#define IDLE_TIMER_PERIOD_MAX MILLISECS(100) -#define IDLE_TIMER_PERIOD_DEFAULT MILLISECS(10) -#define IDLE_TIMER_PERIOD_MIN MICROSECS(100) +#define CB_TIMER_PERIOD_MAX MILLISECS(100) +#define CB_TIMER_PERIOD_DEFAULT MILLISECS(10) +#define CB_TIMER_PERIOD_MIN MICROSECS(100) -static s_time_t __read_mostly idle_timer_period; +static s_time_t __read_mostly cb_timer_period; /* - * Increment and decrement values for the idle timer handler. The algorithm + * Increment and decrement values for the callback timer handler. The algorithm * works as follows: * - if the timer actually fires, and it finds out that the grace period isn't - * over yet, we add IDLE_TIMER_PERIOD_INCR to the timer's period; + * over yet, we add CB_TIMER_PERIOD_INCR to the timer's period; * - if the timer actually fires and it finds the grace period over, we * subtract IDLE_TIMER_PERIOD_DECR from the timer's period. */ -#define IDLE_TIMER_PERIOD_INCR MILLISECS(10) -#define IDLE_TIMER_PERIOD_DECR MICROSECS(100) +#define CB_TIMER_PERIOD_INCR MILLISECS(10) +#define CB_TIMER_PERIOD_DECR MICROSECS(100) static DEFINE_PER_CPU(struct rcu_data, rcu_data); @@ -309,7 +309,7 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp) * This barrier is paired with the one in rcu_idle_enter(). */ smp_mb(); - cpumask_andnot(&rcp->cpumask, &cpu_online_map, &rcp->idle_cpumask); + cpumask_andnot(&rcp->cpumask, &cpu_online_map, &rcp->ignore_cpumask); } } @@ -455,7 +455,7 @@ int rcu_needs_cpu(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_data, cpu); - return (rdp->curlist && !rdp->idle_timer_active) || rcu_pending(cpu); + return (rdp->curlist && !rdp->cb_timer_active) || rcu_pending(cpu); } /* @@ -463,7 +463,7 @@ int rcu_needs_cpu(int cpu) * periodically poke rcu_pedning(), so that it will invoke the callback * not too late after the end of the grace period. */ -void rcu_idle_timer_start() +static void cb_timer_start(void) { struct rcu_data *rdp = &this_cpu(rcu_data); @@ -475,48 +475,48 @@ void rcu_idle_timer_start() if (likely(!rdp->curlist)) return; - set_timer(&rdp->idle_timer, NOW() + idle_timer_period); - rdp->idle_timer_active = true; + set_timer(&rdp->cb_timer, NOW() + cb_timer_period); + rdp->cb_timer_active = true; } -void rcu_idle_timer_stop() +static void cb_timer_stop(void) { struct rcu_data *rdp = &this_cpu(rcu_data); - if (likely(!rdp->idle_timer_active)) + if (likely(!rdp->cb_timer_active)) return; - rdp->idle_timer_active = false; + rdp->cb_timer_active = false; /* * In general, as the CPU is becoming active again, we don't need the - * idle timer, and so we want to stop it. + * callback timer, and so we want to stop it. * - * However, in case we are here because idle_timer has (just) fired and + * However, in case we are here because cb_timer has (just) fired and * has woken up the CPU, we skip stop_timer() now. In fact, when a CPU * wakes up from idle, this code always runs before do_softirq() has the * chance to check and deal with TIMER_SOFTIRQ. And if we stop the timer * now, the TIMER_SOFTIRQ handler will see it as inactive, and will not - * call rcu_idle_timer_handler(). + * call cb_timer_handler(). * * Therefore, if we see that the timer is expired already, we leave it * alone. The TIMER_SOFTIRQ handler will then run the timer routine, and * deactivate it. */ - if ( !timer_is_expired(&rdp->idle_timer) ) - stop_timer(&rdp->idle_timer); + if ( !timer_is_expired(&rdp->cb_timer) ) + stop_timer(&rdp->cb_timer); } -static void rcu_idle_timer_handler(void* data) +static void cb_timer_handler(void* data) { - perfc_incr(rcu_idle_timer); + perfc_incr(rcu_callback_timer); if ( !cpumask_empty(&rcu_ctrlblk.cpumask) ) - idle_timer_period = min(idle_timer_period + IDLE_TIMER_PERIOD_INCR, - IDLE_TIMER_PERIOD_MAX); + cb_timer_period = min(cb_timer_period + CB_TIMER_PERIOD_INCR, + CB_TIMER_PERIOD_MAX); else - idle_timer_period = max(idle_timer_period - IDLE_TIMER_PERIOD_DECR, - IDLE_TIMER_PERIOD_MIN); + cb_timer_period = max(cb_timer_period - CB_TIMER_PERIOD_DECR, + CB_TIMER_PERIOD_MIN); } void rcu_check_callbacks(int cpu) @@ -537,7 +537,7 @@ static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, static void rcu_offline_cpu(struct rcu_data *this_rdp, struct rcu_ctrlblk *rcp, struct rcu_data *rdp) { - kill_timer(&rdp->idle_timer); + kill_timer(&rdp->cb_timer); /* If the cpu going offline owns the grace period we can block * indefinitely waiting for it, so flush it here. @@ -567,7 +567,7 @@ static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, rdp->qs_pending = 0; rdp->cpu = cpu; rdp->blimit = blimit; - init_timer(&rdp->idle_timer, rcu_idle_timer_handler, rdp, cpu); + init_timer(&rdp->cb_timer, cb_timer_handler, rdp, cpu); } static int cpu_callback( @@ -596,25 +596,39 @@ static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; +/* + * We're changing the name of the parameter, to better reflect the fact that + * the timer is used for callbacks in general, when the CPU is either idle + * or executing guest code. We still accept the old parameter but, if both + * are specified, the new one ("rcu-callback-timer-period-ms") has priority. + */ +#define CB_TIMER_PERIOD_DEFAULT_MS ( CB_TIMER_PERIOD_DEFAULT / MILLISECS(1) ) +static unsigned int __initdata cb_timer_period_ms = CB_TIMER_PERIOD_DEFAULT_MS; +integer_param("rcu-callback-timer-period-ms", cb_timer_period_ms); + +static unsigned int __initdata idle_timer_period_ms = CB_TIMER_PERIOD_DEFAULT_MS; +integer_param("rcu-idle-timer-period-ms", idle_timer_period_ms); + void __init rcu_init(void) { void *cpu = (void *)(long)smp_processor_id(); - static unsigned int __initdata idle_timer_period_ms = - IDLE_TIMER_PERIOD_DEFAULT / MILLISECS(1); - integer_param("rcu-idle-timer-period-ms", idle_timer_period_ms); + + if (idle_timer_period_ms != CB_TIMER_PERIOD_DEFAULT_MS && + cb_timer_period_ms == CB_TIMER_PERIOD_DEFAULT_MS) + cb_timer_period_ms = idle_timer_period_ms; /* We don't allow 0, or anything higher than IDLE_TIMER_PERIOD_MAX */ - if ( idle_timer_period_ms == 0 || - idle_timer_period_ms > IDLE_TIMER_PERIOD_MAX / MILLISECS(1) ) + if ( cb_timer_period_ms == 0 || + cb_timer_period_ms > CB_TIMER_PERIOD_MAX / MILLISECS(1) ) { - idle_timer_period_ms = IDLE_TIMER_PERIOD_DEFAULT / MILLISECS(1); - printk("WARNING: rcu-idle-timer-period-ms outside of " + cb_timer_period_ms = CB_TIMER_PERIOD_DEFAULT / MILLISECS(1); + printk("WARNING: rcu-callback-timer-period-ms outside of " "(0,%"PRI_stime"]. Resetting it to %u.\n", - IDLE_TIMER_PERIOD_MAX / MILLISECS(1), idle_timer_period_ms); + CB_TIMER_PERIOD_MAX / MILLISECS(1), cb_timer_period_ms); } - idle_timer_period = MILLISECS(idle_timer_period_ms); + cb_timer_period = MILLISECS(cb_timer_period_ms); - cpumask_clear(&rcu_ctrlblk.idle_cpumask); + cpumask_clear(&rcu_ctrlblk.ignore_cpumask); cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); register_cpu_notifier(&cpu_nfb); open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); @@ -626,8 +640,8 @@ void __init rcu_init(void) */ void rcu_idle_enter(unsigned int cpu) { - ASSERT(!cpumask_test_cpu(cpu, &rcu_ctrlblk.idle_cpumask)); - cpumask_set_cpu(cpu, &rcu_ctrlblk.idle_cpumask); + ASSERT(!cpumask_test_cpu(cpu, &rcu_ctrlblk.ignore_cpumask)); + cpumask_set_cpu(cpu, &rcu_ctrlblk.ignore_cpumask); /* * If some other CPU is starting a new grace period, we'll notice that * by seeing a new value in rcp->cur (different than our quiescbatch). @@ -637,10 +651,12 @@ void rcu_idle_enter(unsigned int cpu) * Se the comment before cpumask_andnot() in rcu_start_batch(). */ smp_mb(); + cb_timer_start(); } void rcu_idle_exit(unsigned int cpu) { - ASSERT(cpumask_test_cpu(cpu, &rcu_ctrlblk.idle_cpumask)); - cpumask_clear_cpu(cpu, &rcu_ctrlblk.idle_cpumask); + cb_timer_stop(); + ASSERT(cpumask_test_cpu(cpu, &rcu_ctrlblk.ignore_cpumask)); + cpumask_clear_cpu(cpu, &rcu_ctrlblk.ignore_cpumask); } diff --git a/xen/include/xen/perfc_defn.h b/xen/include/xen/perfc_defn.h index 08b182ccd9..d142534383 100644 --- a/xen/include/xen/perfc_defn.h +++ b/xen/include/xen/perfc_defn.h @@ -12,7 +12,7 @@ PERFCOUNTER(calls_from_multicall, "calls from multicall") PERFCOUNTER(irqs, "#interrupts") PERFCOUNTER(ipis, "#IPIs") -PERFCOUNTER(rcu_idle_timer, "RCU: idle_timer") +PERFCOUNTER(rcu_callback_timer, "RCU: callback_timer") /* Generic scheduler counters (applicable to all schedulers) */ PERFCOUNTER(sched_irq, "sched: timer")