This patch enables SMP and IPIs. It works reliably on JS20 and JS21
blades. It is not quite ready for submission, but I would greatly
appreciate any comments.
Note that the flurry of IPIs generated by domain creation seems to be a
waste of time. Xen on x86 doesn't actually to do anything in response
to them, so I have made Xen on PPC properly deliver the event check but
then do nothing as well. Comments?
The important thing this patch is missing is the ability to invoke
functions on a remote CPU, and I have left it out because I am not yet
happy with the convention, which is to grab a lock, put the target
address in a single global shared variable, invoke a memory barrier, and
send the IPI. I am hoping to avoid a lock around that operation by
making a per-IPI-destination-CPU structure with lock, address, and ack
fields. Comments?
---
changeset : cc0cd5518e10
machines : cso103 kpblade1 cso91
pass : 35
fail : 0
transient : 0
total : 35
reliability : 100.0%
---
arch/powerpc/mpic.c | 9 ---
arch/powerpc/mpic_init.c | 42 +++++++++++++++++
arch/powerpc/setup.c | 40 ++++++++--------
arch/powerpc/smp.c | 59 ++++++++++++++++++++++++-
include/asm-powerpc/mach-default/irq_vectors.h | 11 ++--
include/asm-powerpc/smp.h | 10 ++++
6 files changed, 136 insertions(+), 35 deletions(-)
diff -r 3dfeb3e4a03f xen/arch/powerpc/mpic.c
--- a/xen/arch/powerpc/mpic.c Fri Oct 13 11:00:32 2006 -0400
+++ b/xen/arch/powerpc/mpic.c Mon Oct 23 22:57:51 2006 -0400
@@ -27,10 +27,6 @@
#define alloc_bootmem(x) xmalloc_bytes(x)
-#define request_irq(irq, handler, f, devname, dev_id) \
- panic("IPI requested: %d: %p: %s: %p\n", irq, handler, devname, dev_id)
-
-typedef int irqreturn_t;
#define IRQ_NONE (0)
#define IRQ_HANDLED (1)
@@ -96,11 +92,6 @@ typedef int irqreturn_t;
#endif
#include <asm/mpic.h>
#include <asm/smp.h>
-
-static inline void smp_message_recv(int msg, struct pt_regs *regs)
-{
- return;
-}
#ifdef DEBUG
#define DBG(fmt...) printk(fmt)
diff -r 3dfeb3e4a03f xen/arch/powerpc/mpic_init.c
--- a/xen/arch/powerpc/mpic_init.c Fri Oct 13 11:00:32 2006 -0400
+++ b/xen/arch/powerpc/mpic_init.c Mon Oct 23 15:32:55 2006 -0400
@@ -358,6 +358,43 @@ static struct hw_interrupt_type *share_m
#endif
+static inline struct mpic * mpic_from_ipi(unsigned int ipi)
+{
+ return container_of(irq_desc[ipi].handler, struct mpic, hc_ipi);
+}
+
+static unsigned int mpic_startup_ipi(unsigned int irq)
+{
+ printk("%s\n", __func__);
+ struct mpic *pic = mpic_from_ipi(irq);
+ pic->hc_ipi.enable(irq);
+ return 0;
+}
+
+int request_irq(unsigned int irq,
+ irqreturn_t (*handler)(int, void *, struct cpu_user_regs *),
+ unsigned long irqflags, const char * devname, void *dev_id)
+{
+ int retval;
+ struct irqaction *action;
+
+ printk("%s: %d: %p: %s: %p\n", __func__, irq, handler, devname, dev_id);
+
+ action = xmalloc_bytes(sizeof(struct irqaction));
+ if (!action)
+ BUG();
+
+ action->handler = (void (*)(int, void *, struct cpu_user_regs *))handler;
+ action->name = devname;
+ action->dev_id = dev_id;
+
+ retval = setup_irq(irq, action);
+ if (retval)
+ BUG();
+
+ return retval;
+}
+
struct hw_interrupt_type *xen_mpic_init(struct hw_interrupt_type *xen_irq)
{
unsigned int isu_size;
@@ -397,6 +434,11 @@ struct hw_interrupt_type *xen_mpic_init(
hit = share_mpic(&mpic->hc_irq, xen_irq);
printk("%s: success\n", __func__);
+
+ mpic->hc_ipi.ack = xen_irq->ack;
+ mpic->hc_ipi.startup = mpic_startup_ipi;
+ mpic_request_ipis();
+
return hit;
}
diff -r 3dfeb3e4a03f xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c Fri Oct 13 11:00:32 2006 -0400
+++ b/xen/arch/powerpc/setup.c Fri Oct 13 19:00:34 2006 -0400
@@ -37,6 +37,7 @@
#include <xen/keyhandler.h>
#include <acm/acm_hooks.h>
#include <public/version.h>
+#include <asm/mpic.h>
#include <asm/processor.h>
#include <asm/desc.h>
#include <asm/cache.h>
@@ -88,6 +89,8 @@ struct ns16550_defaults ns16550;
extern char __per_cpu_start[], __per_cpu_data_end[], __per_cpu_end[];
+static struct domain *idle_domain;
+
volatile struct processor_area * volatile global_cpu_table[NR_CPUS];
int is_kernel_text(unsigned long addr)
@@ -159,8 +162,6 @@ static void percpu_free_unused_areas(voi
static void __init start_of_day(void)
{
- struct domain *idle_domain;
-
init_IRQ();
scheduler_init();
@@ -174,23 +175,6 @@ static void __init start_of_day(void)
/* for some reason we need to set our own bit in the thread map */
cpu_set(0, cpu_sibling_map[0]);
-
- percpu_free_unused_areas();
-
- {
- /* FIXME: Xen assumes that an online CPU is a schedualable
- * CPU, but we just are not there yet. Remove this fragment when
- * scheduling processors actually works. */
- int cpuid;
-
- printk("WARNING!: Taking all secondary CPUs offline\n");
-
- for_each_online_cpu(cpuid) {
- if (cpuid == 0)
- continue;
- cpu_clear(cpuid, cpu_online_map);
- }
- }
initialize_keytable();
/* Register another key that will allow for the the Harware Probe
@@ -263,9 +247,20 @@ static int kick_secondary_cpus(int maxcp
/* This is the first C code that secondary processors invoke. */
int secondary_cpu_init(int cpuid, unsigned long r4)
{
+ struct vcpu *vcpu;
+
cpu_initialize(cpuid);
smp_generic_take_timebase();
cpu_set(cpuid, cpu_online_map);
+
+ vcpu = alloc_vcpu(idle_domain, cpuid, cpuid);
+ BUG_ON(vcpu == NULL);
+
+ set_current(idle_domain->vcpu[cpuid]);
+ idle_vcpu[cpuid] = current;
+ mpic_setup_this_cpu();
+ startup_cpu_idle_loop();
+
while(1);
}
@@ -340,6 +335,10 @@ static void __init __start_xen(multiboot
debugger_trap_immediate();
#endif
+ start_of_day();
+
+ mpic_setup_this_cpu();
+
/* Deal with secondary processors. */
if (opt_nosmp || ofd_boot_cpu == -1) {
printk("nosmp: leaving secondary processors spinning forever\n");
@@ -348,7 +347,8 @@ static void __init __start_xen(multiboot
kick_secondary_cpus(max_cpus);
}
- start_of_day();
+ /* This cannot be called before secondary cpus are marked online. */
+ percpu_free_unused_areas();
/* Create initial domain 0. */
dom0 = domain_create(0);
diff -r 3dfeb3e4a03f xen/arch/powerpc/smp.c
--- a/xen/arch/powerpc/smp.c Fri Oct 13 11:00:32 2006 -0400
+++ b/xen/arch/powerpc/smp.c Mon Oct 23 23:15:45 2006 -0400
@@ -22,6 +22,8 @@
#include <xen/smp.h>
#include <asm/flushtlb.h>
#include <asm/debugger.h>
+#include <asm/mpic.h>
+#include <asm/mach-default/irq_vectors.h>
int smp_num_siblings = 1;
int smp_num_cpus = 1;
@@ -46,11 +48,30 @@ void __flush_tlb_mask(cpumask_t mask, un
unimplemented();
}
+static void send_IPI_mask(cpumask_t mask, int vector)
+{
+ unsigned long cpus;
+
+ switch(vector) {
+ case CALL_FUNCTION_VECTOR:
+ case EVENT_CHECK_VECTOR:
+ case INVALIDATE_TLB_VECTOR:
+ break;
+ default:
+ BUG();
+ }
+
+ BUG_ON(sizeof(mask.bits) != sizeof(unsigned long));
+ cpus = mask.bits[0];
+
+ mpic_send_ipi(vector, cpus);
+}
+
void smp_send_event_check_mask(cpumask_t mask)
{
cpu_clear(smp_processor_id(), mask);
if (!cpus_empty(mask))
- unimplemented();
+ send_IPI_mask(mask, EVENT_CHECK_VECTOR);
}
@@ -78,3 +99,39 @@ int on_selected_cpus(
unimplemented();
return 0;
}
+
+void smp_call_function_interrupt(struct cpu_user_regs *regs)
+{
+ BUG();
+ return;
+}
+
+void smp_event_check_interrupt(void)
+{
+ /* Like x86 we don't actually do anything with this IPI. */
+ return;
+}
+
+void smp_invalidate_interrupt(void)
+{
+ BUG();
+ return;
+}
+
+void smp_message_recv(int msg, struct cpu_user_regs *regs)
+{
+ switch(msg) {
+ case CALL_FUNCTION_VECTOR:
+ smp_call_function_interrupt(regs);
+ break;
+ case EVENT_CHECK_VECTOR:
+ smp_event_check_interrupt();
+ break;
+ case INVALIDATE_TLB_VECTOR:
+ smp_invalidate_interrupt();
+ break;
+ default:
+ BUG();
+ break;
+ }
+}
diff -r 3dfeb3e4a03f xen/include/asm-powerpc/mach-default/irq_vectors.h
--- a/xen/include/asm-powerpc/mach-default/irq_vectors.h Fri Oct 13
11:00:32 2006 -0400
+++ b/xen/include/asm-powerpc/mach-default/irq_vectors.h Mon Oct 23
21:00:34 2006 -0400
@@ -37,8 +37,6 @@
#define FAST_TRAP -1 /* 0x80 */
#define FIRST_SYSTEM_VECTOR -1
-#if 0
-
/*
* Vectors 0-16 in some cases are used for ISA interrupts.
*/
@@ -54,9 +52,12 @@
*/
#define SPURIOUS_APIC_VECTOR 0xff
#define ERROR_APIC_VECTOR 0xfe
-#define INVALIDATE_TLB_VECTOR 0xfd
-#define EVENT_CHECK_VECTOR 0xfc
-#define CALL_FUNCTION_VECTOR 0xfb
+
+#define CALL_FUNCTION_VECTOR 0x0
+#define EVENT_CHECK_VECTOR 0x1
+#define INVALIDATE_TLB_VECTOR 0x3
+
+#if 0
#define THERMAL_APIC_VECTOR 0xf0
/*
diff -r 3dfeb3e4a03f xen/include/asm-powerpc/smp.h
--- a/xen/include/asm-powerpc/smp.h Fri Oct 13 11:00:32 2006 -0400
+++ b/xen/include/asm-powerpc/smp.h Mon Oct 23 23:02:25 2006 -0400
@@ -35,4 +35,14 @@ extern cpumask_t cpu_core_map[];
extern cpumask_t cpu_core_map[];
extern void __devinit smp_generic_take_timebase(void);
extern void __devinit smp_generic_give_timebase(void);
+
+#define SA_INTERRUPT 0x20000000u
+typedef int irqreturn_t;
+extern int request_irq(unsigned int irq,
+ irqreturn_t (*handler)(int, void *, struct cpu_user_regs *),
+ unsigned long irqflags, const char * devname, void *dev_id);
+void smp_message_recv(int msg, struct cpu_user_regs *regs);
+void smp_call_function_interrupt(struct cpu_user_regs *regs);
+void smp_event_check_interrupt(void);
+void smp_invalidate_interrupt(void);
#endif
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-ppc-devel
|
Home