On Aug 11, 2006, at 10:26 PM, Amos Waterland wrote:
This patch gets things to the point that each secondary processor can
call the same C routine that the primary processor did
I would like the initial CPU enumeration to be simpler, like it was
in the last patch:
- one int to communicate
- an array of bytes (or bits) for the barrier
These two would have temporary storage.
I do believe we will need a global array only contains pointers to
the pareas:
struct processor_area *global_cpu_table[NR_CPUS];
Mostly for debugging and maybe some crosscalls or IPIs
Note: I consider NR_CPUS a statement of what the code supports not
what the machine supports, so it is ment to be artificially high.
There is no reason that this cannot be dymanically allocated,
especially once if we can express the architectural limit, ie.
970FX:max_cpus=2, 970MP:max_cpus=4 etc...
Question to all:
The "cpuid" from firmware (OF "reg" property) is significant only to
uniquely identify the interrupt line that FW assigned to it (usually
for IPIs). It is not necessarily the processors PIR register value,
nor does it have any other meaning. Should we use it as our global
index or use our own numbering system
to invalidate the
segment table and initialize the HID registers.
Segment table? what segment table?
The logic needs a lot of cleanup, but right now I am worrying about
whether we need one idle domain per processor. We surely can't do
this
from more than one processor ...
set_current(idle_domain->vcpu[0]);
idle_vcpu[0] = current;
correct, you can't do that :)
The idle domain is our first SMP domain, so only _one_ idle domain
please.
Any comments would be appreciated.
---
arch/powerpc/boot_of.c | 70 ++++++++++++++++++++++++
++++++++----
arch/powerpc/of-devtree.h | 7 +++
arch/powerpc/powerpc64/exceptions.S | 38 +++++++++++++++++++
arch/powerpc/powerpc64/ppc970.c | 7 ++-
arch/powerpc/setup.c | 35 +++++++++++++++++-
include/asm-powerpc/config.h | 2 -
include/asm-powerpc/processor.h | 2 -
7 files changed, 149 insertions(+), 12 deletions(-)
diff -r bb510c274af8 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c Fri Aug 11 21:58:19 2006 -0400
@@ -30,6 +30,9 @@
#include <asm/io.h>
#include "exceptions.h"
#include "of-devtree.h"
+
+/* Secondary processors index into this by their processor id. */
+volatile struct per_cpu __per_cpu_init[NR_CPUS];
If there is no OF then this file will never get run, or even
compiled, so _no_ "persistent" storage should be declared here,
declare it in setup.c.
I should have mentioned this earlier but it looks like you intend for
this to live beyond that.
static ulong of_vec;
static ulong of_msr;
@@ -926,8 +929,8 @@ static void boot_of_module(ulong r3, ulo
static int __init boot_of_cpus(void)
{
- int cpus;
- int cpu;
+ int i, cpus;
+ int cpu, bootcpu;
int result;
u32 cpu_clock[2];
@@ -952,10 +955,53 @@ static int __init boot_of_cpus(void)
cpu_khz /= 1000;
of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
- /* FIXME: should not depend on the boot CPU bring the first
child */
- cpu = of_getpeer(cpu);
- while (cpu > 0) {
- of_start_cpu(cpu, (ulong)spin_start, 0);
+ /* Look up which CPU we are running on right now. */
+ result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof
(bootcpu));
+ if (result == OF_FAILURE)
+ of_panic("Failed to look up boot cpu\n");
+
+ for (i = 0, cpu = of_getpeer(cpu); i < NR_CPUS && cpu > 0; i++) {
+ unsigned int cpuid, ping, pong;
+ unsigned long now, then, timeout;
+
+ if (cpu == bootcpu) {
+ of_printf("skipping boot cpu!\n");
+ continue;
+ }
+
+ result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+ if (result == OF_FAILURE)
+ of_panic("cpuid lookup failed\n");
if you are going to use cpuid as an index then you need to:
if (cpuid >= NR_CPUS)
of_panic();
+
+ of_printf("spinning up secondary processor #%d: ", cpuid);
+
+ __per_cpu_init[cpuid].sp = 0x0;
+ __per_cpu_init[cpuid].id = ~0x0;
+ ping = __per_cpu_init[cpuid].id;
+ pong = __per_cpu_init[cpuid].id;
+ of_printf("ping = 0x%x: ", ping);
+
+ mb();
+ result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+ if (result == OF_FAILURE)
+ of_panic("start cpu failed\n");
+
+ /* We will give the secondary processor five seconds to
reply. */
+ then = mftb();
+ timeout = then + (5 * timebase_freq);
+
+ do {
+ now = mftb();
+ if (now >= timeout) {
+ of_printf("BROKEN: ");
+ break;
+ }
+
+ mb();
+ pong = __per_cpu_init[cpuid].id;
+ } while (pong == ping);
+ of_printf("pong = 0x%x\n", pong);
+
cpu = of_getpeer(cpu);
}
return 1;
@@ -1003,9 +1049,21 @@ multiboot_info_t __init *boot_of_init(
boot_of_rtas();
/* end of OF */
+ of_printf("Quiescing Open Firmware ...\n");
of_call("quiesce", 0, 0, NULL);
return &mbi;
+}
+
+int secondary_cpu_init(int cpuid);
+
+int secondary_cpu_init(int cpuid)
+{
+ printk("CPU #%d: Hello World!\n", cpuid);
+
+ cpu_initialize(cpuid);
+
+ return 0;
}
/*
diff -r bb510c274af8 xen/arch/powerpc/of-devtree.h
--- a/xen/arch/powerpc/of-devtree.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/of-devtree.h Fri Aug 11 19:55:51 2006 -0400
@@ -23,6 +23,13 @@
#include <xen/types.h>
#include <public/xen.h>
+
+/* Data structure used by secondary processors to discover their
stack. */
+struct per_cpu {
+ int id;
+ void *sp;
+ void *toc;
+};
enum {
OF_FAILURE = -1,
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 13:30:48
2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 21:11:00
2006 -0400
@@ -514,6 +514,44 @@ _GLOBAL(sleep)
mtmsrd r3
blr
+/* Get these from asm-offsets, talk to Jimi about int->long
alignment hole. */
+#define ID_OFFSET 0
+#define SP_OFFSET 8
+#define TC_OFFSET 16
+#define PER_CPU_SIZE 24
If you _do_ need these make sure they are part of the:
arch/powerpc/powerpc64/asm-offsets.c
logic, but I don;t think you need them.. all you should need is:
PAREA_stack
which is already defined.
+
+/* Firmware is told to spin up secondary processors at this
address. We
+ * only need a function entry point instead of a descriptor since
this is
+ * never called from C code.
+ */
.globl spin_start
spin_start:
+ /* Our physical cpu number is passed in r3, so index into
array. */
+ muli r5, r3, PER_CPU_SIZE
+ LOADADDR(r27, __per_cpu_init)
+ add r27, r27, r5
+ stw r3, ID_OFFSET(r27)
+ sync
+ /* At some point these should go in different cache lines. */
+1: ld r6, SP_OFFSET(r27)
+ cmpldi r6, 0
+ beq 1b
+ /* The primary cpu has an allocator now and wants our
attention. */
+ li r6, 0x0
+ std r6, SP_OFFSET(r27)
+ sync
+ /* Wait for the primary cpu to give us our stack address. */
+2: ld r6, SP_OFFSET(r27)
See the other comments about parea
+ cmpldi r6, 0
+ beq 2b
+ /* The primary cpu has allocated a stack for us, so rock and
roll. */
+ mr r1, r6
The code assembler should be and you can get rid of all toc
references elsewhere in the patch.
LOADADDR(r8, secondary_cpu_init)
CALL_CFUNC(r8)
NOTE: the missing '.'
+ ld r2, TC_OFFSET(r27)
+ LOADADDR(r8, .secondary_cpu_init)
+ mtctr r8
+ bctrl
+ /* For now, the primary processor is waiting for us to return
from C. */
+ li r3, 0
+ std r3, SP_OFFSET(r27)
+ sync
b .
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/ppc970.c
--- a/xen/arch/powerpc/powerpc64/ppc970.c Fri Aug 11 13:30:48 2006
-0400
+++ b/xen/arch/powerpc/powerpc64/ppc970.c Fri Aug 11 21:56:15 2006
-0400
@@ -37,10 +37,11 @@ unsigned int cpu_rma_order(void)
return 14; /* 1<<14<<PAGE_SIZE = 64M */
}
-void cpu_initialize(void)
+void cpu_initialize(int cpuid)
{
ulong stack;
+ /* This is SMP safe because the compiler must use r13 for it. */
parea = xmalloc(struct processor_area);
ASSERT(parea != NULL);
The parea and the stack should be allocated elsewhere since it is not
970 specific, though parea needs to be assigned in this function, see
below.
this becomes:
parea = global_CPU[cpuid];
@@ -48,7 +49,7 @@ void cpu_initialize(void)
ASSERT(stack != 0);
parea->hyp_stack_base = (void *)(stack + STACK_SIZE);
- printk("stack is here: %p\n", parea->hyp_stack_base);
+ printk("CPU #%d: stack is here: %p\n", cpuid, parea-
>hyp_stack_base);
mthsprg0((ulong)parea); /* now ready for exceptions */
@@ -78,7 +79,7 @@ void cpu_initialize(void)
s |= 1UL << (63-3); /* ser-gp */
hid0.word |= s;
#endif
- printk("hid0: 0x%016lx\n", hid0.word);
+ printk("CPU #%d: hid0: 0x%016lx\n", cpuid, hid0.word);
mthid0(hid0.word);
union hid1 hid1;
diff -r bb510c274af8 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/setup.c Fri Aug 11 21:55:40 2006 -0400
@@ -75,6 +75,8 @@ extern void idle_loop(void);
/* move us to a header file */
extern void initialize_keytable(void);
+extern volatile struct per_cpu __per_cpu_init[];
+
int is_kernel_text(unsigned long addr)
{
if (addr >= (unsigned long) &_start &&
@@ -314,7 +316,7 @@ static void __init __start_xen(multiboot
printk("Xen heap: %luMB (%lukB)\n", heap_size >> 20, heap_size
>> 10);
- cpu_initialize();
+ cpu_initialize(0);
#ifdef CONFIG_GDB
initialise_gdb();
@@ -323,6 +325,37 @@ static void __init __start_xen(multiboot
#endif
start_of_day();
+
+ {
+ int i;
+
+ for (i = 1; i <= NR_CPUS; i++) {
+ void *stack, *toc;
There should be no need to set the TOC since it gets set when
assembler calls C.
+ void *syn = (void *)~0x0;
+
+ if (__per_cpu_init[i].id > 0) {
+ printk("CPU #%d is waiting for a stack ...\n", i);
+
+ __per_cpu_init[i].sp = syn;
+ do {
+ mb();
+ } while (__per_cpu_init[i].sp == syn);
+
+ stack = xmalloc_bytes(STACK_SIZE);
You should allocate both parea and stack (using STACK_ORDER and
offset) here. (as stolen from cpu_initialize())
global_cpu[cpuid] = xmalloc(struct processor_area);
stack = (ulong)alloc_xenheap_pages(STACK_ORDER);
global_cpu[cpuid]->hyp_stack_base = (void *)(stack + STACK_SIZE);
+ if (stack == NULL)
+ panic("failed to allocate stack\n");
+
+ __per_cpu_init[i].sp = stack;
+ asm("mr %0, 2" : "=r" (toc));
+ __per_cpu_init[i].toc = toc;
+ mb();
+
+ do {
+ mb();
+ } while (__per_cpu_init[i].sp != 0x0);
+ }
+ }
+ }
/* Create initial domain 0. */
dom0 = domain_create(0);
diff -r bb510c274af8 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/include/asm-powerpc/config.h Fri Aug 11 16:56:58 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
#define CONFIG_GDB 1
#define CONFIG_SMP 1
#define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 6
IMNSHO there are two possible values for NR_CPUS: 1 and
BITS_PER_LONG, am I messed up here?
#ifndef ELFSIZE
#define ELFSIZE 64
diff -r bb510c274af8 xen/include/asm-powerpc/processor.h
--- a/xen/include/asm-powerpc/processor.h Fri Aug 11 13:30:48 2006
-0400
+++ b/xen/include/asm-powerpc/processor.h Fri Aug 11 21:54:51 2006
-0400
@@ -40,7 +40,7 @@ extern void show_registers(struct cpu_us
extern void show_registers(struct cpu_user_regs *);
extern void show_execution_state(struct cpu_user_regs *);
extern unsigned int cpu_rma_order(void);
-extern void cpu_initialize(void);
+extern void cpu_initialize(int cpuid);
extern void cpu_init_vcpu(struct vcpu *);
extern void save_cpu_sprs(struct vcpu *);
extern void load_cpu_sprs(struct vcpu *);
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-ppc-devel
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-ppc-devel
|
Home