diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/head.S
--- a/xen/arch/ia64/linux-xen/head.S	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/head.S	Tue Sep 27 12:48:19 2005
@@ -324,6 +324,7 @@
 	mov r16=-1
 (isBP)	br.cond.dpnt .load_current // BP stack is on region 5 --- no need to map it
 
+#ifdef XEN_TG
 	// load mapping for stack (virtaddr in r2, physaddr in r3)
 	rsm psr.ic
 	movl r17=PAGE_KERNEL
@@ -353,7 +354,8 @@
 	ssm psr.ic
 	srlz.d
   	;;
-
+#endif
+	
 .load_current:
 	// load the "current" pointer (r13) and ar.k6 with the current task
 #if defined(XEN) && defined(CONFIG_VTI)
diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/irq_ia64.c
--- a/xen/arch/ia64/linux-xen/irq_ia64.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/irq_ia64.c	Tue Sep 27 12:48:19 2005
@@ -281,5 +281,6 @@
 	ipi_data = (delivery_mode << 8) | (vector & 0xff);
 	ipi_addr = ipi_base_addr + ((phys_cpu_id << 4) | ((redirect & 1) << 3));
 
+	printf ("ipi: %lx -> %p\n", ipi_data, ipi_addr);
 	writeq(ipi_data, ipi_addr);
 }
diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/mm_contig.c
--- a/xen/arch/ia64/linux-xen/mm_contig.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/mm_contig.c	Tue Sep 27 12:48:19 2005
@@ -193,8 +193,8 @@
 	 */
 	if (smp_processor_id() == 0) {
 #ifdef XEN
-		cpu_data = alloc_xenheap_pages(PERCPU_PAGE_SHIFT -
-			PAGE_SHIFT + get_order(NR_CPUS));
+		cpu_data = alloc_xenheap_pages(get_order(NR_CPUS
+							 * PERCPU_PAGE_SIZE));
 #else
 		cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS,
 					   PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
@@ -202,6 +202,8 @@
 		for (cpu = 0; cpu < NR_CPUS; cpu++) {
 			memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start);
 			__per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start;
+			printf ("per_cpu_init: cpu=%d, addr=%p, offset=%p\n",
+				cpu, cpu_data, __per_cpu_offset[cpu]);
 			cpu_data += PERCPU_PAGE_SIZE;
 			per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu];
 		}
diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/setup.c
--- a/xen/arch/ia64/linux-xen/setup.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/setup.c	Tue Sep 27 12:48:19 2005
@@ -377,13 +377,9 @@
 	ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist);
 
 	*cmdline_p = __va(ia64_boot_param->command_line);
-#ifdef XEN
-	efi_init();
-#else
+
+#ifndef XEN
 	strlcpy(saved_command_line, *cmdline_p, COMMAND_LINE_SIZE);
-
-	efi_init();
-	io_port_init();
 #endif
 
 #ifdef CONFIG_IA64_GENERIC
@@ -414,11 +410,22 @@
 #ifdef XEN
 	early_cmdline_parse(cmdline_p);
 	cmdline_parse(*cmdline_p);
-#undef CONFIG_ACPI_BOOT
 #endif
 	if (early_console_setup(*cmdline_p) == 0)
 		mark_bsp_online();
 
+	efi_init();
+	io_port_init();
+
+
+#ifndef XEN
+	find_memory();
+#else
+}
+
+void __init
+late_setup_arch (char **cmdline_p)
+{
 #ifdef CONFIG_ACPI_BOOT
 	/* Initialize the ACPI boot-time table parser */
 	acpi_table_init();
@@ -431,22 +438,14 @@
 # endif
 #endif /* CONFIG_APCI_BOOT */
 
-#ifndef XEN
-	find_memory();
-#else
-	io_port_init();
-}
-
-void __init
-late_setup_arch (char **cmdline_p)
-{
-#undef CONFIG_ACPI_BOOT
 	acpi_table_init();
 #endif
 	/* process SAL system table: */
 	ia64_sal_init(efi.sal_systab);
 
 #ifdef CONFIG_SMP
+	init_smp_config ();
+
 	cpu_physical_id(0) = hard_smp_processor_id();
 
 	cpu_set(0, cpu_sibling_map[0]);
@@ -768,6 +767,9 @@
 
 	cpu_data = per_cpu_init();
 
+	printf ("cpu_init: current=%p, current->domain->arch.mm=%p\n",
+		current, current->domain->arch.mm);
+
 	/*
 	 * We set ar.k3 so that assembly code in MCA handler can compute
 	 * physical addresses of per cpu variables with a simple:
@@ -845,6 +847,7 @@
 	/* Clear ITC to eliminiate sched_clock() overflows in human time.  */
 	ia64_set_itc(0);
 
+
 	/* disable all local interrupt sources: */
 	ia64_set_itv(1 << 16);
 	ia64_set_lrr0(1 << 16);
@@ -876,12 +879,23 @@
 		       "stacked regs\n");
 		num_phys_stacked = 96;
 	}
+
 	/* size of physical stacked register partition plus 8 bytes: */
 	__get_cpu_var(ia64_phys_stacked_size_p8) = num_phys_stacked*8 + 8;
 	platform_cpu_init();
 #ifndef XEN
 	pm_idle = default_idle;
 #endif
+
+#ifdef XEN
+    /* surrender usage of kernel registers to domain, use percpu area instead */
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_IO_BASE] = ia64_get_kr(IA64_KR_IO_BASE);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_PER_CPU_DATA] = ia64_get_kr(IA64_KR_PER_CPU_DATA);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT_STACK] = ia64_get_kr(IA64_KR_CURRENT_STACK);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_FPU_OWNER] = ia64_get_kr(IA64_KR_FPU_OWNER);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT] = ia64_get_kr(IA64_KR_CURRENT);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_PT_BASE] = ia64_get_kr(IA64_KR_PT_BASE);
+#endif
 }
 
 void
diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/smp.c
--- a/xen/arch/ia64/linux-xen/smp.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/smp.c	Tue Sep 27 12:48:19 2005
@@ -63,9 +63,18 @@
 //Huh? This seems to be used on ia64 even if !CONFIG_SMP
 void smp_send_event_check_mask(cpumask_t mask)
 {
-	printf("smp_send_event_check_mask called\n");
-	//dummy();
-	//send_IPI_mask(cpu_mask, EVENT_CHECK_VECTOR);
+    int cpu;
+
+    /*  Not for me.  */
+    cpu_clear(smp_processor_id(), mask);
+    if (cpus_empty(mask))
+        return;
+
+    printf("smp_send_event_check_mask called\n");
+
+    for (cpu = 0; cpu < NR_CPUS; ++cpu)
+        if (cpu_isset(cpu, mask))
+	    platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
 }
 
 
@@ -149,6 +158,8 @@
 	int this_cpu = get_cpu();
 	unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
 	unsigned long ops;
+
+	printf ("handle_IPI on %d\n", this_cpu);
 
 	mb();	/* Order interrupt and bit testing. */
 	while ((ops = xchg(pending_ipis, 0)) != 0) {
@@ -249,6 +260,7 @@
 	send_IPI_single(smp_processor_id(), op);
 }
 
+#ifndef XEN
 /*
  * Called with preeemption disabled.
  */
@@ -257,6 +269,7 @@
 {
 	platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
 }
+#endif
 
 void
 smp_flush_tlb_all (void)
diff -r 10b1d30d3f66 xen/arch/ia64/linux-xen/smpboot.c
--- a/xen/arch/ia64/linux-xen/smpboot.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/linux-xen/smpboot.c	Tue Sep 27 12:48:19 2005
@@ -351,6 +351,8 @@
 		BUG();
 	}
 
+	printf ("smp_callin: cpuid=%d, phys_id=%d\n", cpuid, phys_id);
+
 	lock_ipi_calllock();
 	cpu_set(cpuid, cpu_online_map);
 	unlock_ipi_calllock();
@@ -477,9 +479,25 @@
 
 do_rest:
 	task_for_booting_cpu = c_idle.idle;
-#endif
-
-	Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
+#else
+	struct domain *idle;
+	struct vcpu *v;
+	void *stack;
+
+	if ( (idle = do_createdomain(IDLE_DOMAIN_ID, cpu)) == NULL )
+		panic("failed 'createdomain' for CPU %d", cpu);
+	set_bit(_DOMF_idle_domain, &idle->domain_flags);
+	v = idle->vcpu[0];
+
+	printf ("do_boot_cpu: cpu=%d, domain=%p, vcpu=%p\n", cpu, idle, v);
+
+	task_for_booting_cpu = v;
+
+	/* Set cpu number.  */
+	get_thread_info(v)->cpu = cpu;
+#endif
+
+	Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x (task=0x%lx).\n", ap_wakeup_vector, cpu, sapicid, v);
 
 	set_brendez_area(cpu);
 	platform_send_ipi(cpu, ap_wakeup_vector, IA64_IPI_DM_INT, 0);
@@ -487,7 +505,8 @@
 	/*
 	 * Wait 10s total for the AP to start
 	 */
-	Dprintk("Waiting on callin_map ...");
+	Dprintk("Waiting on callin_map (cyc=%lu)...",
+		local_cpu_data->cyc_per_usec);
 	for (timeout = 0; timeout < 100000; timeout++) {
 		if (cpu_isset(cpu, cpu_callin_map))
 			break;  /* It has booted */
diff -r 10b1d30d3f66 xen/arch/ia64/xen/acpi.c
--- a/xen/arch/ia64/xen/acpi.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/acpi.c	Tue Sep 27 12:48:19 2005
@@ -121,6 +121,7 @@
 #ifdef CONFIG_ACPI_BOOT
 
 #define ACPI_MAX_PLATFORM_INTERRUPTS	256
+#define NR_IOSAPICS 4
 
 #if 0
 /* Array to record platform interrupt vectors for generic interrupt routing. */
@@ -162,7 +163,6 @@
 struct acpi_table_madt *	acpi_madt __initdata;
 static u8			has_8259;
 
-#if 0
 static int __init
 acpi_parse_lapic_addr_ovr (
 	acpi_table_entry_header *header, const unsigned long end)
@@ -247,11 +247,12 @@
 
 	acpi_table_print_madt_entry(header);
 
+#if 0
 	iosapic_init(iosapic->address, iosapic->global_irq_base);
-
-	return 0;
-}
-
+#endif
+
+	return 0;
+}
 
 static int __init
 acpi_parse_plat_int_src (
@@ -267,6 +268,7 @@
 
 	acpi_table_print_madt_entry(header);
 
+#if 0
 	/*
 	 * Get vector assignment for this interrupt, set attributes,
 	 * and program the IOSAPIC routing table.
@@ -280,6 +282,7 @@
 						(plintsrc->flags.trigger == 1) ? IOSAPIC_EDGE : IOSAPIC_LEVEL);
 
 	platform_intr_list[plintsrc->type] = vector;
+#endif
 	return 0;
 }
 
@@ -297,12 +300,13 @@
 
 	acpi_table_print_madt_entry(header);
 
+#if 0
 	iosapic_override_isa_irq(p->bus_irq, p->global_irq,
 				 (p->flags.polarity == 1) ? IOSAPIC_POL_HIGH : IOSAPIC_POL_LOW,
 				 (p->flags.trigger == 1) ? IOSAPIC_EDGE : IOSAPIC_LEVEL);
-	return 0;
-}
-
+#endif
+	return 0;
+}
 
 static int __init
 acpi_parse_nmi_src (acpi_table_entry_header *header, const unsigned long end)
@@ -331,8 +335,10 @@
 		 */
 		sal_platform_features |= IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT;
 
+#if 0
 		/*Start cyclone clock*/
 		cyclone_setup(0);
+#endif
 	}
 }
 
@@ -350,7 +356,9 @@
 #else
 	has_8259 = acpi_madt->flags.pcat_compat;
 #endif
+#if 0
 	iosapic_system_init(has_8259);
+#endif
 
 	/* Get base address of IPI Message Block */
 
@@ -364,7 +372,6 @@
 
 	return 0;
 }
-#endif
 
 #ifdef CONFIG_ACPI_NUMA
 
@@ -529,6 +536,7 @@
 	return acpi_register_irq(gsi, polarity, trigger);
 }
 EXPORT_SYMBOL(acpi_register_gsi);
+#endif
 static int __init
 acpi_parse_fadt (unsigned long phys_addr, unsigned long size)
 {
@@ -550,10 +558,11 @@
 	if (fadt->iapc_boot_arch & BAF_LEGACY_DEVICES)
 		acpi_legacy_devices = 1;
 
+#if 0
 	acpi_register_gsi(fadt->sci_int, ACPI_ACTIVE_LOW, ACPI_LEVEL_SENSITIVE);
-	return 0;
-}
-#endif
+#endif
+	return 0;
+}
 
 unsigned long __init
 acpi_find_rsdp (void)
@@ -567,7 +576,6 @@
 	return rsdp_phys;
 }
 
-#if 0
 int __init
 acpi_boot_init (void)
 {
@@ -646,6 +654,7 @@
 	printk(KERN_INFO "%d CPUs available, %d CPUs total\n", available_cpus, total_cpus);
 	return 0;
 }
+#if 0
 int
 acpi_gsi_to_irq (u32 gsi, unsigned int *irq)
 {
diff -r 10b1d30d3f66 xen/arch/ia64/xen/domain.c
--- a/xen/arch/ia64/xen/domain.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/domain.c	Tue Sep 27 12:48:19 2005
@@ -91,8 +91,9 @@
 {
 	if ( hlt_counter == 0 )
 	{
+	  int cpu = smp_processor_id();
 	local_irq_disable();
-	    if ( !softirq_pending(smp_processor_id()) )
+	    if ( !softirq_pending(cpu))
 	        safe_halt();
 	    //else
 		local_irq_enable();
@@ -233,17 +234,21 @@
 	d->arch.breakimm = 0x1000;
 	v->arch.breakimm = d->arch.breakimm;
 
-	d->arch.mm = xmalloc(struct mm_struct);
-	if (unlikely(!d->arch.mm)) {
-		printk("Can't allocate mm_struct for domain %d\n",d->domain_id);
-		return -ENOMEM;
-	}
-	memset(d->arch.mm, 0, sizeof(*d->arch.mm));
-	d->arch.mm->pgd = pgd_alloc(d->arch.mm);
-	if (unlikely(!d->arch.mm->pgd)) {
-		printk("Can't allocate pgd for domain %d\n",d->domain_id);
-		return -ENOMEM;
-	}
+	if (d->domain_id != IDLE_DOMAIN_ID) {
+		d->arch.mm = xmalloc(struct mm_struct);
+		if (unlikely(!d->arch.mm)) {
+			printk("Can't allocate mm_struct for domain %d\n",d->domain_id);
+			return -ENOMEM;
+		}
+		memset(d->arch.mm, 0, sizeof(*d->arch.mm));
+		d->arch.mm->pgd = pgd_alloc(d->arch.mm);
+		if (unlikely(!d->arch.mm->pgd)) {
+			printk("Can't allocate pgd for domain %d\n",d->domain_id);
+			return -ENOMEM;
+		}
+	} else
+		d->arch.mm = NULL;
+	printf ("arch_do_create_domain: domain=%p\n", d);
 }
 
 void arch_getdomaininfo_ctxt(struct vcpu *v, struct vcpu_guest_context *c)
diff -r 10b1d30d3f66 xen/arch/ia64/xen/ivt.S
--- a/xen/arch/ia64/xen/ivt.S	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/ivt.S	Tue Sep 27 12:48:19 2005
@@ -77,7 +77,7 @@
 # define PSR_DEFAULT_BITS	0
 #endif
 
-#if 0
+#if 1
   /*
    * This lets you track the last eight faults that occurred on the CPU.  Make sure ar.k2 isn't
    * needed for something else before enabling this...
@@ -484,6 +484,7 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
 ENTRY(nested_dtlb_miss)
+	DBG_FAULT(5)
 	/*
 	 * In the absence of kernel bugs, we get here when the virtually mapped linear
 	 * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
@@ -552,10 +553,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
 ENTRY(ikey_miss)
+	DBG_FAULT(6)
 #ifdef XEN
 	REFLECT(6)
 #endif
-	DBG_FAULT(6)
 	FAULT(6)
 END(ikey_miss)
 
@@ -597,10 +598,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
 ENTRY(dkey_miss)
+	DBG_FAULT(7)
 #ifdef XEN
 	REFLECT(7)
 #endif
-	DBG_FAULT(7)
 	FAULT(7)
 END(dkey_miss)
 
@@ -608,10 +609,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
 ENTRY(dirty_bit)
+	DBG_FAULT(8)
 #ifdef XEN
 	REFLECT(8)
 #endif
-	DBG_FAULT(8)
 	/*
 	 * What we do here is to simply turn on the dirty bit in the PTE.  We need to
 	 * update both the page-table and the TLB entry.  To efficiently access the PTE,
@@ -673,6 +674,7 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
 ENTRY(iaccess_bit)
+	DBG_FAULT(9)
 #ifdef XEN
 	mov r31=pr;
 	mov r16=cr.isr
@@ -681,7 +683,6 @@
 	movl r20=0x2400
 	br.sptk.many fast_access_reflect;;
 #endif
-	DBG_FAULT(9)
 	// Like Entry 8, except for instruction access
 	mov r16=cr.ifa				// get the address that caused the fault
 	movl r30=1f				// load continuation point in case of nested fault
@@ -746,6 +747,7 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
 ENTRY(daccess_bit)
+	DBG_FAULT(10)
 #ifdef XEN
 	mov r31=pr;
 	mov r16=cr.isr
@@ -754,7 +756,6 @@
 	movl r20=0x2800
 	br.sptk.many fast_access_reflect;;
 #endif
-	DBG_FAULT(10)
 	// Like Entry 8, except for data access
 	mov r16=cr.ifa				// get the address that caused the fault
 	movl r30=1f				// load continuation point in case of nested fault
@@ -1360,10 +1361,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
 ENTRY(page_not_present)
+	DBG_FAULT(20)
 #ifdef XEN
 	REFLECT(20)
 #endif
-	DBG_FAULT(20)
 	mov r16=cr.ifa
 	rsm psr.dt
 	/*
@@ -1383,10 +1384,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
 ENTRY(key_permission)
+	DBG_FAULT(21)
 #ifdef XEN
 	REFLECT(21)
 #endif
-	DBG_FAULT(21)
 	mov r16=cr.ifa
 	rsm psr.dt
 	mov r31=pr
@@ -1399,10 +1400,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
 ENTRY(iaccess_rights)
+	DBG_FAULT(22)
 #ifdef XEN
 	REFLECT(22)
 #endif
-	DBG_FAULT(22)
 	mov r16=cr.ifa
 	rsm psr.dt
 	mov r31=pr
@@ -1415,6 +1416,7 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
 ENTRY(daccess_rights)
+	DBG_FAULT(23)
 #ifdef XEN
 	mov r31=pr;
 	mov r16=cr.isr
@@ -1423,7 +1425,6 @@
 	movl r20=0x5300
 	br.sptk.many fast_access_reflect;;
 #endif
-	DBG_FAULT(23)
 	mov r16=cr.ifa
 	rsm psr.dt
 	mov r31=pr
@@ -1456,10 +1457,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
 ENTRY(disabled_fp_reg)
+	DBG_FAULT(25)
 #ifdef XEN
 	REFLECT(25)
 #endif
-	DBG_FAULT(25)
 	rsm psr.dfh		// ensure we can access fph
 	;;
 	srlz.d
@@ -1472,10 +1473,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
 ENTRY(nat_consumption)
+	DBG_FAULT(26)
 #ifdef XEN
 	REFLECT(26)
 #endif
-	DBG_FAULT(26)
 	FAULT(26)
 END(nat_consumption)
 
@@ -1483,11 +1484,11 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5700 Entry 27 (size 16 bundles) Speculation (40)
 ENTRY(speculation_vector)
+	DBG_FAULT(27)
 #ifdef XEN
 	// this probably need not reflect...
 	REFLECT(27)
 #endif
-	DBG_FAULT(27)
 	/*
 	 * A [f]chk.[as] instruction needs to take the branch to the recovery code but
 	 * this part of the architecture is not implemented in hardware on some CPUs, such
@@ -1530,10 +1531,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
 ENTRY(debug_vector)
+	DBG_FAULT(29)
 #ifdef XEN
 	REFLECT(29)
 #endif
-	DBG_FAULT(29)
 	FAULT(29)
 END(debug_vector)
 
@@ -1541,10 +1542,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
 ENTRY(unaligned_access)
+	DBG_FAULT(30)
 #ifdef XEN
 	REFLECT(30)
 #endif
-	DBG_FAULT(30)
 	mov r16=cr.ipsr
 	mov r31=pr		// prepare to save predicates
 	;;
@@ -1555,10 +1556,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
 ENTRY(unsupported_data_reference)
+	DBG_FAULT(31)
 #ifdef XEN
 	REFLECT(31)
 #endif
-	DBG_FAULT(31)
 	FAULT(31)
 END(unsupported_data_reference)
 
@@ -1566,10 +1567,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
 ENTRY(floating_point_fault)
+	DBG_FAULT(32)
 #ifdef XEN
 	REFLECT(32)
 #endif
-	DBG_FAULT(32)
 	FAULT(32)
 END(floating_point_fault)
 
@@ -1577,10 +1578,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
 ENTRY(floating_point_trap)
+	DBG_FAULT(33)
 #ifdef XEN
 	REFLECT(33)
 #endif
-	DBG_FAULT(33)
 	FAULT(33)
 END(floating_point_trap)
 
@@ -1588,10 +1589,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
 ENTRY(lower_privilege_trap)
+	DBG_FAULT(34)
 #ifdef XEN
 	REFLECT(34)
 #endif
-	DBG_FAULT(34)
 	FAULT(34)
 END(lower_privilege_trap)
 
@@ -1599,10 +1600,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
 ENTRY(taken_branch_trap)
+	DBG_FAULT(35)
 #ifdef XEN
 	REFLECT(35)
 #endif
-	DBG_FAULT(35)
 	FAULT(35)
 END(taken_branch_trap)
 
@@ -1610,10 +1611,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
 ENTRY(single_step_trap)
+	DBG_FAULT(36)
 #ifdef XEN
 	REFLECT(36)
 #endif
-	DBG_FAULT(36)
 	FAULT(36)
 END(single_step_trap)
 
@@ -1669,10 +1670,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77)
 ENTRY(ia32_exception)
+	DBG_FAULT(45)
 #ifdef XEN
 	REFLECT(45)
 #endif
-	DBG_FAULT(45)
 	FAULT(45)
 END(ia32_exception)
 
@@ -1680,10 +1681,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept  (30,31,59,70,71)
 ENTRY(ia32_intercept)
+	DBG_FAULT(46)
 #ifdef XEN
 	REFLECT(46)
 #endif
-	DBG_FAULT(46)
 #ifdef	CONFIG_IA32_SUPPORT
 	mov r31=pr
 	mov r16=cr.isr
@@ -1713,10 +1714,10 @@
 /////////////////////////////////////////////////////////////////////////////////////////
 // 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt  (74)
 ENTRY(ia32_interrupt)
+	DBG_FAULT(47)
 #ifdef XEN
 	REFLECT(47)
 #endif
-	DBG_FAULT(47)
 #ifdef CONFIG_IA32_SUPPORT
 	mov r31=pr
 	br.sptk.many dispatch_to_ia32_handler
diff -r 10b1d30d3f66 xen/arch/ia64/xen/process.c
--- a/xen/arch/ia64/xen/process.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/process.c	Tue Sep 27 12:48:19 2005
@@ -61,11 +61,20 @@
 	return 0;
 }
 
+#include <xen/sched-if.h>
+
+extern struct schedule_data schedule_data[NR_CPUS];
+
+
 void schedule_tail(struct vcpu *next)
 {
 	unsigned long rr7;
 	//printk("current=%lx,shared_info=%lx\n",current,current->vcpu_info);
 	//printk("next=%lx,shared_info=%lx\n",next,next->vcpu_info);
+
+	// TG: Real HACK FIXME.
+	spin_unlock_irq(&schedule_data[current->processor].schedule_lock);
+
 #ifdef CONFIG_VTI
 	/* rr7 will be postponed to last point when resuming back to guest */
 	vmx_load_all_rr(current);
@@ -541,6 +550,10 @@
 	arg0 = vcpu_get_gr(current,32);
 	switch(ssc) {
 	    case SSC_PUTCHAR:
+#if 0
+	        if (spin_is_locked (&schedule_data[0].schedule_lock))
+		  printf ("??? sched spin locked\n");
+#endif
 		buf[0] = arg0;
 		buf[1] = '\0';
 		printf(buf);
diff -r 10b1d30d3f66 xen/arch/ia64/xen/xenirq.c
--- a/xen/arch/ia64/xen/xenirq.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/xenirq.c	Tue Sep 27 12:48:19 2005
@@ -35,7 +35,7 @@
 int
 xen_do_IRQ(ia64_vector vector)
 {
-	if (vector != 0xef) {
+	if (vector != IA64_TIMER_VECTOR && vector != IA64_IPI_VECTOR) {
 		extern void vcpu_pend_interrupt(void *, int);
 #if 0
 		if (firsttime[vector]) {
diff -r 10b1d30d3f66 xen/arch/ia64/xen/xenmisc.c
--- a/xen/arch/ia64/xen/xenmisc.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/xenmisc.c	Tue Sep 27 12:48:19 2005
@@ -295,7 +295,7 @@
 //prev->domain->domain_id,(long)prev&0xffffff,next->domain->domain_id,(long)next&0xffffff);
 //if (prev->domain->domain_id == 1 && next->domain->domain_id == 0) cs10foo();
 //if (prev->domain->domain_id == 0 && next->domain->domain_id == 1) cs01foo();
-//printk("@@sw %d->%d\n",prev->domain->domain_id,next->domain->domain_id);
+  printk("@@sw %d->%d\n",prev->domain->domain_id,next->domain->domain_id);
 #ifdef CONFIG_VTI
 	vtm_domain_out(prev);
 #endif
diff -r 10b1d30d3f66 xen/arch/ia64/xen/xensetup.c
--- a/xen/arch/ia64/xen/xensetup.c	Thu Sep  8 15:18:40 2005
+++ b/xen/arch/ia64/xen/xensetup.c	Tue Sep 27 12:48:19 2005
@@ -252,11 +252,22 @@
 printk("About to call scheduler_init()\n");
     scheduler_init();
     local_irq_disable();
+    init_IRQ ();
 printk("About to call init_xen_time()\n");
     init_xen_time(); /* initialise the time */
 printk("About to call ac_timer_init()\n");
     ac_timer_init();
-// init_xen_time(); ???
+
+#if 0 /* Done in cpu_init */
+    /* surrender usage of kernel registers to domain, use percpu area instead */
+    /* This must be done before enabling interrupts.  */
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_IO_BASE] = ia64_get_kr(IA64_KR_IO_BASE);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_PER_CPU_DATA] = ia64_get_kr(IA64_KR_PER_CPU_DATA);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT_STACK] = ia64_get_kr(IA64_KR_CURRENT_STACK);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_FPU_OWNER] = ia64_get_kr(IA64_KR_FPU_OWNER);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT] = ia64_get_kr(IA64_KR_CURRENT);
+    __get_cpu_var(cpu_kr)._kr[IA64_KR_PT_BASE] = ia64_get_kr(IA64_KR_PT_BASE);
+#endif
 
 #ifdef CONFIG_SMP
     if ( opt_nosmp )
@@ -274,6 +285,9 @@
         cpu_set(i, cpu_present_map);
 
     //BUG_ON(!local_irq_is_enabled());
+
+    /*  Enable IRQ to receive IPI (needed for ITC sync).  */
+    local_irq_enable();
 
 printk("num_online_cpus=%d, max_cpus=%d\n",num_online_cpus(),max_cpus);
     for_each_present_cpu ( i )
@@ -286,24 +300,16 @@
 	}
     }
 
+    local_irq_disable();
+
     printk("Brought up %ld CPUs\n", (long)num_online_cpus());
     smp_cpus_done(max_cpus);
 #endif
 
-
-	// FIXME: Should the following be swapped and moved later?
-    schedulers_start();
     do_initcalls();
 printk("About to call sort_main_extable()\n");
     sort_main_extable();
 
-    /* surrender usage of kernel registers to domain, use percpu area instead */
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_IO_BASE] = ia64_get_kr(IA64_KR_IO_BASE);
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_PER_CPU_DATA] = ia64_get_kr(IA64_KR_PER_CPU_DATA);
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT_STACK] = ia64_get_kr(IA64_KR_CURRENT_STACK);
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_FPU_OWNER] = ia64_get_kr(IA64_KR_FPU_OWNER);
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_CURRENT] = ia64_get_kr(IA64_KR_CURRENT);
-    __get_cpu_var(cpu_kr)._kr[IA64_KR_PT_BASE] = ia64_get_kr(IA64_KR_PT_BASE);
 
     /* Create initial domain 0. */
 printk("About to call do_createdomain()\n");
@@ -341,6 +347,11 @@
                         0,
 			0) != 0)
         panic("Could not set up DOM0 guest OS\n");
+
+    /* PIN domain0 on CPU 0.  */
+    dom0->vcpu[0]->cpumap=1;
+    set_bit(_VCPUF_cpu_pinned, &dom0->vcpu[0]->vcpu_flags);
+
 #ifdef CLONE_DOMAIN0
     {
     int i;
@@ -379,8 +390,14 @@
     }
 #endif
     domain_unpause_by_systemcontroller(dom0);
+
+    printf("About to call schedulers_start dom0=%p, idle0_dom=%p\n",
+	   dom0, &idle0_domain);
+    schedulers_start();
+
     domain0_ready = 1;
     local_irq_enable();
+
 printk("About to call startup_cpu_idle_loop()\n");
     startup_cpu_idle_loop();
 }
diff -r 10b1d30d3f66 xen/include/asm-ia64/config.h
--- a/xen/include/asm-ia64/config.h	Thu Sep  8 15:18:40 2005
+++ b/xen/include/asm-ia64/config.h	Tue Sep 27 12:48:19 2005
@@ -21,7 +21,7 @@
 #define CONFIG_EFI_PCDP
 #define CONFIG_SERIAL_SGI_L1_CONSOLE
 
-#undef CONFIG_XEN_SMP
+#define CONFIG_XEN_SMP
 
 #ifdef CONFIG_XEN_SMP
 #define CONFIG_SMP 1
@@ -120,8 +120,7 @@
 #ifdef CONFIG_SMP
 #warning "Lots of things to fix to enable CONFIG_SMP!"
 #endif
-// FIXME SMP
-#define	get_cpu()	0
+#define	get_cpu()	smp_processor_id()
 #define put_cpu()	do {} while(0)
 
 // needed for common/dom0_ops.c until hyperthreading is supported
@@ -137,6 +136,7 @@
 // function calls; see decl in xen/include/xen/sched.h
 #undef free_task_struct
 #undef alloc_task_struct
+#define get_thread_info(v) alloc_thread_info(v)
 
 // initial task has a different name in Xen
 //#define	idle0_task	init_task
@@ -296,7 +296,11 @@
 #endif /* __XEN_IA64_CONFIG_H__ */
 
 // needed for include/xen/smp.h
+#ifdef CONFIG_SMP
+#define __smp_processor_id()	current_thread_info()->cpu
+#else
 #define __smp_processor_id()	0
+#endif
 
 
 // FOLLOWING ADDED FOR XEN POST-NGIO and/or LINUX 2.6.7
diff -r 10b1d30d3f66 xen/include/asm-ia64/linux-xen/asm/spinlock.h
--- a/xen/include/asm-ia64/linux-xen/asm/spinlock.h	Thu Sep  8 15:18:40 2005
+++ b/xen/include/asm-ia64/linux-xen/asm/spinlock.h	Tue Sep 27 12:48:19 2005
@@ -17,10 +17,15 @@
 #include <asm/intrinsics.h>
 #include <asm/system.h>
 
+#define DEBUG_SPINLOCK
+
 typedef struct {
 	volatile unsigned int lock;
 #ifdef CONFIG_PREEMPT
 	unsigned int break_lock;
+#endif
+#ifdef DEBUG_SPINLOCK
+	void *locker;
 #endif
 #ifdef XEN
 	unsigned char recurse_cpu;
@@ -95,6 +100,10 @@
 		      "(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;"
 		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # endif /* CONFIG_MCKINLEY */
+#endif
+
+#ifdef DEBUG_SPINLOCK
+	asm volatile ("mov %0=ip" : "=r" (lock->locker));
 #endif
 }
 #define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
diff -r 10b1d30d3f66 xen/include/asm-ia64/linux-xen/linux/hardirq.h
--- a/xen/include/asm-ia64/linux-xen/linux/hardirq.h	Thu Sep  8 15:18:40 2005
+++ b/xen/include/asm-ia64/linux-xen/linux/hardirq.h	Tue Sep 27 12:48:19 2005
@@ -67,11 +67,7 @@
  */
 #define in_irq()		(hardirq_count())
 #define in_softirq()		(softirq_count())
-#ifdef XEN
-#define in_interrupt()		0 		// FIXME SMP LATER
-#else
 #define in_interrupt()		(irq_count())
-#endif
 
 #if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
 # define in_atomic()	((preempt_count() & ~PREEMPT_ACTIVE) != kernel_locked())