|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v2] x86/smt: Support for enabling/disabling SMT at runtime
Currently, a user can in combine the output of `xl info -n`, the APCI tables,
and some manual CPUID data to figure out which CPU numbers to feed into
`xen-hptool cpu-offline` to effectively disable SMT at runtime.
A more convenient option is to teach Xen how to perform this action.
Extend XEN_SYSCTL_cpu_hotplug with two new operations. Introduce a new
smt_up_down_helper() which wrap the cpu_{up,down}_helper() helpers with logic
which understands siblings based on their APIC_ID.
Add libxc stubs, and extend xen-hptool with smt-{enable,disable} options.
These are intended to be shorthands for a loop over cpu-{online,offline}. It
is intended for use in production scenarios where debugging options such as
`maxcpus=` or other manual plug/unplug configuration has not been used.
Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
---
CC: Jan Beulich <JBeulich@xxxxxxxx>
CC: Wei Liu <wei.liu2@xxxxxxxxxx>
CC: Roger Pau Monné <roger.pau@xxxxxxxxxx>
v2:
* Fold smt_{up,down}_helper() into smt_up_down_helper()
* Fix calculation of sibling_mask
* More clearly document the behaviour and expected restrictions.
While testing this, I've found a further bug. This is preexisting and can be
provoked on staging, although it is easier to provoke with this patch.
(XEN) Assertion 'entry->prev->next == entry' failed at
/local/xen.git/xen/include/xen/list.h:184
(XEN) ----[ Xen-4.13-unstable x86_64 debug=y Not tainted ]----
(XEN) CPU: 0
(XEN) RIP: e008:[<ffff82d080241fad>] migrate_timer+0x1df/0x2da
(XEN) RFLAGS: 0000000000010087 CONTEXT: hypervisor
(XEN) rax: ffff830837088038 rbx: ffff830575928890 rcx: ffff83084d69e320
(XEN) rdx: ffff830575928898 rsi: ffff82d08094002f rdi: ffff830575928890
(XEN) rbp: ffff8300abc27d28 rsp: ffff8300abc27cc8 r8: 0000000000000000
(XEN) r9: ffff830575928780 r10: ffff830575928010 r11: 0000000000000000
(XEN) r12: 0000000000000000 r13: 0000000000000000 r14: ffff82d080962080
(XEN) r15: ffff82d080962080 cr0: 000000008005003b cr4: 00000000001526e0
(XEN) cr3: 00000000abc17000 cr2: ffff880072a9f8c0
(XEN) fsb: 0000000000000000 gsb: ffff88007ce00000 gss: 0000000000000000
(XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: e010 cs: e008
(XEN) Xen code around <ffff82d080241fad> (migrate_timer+0x1df/0x2da):
(XEN) 8b 4b 10 48 3b 11 74 02 <0f> 0b 48 89 48 08 48 89 01 48 b8 00 01 10 00 01
(XEN) Xen stack trace from rsp=ffff8300abc27cc8:
<snip>
(XEN) Xen call trace:
(XEN) [<ffff82d080241fad>] migrate_timer+0x1df/0x2da
(XEN) [<ffff82d0802998b7>] do_IRQ+0x4fc/0x64e
(XEN) [<ffff82d0803988aa>] common_interrupt+0x10a/0x120
(XEN) [<ffff82d0802f861c>] mwait-idle.c#mwait_idle+0x31d/0x370
(XEN) [<ffff82d080289499>] domain.c#idle_loop+0xa8/0xc3
(XEN)
(XEN)
(XEN) ****************************************
(XEN) Panic on CPU 0:
(XEN) Assertion 'entry->prev->next == entry' failed at
/local/xen.git/xen/include/xen/list.h:184
(XEN) ****************************************
---
tools/libxc/include/xenctrl.h | 2 ++
tools/libxc/xc_cpu_hotplug.c | 26 ++++++++++++++++++++
tools/misc/xen-hptool.c | 56 ++++++++++++++++++++++++++++++++++++++++++
xen/arch/x86/sysctl.c | 57 ++++++++++++++++++++++++++++++++++++++++++-
xen/include/public/sysctl.h | 19 +++++++++++++++
5 files changed, 159 insertions(+), 1 deletion(-)
diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index a3628e5..49a6b2a 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -1854,6 +1854,8 @@ int xc_pm_reset_cxstat(xc_interface *xch, int cpuid);
int xc_cpu_online(xc_interface *xch, int cpu);
int xc_cpu_offline(xc_interface *xch, int cpu);
+int xc_smt_enable(xc_interface *xch);
+int xc_smt_disable(xc_interface *xch);
/*
* cpufreq para name of this structure named
diff --git a/tools/libxc/xc_cpu_hotplug.c b/tools/libxc/xc_cpu_hotplug.c
index 58c2a0f..2ea9825 100644
--- a/tools/libxc/xc_cpu_hotplug.c
+++ b/tools/libxc/xc_cpu_hotplug.c
@@ -46,3 +46,29 @@ int xc_cpu_offline(xc_interface *xch, int cpu)
return ret;
}
+int xc_smt_enable(xc_interface *xch)
+{
+ DECLARE_SYSCTL;
+ int ret;
+
+ sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+ sysctl.u.cpu_hotplug.cpu = 0;
+ sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+ ret = xc_sysctl(xch, &sysctl);
+
+ return ret;
+}
+
+int xc_smt_disable(xc_interface *xch)
+{
+ DECLARE_SYSCTL;
+ int ret;
+
+ sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+ sysctl.u.cpu_hotplug.cpu = 0;
+ sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE;
+ ret = xc_sysctl(xch, &sysctl);
+
+ return ret;
+}
+
diff --git a/tools/misc/xen-hptool.c b/tools/misc/xen-hptool.c
index 40cd966..6e27d9c 100644
--- a/tools/misc/xen-hptool.c
+++ b/tools/misc/xen-hptool.c
@@ -19,6 +19,8 @@ void show_help(void)
" mem-online <mfn> online MEMORY <mfn>\n"
" mem-offline <mfn> offline MEMORY <mfn>\n"
" mem-status <mfn> query Memory status<mfn>\n"
+ " smt-enable onlines all SMT threads\n"
+ " smt-disable offlines all SMT threads\n"
);
}
@@ -304,6 +306,58 @@ static int hp_cpu_offline_func(int argc, char *argv[])
return ret;
}
+static int main_smt_enable(int argc, char *argv[])
+{
+ int ret;
+
+ if ( argc )
+ {
+ show_help();
+ return -1;
+ }
+
+ for ( ;; )
+ {
+ ret = xc_smt_enable(xch);
+ if ( (ret >= 0) || (errno != EBUSY) )
+ break;
+ }
+
+ if ( ret < 0 )
+ fprintf(stderr, "Unable to enable SMT: errno %d, %s\n",
+ errno, strerror(errno));
+ else
+ printf("Enabled SMT\n");
+
+ return ret;
+}
+
+static int main_smt_disable(int argc, char *argv[])
+{
+ int ret;
+
+ if ( argc )
+ {
+ show_help();
+ return -1;
+ }
+
+ for ( ;; )
+ {
+ ret = xc_smt_disable(xch);
+ if ( (ret >= 0) || (errno != EBUSY) )
+ break;
+ }
+
+ if ( ret < 0 )
+ fprintf(stderr, "Unable to disable SMT: errno %d, %s\n",
+ errno, strerror(errno));
+ else
+ printf("Disabled SMT\n");
+
+ return ret;
+}
+
struct {
const char *name;
int (*function)(int argc, char *argv[]);
@@ -314,6 +368,8 @@ struct {
{ "mem-status", hp_mem_query_func},
{ "mem-online", hp_mem_online_func},
{ "mem-offline", hp_mem_offline_func},
+ { "smt-enable", main_smt_enable },
+ { "smt-disable", main_smt_disable },
};
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index cff4415..8acb5f8 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -114,6 +114,48 @@ long cpu_down_helper(void *data)
return ret;
}
+static long smt_up_down_helper(void *data)
+{
+ bool up = (bool)data;
+ unsigned int cpu, sibling_mask = boot_cpu_data.x86_num_siblings - 1;
+ int ret = 0;
+
+ opt_smt = up;
+
+ for_each_present_cpu ( cpu )
+ {
+ /* Skip primary siblings (those whose thread id is 0). */
+ if ( !(x86_cpu_to_apicid[cpu] & sibling_mask) )
+ continue;
+
+ ret = up ? cpu_up_helper(_p(cpu))
+ : cpu_down_helper(_p(cpu));
+
+ if ( ret && ret != -EEXIST )
+ break;
+
+ /*
+ * Ensure forward progress by only considering preemption when we have
+ * changed the state of one or more cpus.
+ */
+ if ( ret != -EEXIST && general_preempt_check() )
+ {
+ /* In tasklet context - can't create a contination. */
+ ret = -EBUSY;
+ break;
+ }
+
+ ret = 0; /* Avoid exiting with -EEXIST in the success case. */
+ }
+
+ if ( !ret )
+ printk(XENLOG_INFO "SMT %s - online CPUs {%*pbl}\n",
+ up ? "enabled" : "disabled",
+ nr_cpu_ids, cpumask_bits(&cpu_online_map));
+
+ return ret;
+}
+
void arch_do_physinfo(struct xen_sysctl_physinfo *pi)
{
memcpy(pi->hw_cap, boot_cpu_data.x86_capability,
@@ -137,11 +179,12 @@ long arch_do_sysctl(
case XEN_SYSCTL_cpu_hotplug:
{
unsigned int cpu = sysctl->u.cpu_hotplug.cpu;
+ unsigned int op = sysctl->u.cpu_hotplug.op;
bool plug;
long (*fn)(void *);
void *hcpu;
- switch ( sysctl->u.cpu_hotplug.op )
+ switch ( op )
{
case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
plug = true;
@@ -155,6 +198,18 @@ long arch_do_sysctl(
hcpu = _p(cpu);
break;
+ case XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE:
+ case XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE:
+ if ( !cpu_has_htt || boot_cpu_data.x86_num_siblings < 2 )
+ {
+ ret = -EOPNOTSUPP;
+ break;
+ }
+ plug = op == XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+ fn = smt_up_down_helper;
+ hcpu = _p(plug);
+ break;
+
default:
ret = -EOPNOTSUPP;
break;
diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
index c49b4dc..9e8903d 100644
--- a/xen/include/public/sysctl.h
+++ b/xen/include/public/sysctl.h
@@ -246,8 +246,27 @@ struct xen_sysctl_get_pmstat {
struct xen_sysctl_cpu_hotplug {
/* IN variables */
uint32_t cpu; /* Physical cpu. */
+
+ /* Single CPU enable/disable. */
#define XEN_SYSCTL_CPU_HOTPLUG_ONLINE 0
#define XEN_SYSCTL_CPU_HOTPLUG_OFFLINE 1
+
+ /*
+ * SMT enable/disable.
+ *
+ * These two ops loop over all present CPUs, and either online or offline
+ * every non-primary sibling thread (those with a thread id which is not
+ * 0).
+ *
+ * They are intended as a shorthand for identifying and feeding the cpu
+ * numbers individually to HOTPLUG_{ON,OFF}LINE.
+ *
+ * These are not expected to be used in conjunction with debugging options
+ * such as `maxcpus=` or when other manual configuration of offline cpus
+ * is in use.
+ */
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE 2
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE 3
uint32_t op; /* hotplug opcode */
};
--
2.1.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |