Re: [PATCH 2/2] x86/xen: Allow per-domain usage of hardware virtualized APIC

[Anthony PERARD <anthony.perard@xxxxxxxxxx>]

On Thu, Jan 27, 2022 at 04:01:33PM +0000, Jane Malalane wrote:
> diff --git a/docs/man/xl.conf.5.pod.in b/docs/man/xl.conf.5.pod.in
> index df20c08137..2d0a59d019 100644
> --- a/docs/man/xl.conf.5.pod.in
> +++ b/docs/man/xl.conf.5.pod.in
> @@ -107,6 +107,18 @@ Sets the default value for the C<max_grant_version> 
> domain config value.
>  
>  Default: maximum grant version supported by the hypervisor.
>  
> +=item B<assisted_xapic=BOOLEAN>
> +
> +If enabled, domains will use xAPIC hardware assisted emulation by default.
> +
> +Default: enabled.
> +
> +=item B<assisted_x2apic=BOOLEAN>
> +
> +If enabled, domains will use x2APIC hardware assisted emulation by default.
> +
> +Default: enabled.
> +
>  =item B<vif.default.script="PATH">
>  
>  Configures the default hotplug script used by virtual network devices.
> diff --git a/tools/libs/light/libxl_arm.c b/tools/libs/light/libxl_arm.c
> index 52f2545498..4d422bef96 100644
> --- a/tools/libs/light/libxl_arm.c
> +++ b/tools/libs/light/libxl_arm.c
> @@ -1384,8 +1384,9 @@ void 
> libxl__arch_domain_create_info_setdefault(libxl__gc *gc,
>      }
>  }
>  
> -void libxl__arch_domain_build_info_setdefault(libxl__gc *gc,
> -                                              libxl_domain_build_info 
> *b_info)
> +int libxl__arch_domain_build_info_setdefault(libxl__gc *gc,
> +                                             libxl_domain_build_info *b_info,
> +                                             const libxl_physinfo *physinfo)
>  {
>      /* ACPI is disabled by default */
>      libxl_defbool_setdefault(&b_info->acpi, false);

It seems there's a missing 'return 0' in this function now ;-).

> diff --git a/tools/libs/light/libxl_create.c b/tools/libs/light/libxl_create.c
> index d7a40d7550..2bae6fef62 100644
> --- a/tools/libs/light/libxl_create.c
> +++ b/tools/libs/light/libxl_create.c
> @@ -264,7 +264,6 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
>      if (!b_info->event_channels)
>          b_info->event_channels = 1023;
>  
> -    libxl__arch_domain_build_info_setdefault(gc, b_info);

I don't think moving this call later is a good idea. It looks like it's
going to break on ARM at least. Instead, calling libxl_get_physinfo()
earlier should be fine I think.

>      libxl_defbool_setdefault(&b_info->dm_restrict, false);
>  
>      if (b_info->iommu_memkb == LIBXL_MEMKB_DEFAULT)
> @@ -456,15 +455,21 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
>          libxl_defbool_setdefault(&b_info->nested_hvm,               false);
>      }
>  
> -    if (b_info->max_grant_version == LIBXL_MAX_GRANT_DEFAULT) {
> -        libxl_physinfo info;
> +    libxl_physinfo info;
>  
> -        rc = libxl_get_physinfo(CTX, &info);
> -        if (rc) {
> -            LOG(ERROR, "failed to get hypervisor info");
> -            return rc;
> -        }
> +    rc = libxl_get_physinfo(CTX, &info);
> +    if (rc) {
> +        LOG(ERROR, "failed to get hypervisor info");
> +        return rc;
> +    }
>  
> +    rc = libxl__arch_domain_build_info_setdefault(gc, b_info, &info);
> +    if (rc) {
> +        LOG(ERROR, "unable to set domain arch build info defaults");
> +        return rc;
> +    }
> +
> +    if (b_info->max_grant_version == LIBXL_MAX_GRANT_DEFAULT) {
>          if (info.cap_gnttab_v2)
>              b_info->max_grant_version = 2;
>          else if (info.cap_gnttab_v1)
> diff --git a/tools/libs/light/libxl_types.idl 
> b/tools/libs/light/libxl_types.idl
> index 42ac6c357b..db5eb0a0b3 100644
> --- a/tools/libs/light/libxl_types.idl
> +++ b/tools/libs/light/libxl_types.idl
> @@ -648,6 +648,8 @@ libxl_domain_build_info = Struct("domain_build_info",[
>                                 ("vuart", libxl_vuart_type),
>                                ])),
>      ("arch_x86", Struct(None, [("msr_relaxed", libxl_defbool),
> +                               ("assisted_xapic", libxl_defbool),
> +                               ("assisted_x2apic", libxl_defbool),

Could you add some LIBXL_HAVE_* macro in libxl.h about those new fields
like you did in the previous patch?

>                                ])),
>      # Alternate p2m is not bound to any architecture or guest type, as it is
>      # supported by x86 HVM and ARM support is planned.
> diff --git a/tools/libs/light/libxl_x86.c b/tools/libs/light/libxl_x86.c
> index 33da51fe89..b257fca756 100644
> --- a/tools/libs/light/libxl_x86.c
> +++ b/tools/libs/light/libxl_x86.c
> @@ -819,11 +825,44 @@ void 
> libxl__arch_domain_create_info_setdefault(libxl__gc *gc,
>  {
>  }
>  
> -void libxl__arch_domain_build_info_setdefault(libxl__gc *gc,
> -                                              libxl_domain_build_info 
> *b_info)
> +int libxl__arch_domain_build_info_setdefault(libxl__gc *gc,
> +                                             libxl_domain_build_info *b_info,
> +                                             const libxl_physinfo *physinfo)
>  {
> +    int rc;
> +    bool assisted_xapic;
> +    bool assisted_x2apic;
> +
>      libxl_defbool_setdefault(&b_info->acpi, true);
>      libxl_defbool_setdefault(&b_info->arch_x86.msr_relaxed, false);
> +
> +    libxl_defbool_setdefault(&b_info->arch_x86.assisted_xapic, false);
> +    libxl_defbool_setdefault(&b_info->arch_x86.assisted_x2apic, false);
> +
> +    assisted_xapic = libxl_defbool_val(b_info->arch_x86.assisted_xapic);
> +    assisted_x2apic = libxl_defbool_val(b_info->arch_x86.assisted_x2apic);
> +
> +    if ((assisted_xapic || assisted_x2apic) &&
> +        b_info->type == LIBXL_DOMAIN_TYPE_PV)
> +    {
> +        LOG(ERROR, "Interrupt Controller Virtualization not supported for 
> PV");
> +        rc = ERROR_INVAL;
> +        goto out;
> +    }
> +
> +    if ((assisted_xapic && !physinfo->cap_assisted_xapic) ||
> +         (assisted_x2apic && !physinfo->cap_assisted_x2apic))
> +    {
> +        LOG(ERROR, "x%sAPIC hardware supported emulation not available",
> +            assisted_xapic && !physinfo->cap_assisted_xapic ? "" : "2");
> +        rc =  ERROR_INVAL;
> +        goto out;
> +    }

Would it make sens to enable assisted_xapic and assisted_x2apic by
default based on hardware support? That way users of libxl could benefit
from the upgrade without having to enable it. Or maybe that could causes
issues, like maybe on migration?

> +
> +    rc = 0;
> +out:
> +    return rc;
> +
>  }
>  
>  int libxl__arch_passthrough_mode_setdefault(libxl__gc *gc,
> diff --git a/tools/xl/xl.c b/tools/xl/xl.c
> index 2d1ec18ea3..b97e491c9c 100644
> --- a/tools/xl/xl.c
> +++ b/tools/xl/xl.c
> @@ -57,6 +57,8 @@ int max_grant_frames = -1;
>  int max_maptrack_frames = -1;
>  int max_grant_version = LIBXL_MAX_GRANT_DEFAULT;
>  libxl_domid domid_policy = INVALID_DOMID;
> +int assisted_xapic = 0;
> +int assisted_x2apic = 0;

This seems to contradict the manual which says "Default: enabled".



I think you need one more change in ocaml bindings, its ABI checker
fails. I found the following change to work, hopefully that the right
thing to do:

diff --git a/tools/ocaml/libs/xc/xenctrl_stubs.c 
b/tools/ocaml/libs/xc/xenctrl_stubs.c
index 5b4fe72c8d..0aa957d379 100644
--- a/tools/ocaml/libs/xc/xenctrl_stubs.c
+++ b/tools/ocaml/libs/xc/xenctrl_stubs.c
@@ -239,7 +239,7 @@ CAMLprim value stub_xc_domain_create(value xch, value 
wanted_domid, value config

                cfg.arch.misc_flags = ocaml_list_to_c_bitmap
                        /* ! x86_arch_misc_flags X86_ none */
-                       /* ! XEN_X86_ XEN_X86_MSR_RELAXED all */
+                       /* ! XEN_X86_ XEN_X86_ASSISTED_X2APIC max */
                        (VAL_MISC_FLAGS);

 #undef VAL_MISC_FLAGS

Thanks,

-- 
Anthony PERARD