[PATCH 23/22] x86/vmx: Adjust NMI handling for FRED

[Andrew Cooper <andrew.cooper3@xxxxxxxxxx>]

Handling NMIs in VT-x is awkward.  It turns out that using INT $2 happens to
be correct for IDT delivery, and can be made to be correct under FRED too.

Xen can now boot in FRED mode and run PVH dom0 even with the watchdog enabled.

Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
---
CC: Jan Beulich <JBeulich@xxxxxxxx>
CC: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
 xen/arch/x86/hvm/vmx/vmx.c | 14 ++++++++++++--
 xen/arch/x86/traps.c       | 16 +++++++++++++++-
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index cb82d52ef035..577a5e2d59c6 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -4209,8 +4209,18 @@ void asmlinkage vmx_vmexit_handler(struct cpu_user_regs 
*regs)
              ((intr_info & INTR_INFO_INTR_TYPE_MASK) ==
               MASK_INSR(X86_ET_NMI, INTR_INFO_INTR_TYPE_MASK)) )
         {
-            do_nmi(regs);
-            enable_nmis();
+            /*
+             * If we exited because of an NMI, NMIs are blocked in hardware,
+             * but software is expected to invoke the handler.
+             *
+             * Use INT $2.  Combined with the current state, it is the correct
+             * architectural state for the NMI handler, and the IRET on the
+             * way back out will unblock NMIs.
+             *
+             * In FRED mode, we can spot this trick and cause the ERETS to
+             * unblock NMIs too.
+             */
+            asm ("int $2");
         }
         break;
     case EXIT_REASON_MCE_DURING_VMENTRY:
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index f58e6dcf13b7..16dd335cadb2 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -2285,8 +2285,22 @@ void asmlinkage entry_from_xen(struct cpu_user_regs 
*regs)
         do_nmi(regs);
         break;
 
-    case X86_ET_HW_EXC:
     case X86_ET_SW_INT:
+        if ( regs->fred_ss.vector == 2 )
+        {
+            /*
+             * Explicit request from the the VMExit handler.  Rewrite the FRED
+             * frame to look like it was a real NMI, and go around again.
+             */
+            regs->fred_ss.swint = false;
+            regs->fred_ss.nmi = true;
+            regs->fred_ss.type = X86_ET_NMI;
+            regs->fred_ss.insnlen = 0;
+
+            return entry_from_xen(regs);
+        }
+        fallthrough;
+    case X86_ET_HW_EXC:
     case X86_ET_PRIV_SW_EXC:
     case X86_ET_SW_EXC:
         switch ( regs->fred_ss.vector )
-- 
2.39.5