[PATCH v6 16/44] x86/mm: add hooks for a sparse view of the directmap

[George Dunlap <gwd@xxxxxxxxxxxxxx>]

The directmap maps all of RAM, and every root page-table carries it,
so while any guest context runs, all guest memory is reachable from
Xen's address space.  For ASI, guest contexts are to run on a sparse
view of the directmap which maps only what Xen must reach through it
at all times, leaving guest memory to transient mappings
(map_domain_page()).

A later patch builds such a sparse view alongside the full directmap,
once the boot allocator has handed its memory to the heap.  It is then
kept in step:

 - xenheap allocations are added, and taken out again before the pages
   are freed.  If that fails the pages stay mapped, and are leaked with
   a warning rather than reused;
 - memory handed to the heap later (boot modules, .init) is taken out,
   and is not handed over if that fails;
 - a PV dom0's initrd, assigned to dom0 in place, is taken out;
 - page-tables from the boot allocator, which the sparse view maps with
   every other boot-time allocation, are taken out when idle_pg_table's
   hierarchy frees them, and are not freed if that fails, so that the
   sparse view never keeps a page the heap may hand out;
 - changes to the full directmap through map_pages_to_xen() and
   modify_xen_mappings() are mirrored.  After boot, map_pages_to_xen()
   maps into the directmap only memory Xen keeps (stack guard and shadow
   stack pages, firmware tables); memory hot-add maps the new range into
   the full directmap only, so that the sparse view never needs
   page-table pages for it.

Add a hook at each of these points, and one where the view is to be
built; x86's are stubs in a new asm/sparse-directmap.h.  map_pages_in()
and modify_mappings_in() are no longer static: memory hot-add calls
map_pages_in() on idle_pg_table, and the sparse view will use both.

Common code gains four hooks for this, arch_heap_pages_init(),
arch_xenheap_needs_scrub() and arch_xenheap_pages_{alloc,free}(), no-ops
until an architecture implements them.  A failed
arch_xenheap_pages_alloc() may have mapped part of the pages:
alloc_xenheap_pages() then takes them out through
arch_xenheap_pages_free(), as a free would, before freeing them, and
leaks them if that fails too.

alloc_xenheap_pages() accepts pages still awaiting scrubbing
(MEMF_no_scrub).  That is harmless while the directmap maps all memory
anyway, but the sparse view would make such a page's contents, a dying
domain's data for instance, reachable from every context on it until
overwritten.  With a sparse view configured, arch_xenheap_needs_scrub()
makes xenheap allocations scrub those pages first, as domain heap
allocations do; clean pages need nothing extra.

arch_mfns_in_directmap() says whether an MFN range is reachable through
the directmap; it now answers for every context, so false once a sparse
view is configured.  Its one caller, init_node_heap(), then keeps a NUMA
node's heap metadata in the xenheap instead of carving it out of the
node's memory through the directmap: carved out of a range handed to the
heap, it would be left out of the sparse view, which every heap
allocation needs it in.

The hooks for now are just stubs.  No functional change.

Assisted-by: Claude Code:claude-opus-5-5
Signed-off-by: George Dunlap <gwd@xxxxxxxxxxxxxx>
---
Changes in v5:
- New in this version, split out of "x86/mm: build and maintain a sparse
  view of the directmap".
---
 xen/arch/x86/include/asm/mm.h               | 12 +++++-
 xen/arch/x86/include/asm/sparse-directmap.h | 30 +++++++++++++++
 xen/arch/x86/mm.c                           | 30 ++++++++++++---
 xen/arch/x86/pv/dom0_build.c                |  4 ++
 xen/arch/x86/setup.c                        |  4 ++
 xen/arch/x86/x86_64/mm.c                    | 10 ++++-
 xen/common/page_alloc.c                     | 41 ++++++++++++++++++++-
 xen/include/xen/mm.h                        | 21 +++++++++++
 8 files changed, 142 insertions(+), 10 deletions(-)
 create mode 100644 xen/arch/x86/include/asm/sparse-directmap.h

diff --git a/xen/arch/x86/include/asm/mm.h b/xen/arch/x86/include/asm/mm.h
index 7b9a0f561d..61dc317dd4 100644
--- a/xen/arch/x86/include/asm/mm.h
+++ b/xen/arch/x86/include/asm/mm.h
@@ -7,6 +7,7 @@
 #include <xen/rwlock.h>
 #include <asm/io.h>
 #include <asm/page.h>
+#include <asm/sparse-directmap.h>
 #include <asm/uaccess.h>
 
 /*
@@ -599,6 +600,11 @@ mfn_t alloc_xen_pagetable(void);
 void free_xen_pagetable(mfn_t mfn);
 void *alloc_mapped_pagetable(mfn_t *pmfn);
 
+int map_pages_in(l4_pgentry_t *root, unsigned long virt, mfn_t mfn,
+                 unsigned long nr_mfns, pte_attr_t flags);
+int modify_mappings_in(l4_pgentry_t *root, unsigned long s, unsigned long e,
+                       pte_attr_t nf);
+
 int __sync_local_execstate(void);
 
 /* Arch-specific portion of memory_op hypercall. */
@@ -660,12 +666,16 @@ void write_32bit_pse_identmap(uint32_t *l2);
 
 /*
  * x86 maps part of physical memory via the directmap region.
- * Return whether the range of MFN falls in the directmap region.
+ * Return whether the range of MFN falls in the directmap region in every
+ * context: never, with a sparse directmap view configured.
  */
 static inline bool arch_mfns_in_directmap(unsigned long mfn, unsigned long nr)
 {
     unsigned long eva = min(DIRECTMAP_VIRT_END, HYPERVISOR_VIRT_END);
 
+    if ( sparse_dmap_configured() )
+        return false;
+
     return (mfn + nr) <= (virt_to_mfn(eva - 1) + 1);
 }
 
diff --git a/xen/arch/x86/include/asm/sparse-directmap.h 
b/xen/arch/x86/include/asm/sparse-directmap.h
new file mode 100644
index 0000000000..53747a98ad
--- /dev/null
+++ b/xen/arch/x86/include/asm/sparse-directmap.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef X86_SPARSE_DIRECTMAP_H
+#define X86_SPARSE_DIRECTMAP_H
+
+#include <xen/mm-frame.h>
+#include <xen/mm-types.h>
+#include <asm/page.h>
+
+static inline bool sparse_dmap_configured(void) { return false; }
+static inline void sparse_dmap_init(void) {}
+static inline int sparse_dmap_mirror_map(unsigned long virt, mfn_t mfn,
+                                         unsigned long nr, pte_attr_t flags)
+{
+    return 0;
+}
+static inline int sparse_dmap_mirror_modify(unsigned long s, unsigned long e,
+                                            pte_attr_t nf)
+{
+    return 0;
+}
+static inline int sparse_dmap_remove(mfn_t mfn, unsigned long nr)
+{
+    return 0;
+}
+static inline int sparse_dmap_drop_pagetable(mfn_t mfn)
+{
+    return 0;
+}
+
+#endif /* X86_SPARSE_DIRECTMAP_H */
diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 39c6fa3d56..904ed7a258 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -127,6 +127,7 @@
 #include <asm/setup.h>
 #include <asm/shadow.h>
 #include <asm/shared.h>
+#include <asm/sparse-directmap.h>
 #include <asm/trampoline.h>
 #include <asm/traps.h>
 #include <asm/x86_emulate.h>
@@ -5442,10 +5443,17 @@ mfn_t xen_map_to_mfn(unsigned long va)
 
 /*
  * Free a page-table page that the hierarchy rooted at @root has stopped
- * using.
+ * using.  One of idle_pg_table's may be in the sparse directmap view: it is
+ * taken out first, under the sparse view's L3 lock inside an L3 lock of
+ * idle_pg_table's hierarchy (the only order in which the two nest), and
+ * kept if that fails.  The sparse view's own page-tables are not in it, and
+ * are freed directly: the caller may hold the sparse view's L3 lock.
  */
 static void free_pagetable_in(const l4_pgentry_t *root, mfn_t mfn)
 {
+    if ( root == idle_pg_table && sparse_dmap_drop_pagetable(mfn) )
+        return;
+
     free_xen_pagetable(mfn);
 }
 
@@ -5453,7 +5461,7 @@ static void free_pagetable_in(const l4_pgentry_t *root, 
mfn_t mfn)
  * map_pages_to_xen() in the hierarchy rooted at @root: idle_pg_table's, or
  * another hierarchy sharing its virtual addresses.
  */
-static int map_pages_in(
+int map_pages_in(
     l4_pgentry_t *root,
     unsigned long virt,
     mfn_t mfn,
@@ -5865,7 +5873,12 @@ int map_pages_to_xen(
     unsigned long nr_mfns,
     pte_attr_t flags)
 {
-    return map_pages_in(idle_pg_table, virt, mfn, nr_mfns, flags);
+    int rc = map_pages_in(idle_pg_table, virt, mfn, nr_mfns, flags);
+
+    if ( !rc )
+        rc = sparse_dmap_mirror_map(virt, mfn, nr_mfns, flags);
+
+    return rc;
 }
 
 int __init populate_pt_range(unsigned long virt, unsigned long nr_mfns)
@@ -5888,8 +5901,8 @@ int __init populate_pt_range(unsigned long virt, unsigned 
long nr_mfns)
  * modify_mappings_in() does so in the hierarchy rooted at @root, as
  * map_pages_in() does.
  */
-static int modify_mappings_in(l4_pgentry_t *root, unsigned long s,
-                              unsigned long e, pte_attr_t nf)
+int modify_mappings_in(l4_pgentry_t *root, unsigned long s,
+                       unsigned long e, pte_attr_t nf)
 {
     bool locking = system_state > SYS_STATE_boot;
     l3_pgentry_t *pl3e = NULL;
@@ -6169,7 +6182,12 @@ static int modify_mappings_in(l4_pgentry_t *root, 
unsigned long s,
 
 int modify_xen_mappings(unsigned long s, unsigned long e, pte_attr_t nf)
 {
-    return modify_mappings_in(idle_pg_table, s, e, nf);
+    int rc = modify_mappings_in(idle_pg_table, s, e, nf);
+
+    if ( !rc )
+        rc = sparse_dmap_mirror_modify(s, e, nf);
+
+    return rc;
 }
 
 int destroy_xen_mappings(unsigned long s, unsigned long e)
diff --git a/xen/arch/x86/pv/dom0_build.c b/xen/arch/x86/pv/dom0_build.c
index dd64af5616..829eafd67d 100644
--- a/xen/arch/x86/pv/dom0_build.c
+++ b/xen/arch/x86/pv/dom0_build.c
@@ -22,6 +22,7 @@
 #include <asm/pv/mm.h>
 #include <asm/setup.h>
 #include <asm/shadow.h>
+#include <asm/sparse-directmap.h>
 
 /* Allow ring-3 access in long mode as guest cannot use ring 1 ... */
 #define BASE_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_USER)
@@ -650,6 +651,9 @@ static int __init dom0_construct(const struct boot_domain 
*bd)
         {
             if ( assign_pages(mfn_to_page(_mfn(initrd_mfn)), count, d, 0) )
                 BUG();
+            /* The initrd is dom0's now: out of the sparse directmap. */
+            if ( sparse_dmap_remove(_mfn(initrd_mfn), PFN_UP(initrd_len)) )
+                panic("Cannot unmap dom0's initrd\n");
             /*
              * We have mapped the initrd directly into dom0, and assigned the
              * pages. Tell the boot_module handling that we've freed it, so the
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index df7cfcc563..e23ded0c86 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -54,6 +54,7 @@
 #include <asm/spec_ctrl.h>
 #include <asm/stubs.h>
 #include <asm/tboot.h>
+#include <asm/sparse-directmap.h>
 #include <asm/trampoline.h>
 #include <asm/traps.h>
 
@@ -1946,6 +1947,9 @@ void asmlinkage __init noreturn __start_xen(void)
         }
     }
 
+    /* The heap now has all memory, and page-tables come from it. */
+    sparse_dmap_init();
+
     bsp_stack = cpu_alloc_stack(0); /* Needs to know IDT vs FRED */
     if ( !bsp_stack )
         panic("No memory for BSP stack\n");
diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
index 7b04537066..c5d9dd64a3 100644
--- a/xen/arch/x86/x86_64/mm.c
+++ b/xen/arch/x86/x86_64/mm.c
@@ -1231,8 +1231,14 @@ int memory_add(unsigned long spfn, unsigned long epfn, 
unsigned int pxm)
     i = virt_to_mfn(HYPERVISOR_VIRT_END - 1) + 1;
     if ( spfn < i )
     {
-        ret = map_pages_to_xen((unsigned long)mfn_to_virt(spfn), _mfn(spfn),
-                               min(epfn, i) - spfn, PAGE_HYPERVISOR);
+        /*
+         * Into the full directmap only, not a sparse directmap view: the
+         * frame table and M2P carved from the start of the range are
+         * reached through their own mappings, and the rest goes to the
+         * heap, which the sparse view leaves out.
+         */
+        ret = map_pages_in(idle_pg_table, (unsigned long)mfn_to_virt(spfn),
+                           _mfn(spfn), min(epfn, i) - spfn, PAGE_HYPERVISOR);
         if ( ret )
             goto destroy_directmap;
     }
diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index 5243dba29f..2150dcbd5e 100644
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -1971,6 +1971,9 @@ static void init_heap_pages(
     unsigned long i;
     bool need_scrub = scrub_debug;
 
+    if ( !arch_heap_pages_init(page_to_mfn(pg), nr_pages) )
+        return;
+
     /*
      * Keep MFN 0 away from the buddy allocator to avoid crossing zone
      * boundary when merging two buddies.
@@ -2540,10 +2543,33 @@ void *alloc_xenheap_pages(unsigned int order, unsigned 
int memflags)
     if ( !(memflags >> _MEMF_bits) )
         memflags |= MEMF_bits(xenheap_bits);
 
-    pg = alloc_domheap_pages(NULL, order, memflags | MEMF_no_scrub);
+    /*
+     * Pages still awaiting scrubbing may do as xenheap pages: the directmap
+     * maps them anyway.  Not so if an arch's sparse directmap view makes them
+     * reachable where the rest of the heap's memory is not.
+     */
+    if ( !arch_xenheap_needs_scrub() )
+        memflags |= MEMF_no_scrub;
+
+    pg = alloc_domheap_pages(NULL, order, memflags);
     if ( unlikely(pg == NULL) )
         return NULL;
 
+    if ( arch_xenheap_pages_alloc(page_to_mfn(pg), order) )
+    {
+        /*
+         * The hook may have mapped part of the pages before failing: take
+         * them out again as a free would, and leak them if that fails too.
+         */
+        if ( arch_xenheap_pages_free(page_to_mfn(pg), order) )
+            printk(XENLOG_WARNING
+                   "Leaking xenheap pages %"PRI_mfn"+%u: still mapped\n",
+                   mfn_x(page_to_mfn(pg)), 1U << order);
+        else
+            free_domheap_pages(pg, order);
+        return NULL;
+    }
+
     for ( i = 0; i < (1u << order); i++ )
         pg[i].count_info |= PGC_xen_heap;
 
@@ -2562,6 +2588,19 @@ void free_xenheap_pages(void *v, unsigned int order)
 
     pg = virt_to_page(v);
 
+    /*
+     * If the pages cannot be taken out of an arch's sparse directmap view,
+     * they may remain reachable through it: leak them rather than hand them
+     * out.
+     */
+    if ( arch_xenheap_pages_free(page_to_mfn(pg), order) )
+    {
+        printk(XENLOG_WARNING
+               "Leaking xenheap pages %"PRI_mfn"+%u: still mapped\n",
+               mfn_x(page_to_mfn(pg)), 1U << order);
+        return;
+    }
+
     for ( i = 0; i < (1u << order); i++ )
         pg[i].count_info &= ~PGC_xen_heap;
 
diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h
index 153c57f5dd..e8f9993286 100644
--- a/xen/include/xen/mm.h
+++ b/xen/include/xen/mm.h
@@ -94,6 +94,27 @@ bool scrub_free_pages(void);
 #define alloc_xenheap_page() (alloc_xenheap_pages(0,0))
 #define free_xenheap_page(v) (free_xenheap_pages(v,0))
 
+/*
+ * Hooks for an architecture keeping a second, sparse view of its directmap,
+ * which maps the xenheap but not the rest of the heap's memory.
+ */
+static inline bool arch_heap_pages_init(mfn_t mfn, unsigned long nr)
+{
+    return true;
+}
+static inline bool arch_xenheap_needs_scrub(void)
+{
+    return false;
+}
+static inline int arch_xenheap_pages_alloc(mfn_t mfn, unsigned int order)
+{
+    return 0;
+}
+static inline int arch_xenheap_pages_free(mfn_t mfn, unsigned int order)
+{
+    return 0;
+}
+
 /* Free an allocation, and zero the pointer to it. */
 #define FREE_XENHEAP_PAGES(p, o) do { \
     void *_ptr_ = (p);                \
-- 
2.56.0