[PATCH v2 10/14] x86/PVH: try to map initrd directly into Dom0

[Jan Beulich <jbeulich@xxxxxxxx>]

Our present handling of initrd is problematic in particular when the
initrd is huge, and when at the same time "dom0_mem=" isn't in use to
significantly shrink Dom0's memory (from the default). In that case,
insufficient memory may exist for pvh_populate_p2m() to succeed (because
initrd memory can be freed only after copying).

Make an attempt to map the blob (including headroom) into Dom0's P2M
instead, thus also saving the copying operation. With __start_xen() having
arranged for a larger than necessary blob, that mapping can typically be
done with superpage mappings.

Reported-by: Marek Marczykowski-Górecki <marmarek@xxxxxxxxxxxxxxxxxxxxxx>
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
pvh_steal_ram() can (theoretically) fail just because of
rangeset_remove_range() there failing, in which case we may be in limbo
state. That's hopefully an acceptable risk.
---
v2: Re-base over changes earlier in the series, in particular to use
    pvh_steal_ram() here. Avoid NULL deref when there's no initrd. Also
    log domain in new/moved printk().

--- a/xen/arch/x86/hvm/dom0_build.c
+++ b/xen/arch/x86/hvm/dom0_build.c
@@ -528,7 +528,9 @@ static void __init pvh_init_p2m(struct d
 static int __init pvh_populate_p2m(
     struct domain *d,
     const struct e820entry *e820,
-    unsigned int nr_e820)
+    unsigned int nr_e820,
+    unsigned long initrd_start,
+    unsigned long initrd_end)
 {
     struct vcpu *v = d->vcpu[0];
     unsigned int i;
@@ -546,7 +548,12 @@ static int __init pvh_populate_p2m(
         addr = PFN_DOWN(e820[i].addr);
         size = PFN_DOWN(e820[i].size);
 
-        rc = pvh_populate_memory_range(d, addr, size);
+        if ( addr + size <= initrd_start || addr >= initrd_end )
+            rc = pvh_populate_memory_range(d, addr, size);
+        else
+            rc = pvh_populate_memory_range(d, addr, initrd_start - addr)
+                 ?: pvh_populate_memory_range(d, initrd_end,
+                                              addr + size - initrd_end);
         if ( rc )
             return rc;
 
@@ -636,6 +643,43 @@ static struct rangeset *__init make_ram_
     return ram;
 }
 
+static int __init pvh_map_initrd(
+    struct domain *d, const struct boot_module *initrd,
+    paddr_t initrd_start, unsigned int order)
+{
+    for ( paddr_t ga = initrd_start, ma = initrd->start;
+          ga < initrd_start + initrd->size;
+          ga += PAGE_SIZE << order, ma += PAGE_SIZE << order )
+    {
+        int rc = p2m_add_page(d, gaddr_to_gfn(ga), maddr_to_mfn(ma),
+                              order, p2m_ram_rw);
+
+        if ( rc )
+        {
+            printk("%pd: failed to map initrd: [%#lx,%#lx): %d\n",
+                   d, PFN_DOWN(ga), PFN_DOWN(ga) + (1UL << order), rc);
+
+            /*
+             * Best effort rollback: The range will be replaced anyway by the
+             * subsequent pvh_populate_p2m().
+             */
+            while ( ga > initrd_start )
+            {
+                ga -= PAGE_SIZE << order;
+                ma -= PAGE_SIZE << order;
+                p2m_remove_page(d, gaddr_to_gfn(ga), maddr_to_mfn(ma), order);
+                order_stats[order]--;
+            }
+
+            return rc;
+        }
+
+        order_stats[order]++;
+    }
+
+    return 0;
+}
+
 static bool __init check_load_address(
     const struct domain *d, const struct elf_binary *elf)
 {
@@ -796,9 +840,98 @@ static int __init pvh_place_kernel(
     return 0;
 }
 
+static void __init pvh_place_initrd(
+    const struct boot_domain *bd,
+    const struct elf_binary *elf,
+    struct rangeset *ram,
+    struct hvm_modlist_entry *mod,
+    paddr_t *blob_start,
+    paddr_t *blob_end)
+{
+    struct domain *d = bd->d;
+    struct boot_module *initrd = bd->initrd;
+    struct boot_module try;
+    unsigned long initrd_len =
+        initrd ? initrd->size - initrd->arch.headroom : 0;
+    size_t align = 1UL << L3_PAGETABLE_SHIFT;
+    int rc;
+
+    *blob_start = INVALID_PADDR;
+    *blob_end   = 0;
+
+    if ( !initrd_len )
+        return;
+
+    try = *initrd;
+
+    mod->size = initrd_len;
+
+    /*
+     * As we promise to place below 4G whatever fits there, make a first pass
+     * limiting permitted addresses.  If we can't fit it there, fall back to
+     * placing above 4G.
+     */
+    for ( paddr_t limit = GB(4); ; )
+    {
+        unsigned long size = ROUNDUP(try.size, PAGE_SIZE);
+        paddr_t start;
+
+        if ( IS_ALIGNED(try.start | size, align) &&
+             !pvh_steal_ram(ram, size - try.arch.headroom, align, limit,
+                            blob_start, NULL) )
+            break;
+
+        if ( align == PAGE_SIZE )
+        {
+            if ( !(limit + 1) )
+                return;
+
+            limit = -1;
+            align = 1UL << L3_PAGETABLE_SHIFT;
+            try = *initrd;
+            continue;
+        }
+
+        align >>= PAGETABLE_ORDER;
+
+        start = ROUNDDOWN(try.start + try.arch.headroom, align);
+        if ( start > try.start )
+        {
+            /* Free (part of) and reduce headroom. */
+            try.arch.headroom -= start - try.start;
+            try.size          -= start - try.start;
+            try.start          = start;
+        }
+    }
+
+    /* Free unused headroom, if any. */
+    if ( try.start > initrd->start )
+    {
+        init_domheap_pages(initrd->start, try.start);
+        *initrd = try;
+    }
+
+    rc = pvh_map_initrd(d, initrd, *blob_start, get_order_from_bytes(align));
+    if ( rc )
+    {
+        printk("%pd: failed to map initrd @ %#lx: %d\n", d, *blob_start, rc);
+        return;
+    }
+
+    rc = assign_pages(maddr_to_page(initrd->start),
+                      PFN_UP(initrd->size), d, dom0_memflags);
+    if ( rc )
+        panic("%pd: cannot assign initrd RAM: %d\n", d, rc);
+
+    mod->paddr = *blob_start + initrd->arch.headroom;
+    *blob_end  = *blob_start + initrd->size;
+    initrd->arch.released = true;
+}
+
 static int __init pvh_load_kernel(
     const struct boot_domain *bd,
     struct elf_binary *elf,
+    struct hvm_modlist_entry *mod,
     paddr_t *start_info_addr,
     struct rangeset *ram)
 {
@@ -809,9 +942,8 @@ static int __init pvh_load_kernel(
     size_t cmdline_len = bd->cmdline ? strlen(bd->cmdline) + 1 : 0;
     const char *initrd_cmdline = NULL;
     size_t extra_space;
-    paddr_t last_addr;
+    paddr_t last_addr, initrd_start = 0;
     struct hvm_start_info start_info = { 0 };
-    struct hvm_modlist_entry mod = { 0 };
     struct vcpu *v = d->vcpu[0];
     int rc;
 
@@ -828,15 +960,18 @@ static int __init pvh_load_kernel(
 
     /*
      * Find a RAM region big enough (and that doesn't overlap with the loaded
-     * kernel) in order to load the initrd and the metadata. Note it could be
-     * split into smaller allocations, done as a single region in order to
-     * simplify it.
+     * kernel) in order to load the initrd (if it wasn't mapped directly above)
+     * and the metadata.  Note it could be split into smaller allocations, done
+     * as a single region in order to simplify it.
      */
     extra_space = sizeof(start_info);
 
     if ( initrd )
     {
-        size_t initrd_space = elf_round_up(elf, initrd_len);
+        size_t initrd_space = 0;
+
+        if ( !mod->paddr )
+            initrd_space = elf_round_up(elf, initrd_len);
 
         if ( initrd->arch.cmdline_pa )
         {
@@ -848,7 +983,9 @@ static int __init pvh_load_kernel(
             initrd_space += strlen(initrd_cmdline) + 1;
 
         if ( initrd_space )
-            extra_space += ROUNDUP(initrd_space, PAGE_SIZE) + sizeof(mod);
+            extra_space += ROUNDUP(initrd_space, PAGE_SIZE) + sizeof(*mod);
+        else if ( mod->paddr )
+            extra_space += sizeof(*mod);
         else
             initrd = NULL;
     }
@@ -860,14 +997,16 @@ static int __init pvh_load_kernel(
      * live below 1Gb (mapped by the first L3 entry of the direct map).  32-bit
      * Linux may map more, but defaults to mapping only 1Gb.
      */
-    if ( pvh_steal_ram(ram, PAGE_ALIGN(extra_space), PAGE_SIZE, GB(1),
-                       &last_addr, NULL) )
+    rc = pvh_steal_ram(ram, PAGE_ALIGN(extra_space), PAGE_SIZE, GB(1),
+                       &last_addr, NULL);
+    if ( rc )
     {
-        printk("Unable to find a memory region to load initrd and metadata\n");
-        return -ENOMEM;
+        printk("%pd: unable to find a memory region to %s metadata: %d\n",
+               d, mod->paddr ? "put" : "load initrd and", rc);
+        return rc;
     }
 
-    if ( initrd != NULL )
+    if ( initrd && !mod->paddr )
     {
         rc = hvm_copy_to_guest_phys(last_addr,
                                     __va(initrd->start + 
initrd->arch.headroom),
@@ -878,28 +1017,29 @@ static int __init pvh_load_kernel(
             return rc;
         }
 
-        mod.paddr = last_addr;
-        mod.size = initrd_len;
+        initrd_start = last_addr;
         last_addr += elf_round_up(elf, initrd_len);
-        if ( initrd_cmdline )
-        {
-            size_t len = strlen(initrd_cmdline) + 1;
-
-            rc = hvm_copy_to_guest_phys(last_addr, initrd_cmdline, len, v);
-            if ( rc )
-            {
-                printk("Unable to copy module command line\n");
-                return rc;
-            }
-            mod.cmdline_paddr = last_addr;
-            last_addr += len;
-        }
-        last_addr = ROUNDUP(last_addr, PAGE_SIZE);
     }
 
     /* Free temporary buffers. */
     free_boot_modules();
 
+    if ( initrd_cmdline )
+    {
+        size_t len = strlen(initrd_cmdline) + 1;
+
+        rc = hvm_copy_to_guest_phys(last_addr, initrd_cmdline, len, v);
+        if ( rc )
+        {
+            printk("%pd: unable to copy module command line\n", d);
+            return rc;
+        }
+        mod->cmdline_paddr = last_addr;
+        last_addr = ROUNDUP(last_addr + len, PAGE_SIZE);
+    }
+    else if ( initrd && !mod->paddr )
+        last_addr = ROUNDUP(last_addr, PAGE_SIZE);
+
     rc = hvm_copy_to_guest_phys(last_addr, bd->cmdline, cmdline_len, v);
     if ( rc )
     {
@@ -917,7 +1057,10 @@ static int __init pvh_load_kernel(
 
     if ( initrd != NULL )
     {
-        rc = hvm_copy_to_guest_phys(last_addr, &mod, sizeof(mod), v);
+        if ( !mod->paddr )
+            mod->paddr = initrd_start;
+
+        rc = hvm_copy_to_guest_phys(last_addr, mod, sizeof(*mod), v);
         if ( rc )
         {
             printk("Unable to copy guest modules\n");
@@ -925,7 +1068,7 @@ static int __init pvh_load_kernel(
         }
         start_info.modlist_paddr = last_addr;
         start_info.nr_modules = 1;
-        last_addr += sizeof(mod);
+        last_addr += sizeof(*mod);
     }
 
     start_info.magic = XEN_HVM_START_MAGIC_VALUE;
@@ -1508,6 +1651,8 @@ int __init dom0_construct_pvh(const stru
     struct rangeset *ram;
     struct acpi_info acpi = {};
     struct vmx_info vmx = {};
+    struct hvm_modlist_entry initrd = {};
+    paddr_t blob_start, blob_end;
     int rc;
 
     printk(XENLOG_INFO "*** Building a PVH Dom%d ***\n", d->domain_id);
@@ -1592,7 +1737,10 @@ int __init dom0_construct_pvh(const stru
             return rc;
     }
 
-    rc = pvh_populate_p2m(d, orig_e820, nr_orig_e820);
+    pvh_place_initrd(bd, &elf, ram, &initrd, &blob_start, &blob_end);
+
+    rc = pvh_populate_p2m(d, orig_e820, nr_orig_e820,
+                          PFN_DOWN(blob_start), PFN_UP(blob_end));
     if ( rc )
     {
         printk("Failed to setup Dom0 physical memory map\n");
@@ -1601,7 +1749,7 @@ int __init dom0_construct_pvh(const stru
 
     xvfree(orig_e820);
 
-    rc = pvh_load_kernel(bd, &elf, &start_info, ram);
+    rc = pvh_load_kernel(bd, &elf, &initrd, &start_info, ram);
     if ( rc )
     {
         printk("Failed to load Dom0 kernel\n");