[Xen-devel] [PATCH v2 2/2] hvmloader, pci: Don't try to relocate memory if 64-bit BAR is bigger than ~2GB

[Konrad Rzeszutek Wilk <konrad.wilk@xxxxxxxxxx>]

Where ~2GB is actually 2GB minus MMIO space used for emulated devices
and minus APIC space.

There is no point. We can try to balloon out the memory between
hvm_info->low_mem_pgend to pci_mem_end and we will never be able
have a hole big enough for 2GB MMIO. As we can't go lower than
0x80000000 and can't go above 0xF0000000 which effectively leaves
us with 1792MB of MMIO space (-32MB for VGA and -16 for platform
device, so in reality: 1744MB).

Instead just let it go above 4GB in the 64-bit zone.

Note that prior to this patch the hvmloader would relocate as much
memory as it could under 4GB:

Low MMIO hole not large enough for all devices, relocating some BARs to 64-bit
Relocating 0xffff pages from 0e0001000 to 210000000 for lowmem MMIO hole
Relocating 0xffff pages from 0d0002000 to 21ffff000 for lowmem MMIO hole
Relocating 0xffff pages from 0c0003000 to 22fffe000 for lowmem MMIO hole
Relocating 0xffff pages from 0b0004000 to 23fffd000 for lowmem MMIO hole
Relocating 0xffff pages from 0a0005000 to 24fffc000 for lowmem MMIO hole
Relocating 0xffff pages from 090006000 to 25fffb000 for lowmem MMIO hole
Relocating 0xffff pages from 080007000 to 26fffa000 for lowmem MMIO hole
Relocating 0x7 pages from 080000000 to 27fff9000 for lowmem MMIO hole

which is completely pointless.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@xxxxxxxxxx>
---
Cc: Jan Beulich <jbeulich@xxxxxxxx>
Cc: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
Cc: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>
Cc: Wei Liu <wei.liu2@xxxxxxxxxx>

v1: First submission.

v2: Check for up to 1744MB.
  - Use defines for 4GB, 2GB.
  - Fix style guide issues.
---
 tools/firmware/hvmloader/pci.c | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/tools/firmware/hvmloader/pci.c b/tools/firmware/hvmloader/pci.c
index 416829d..f6194b9 100644
--- a/tools/firmware/hvmloader/pci.c
+++ b/tools/firmware/hvmloader/pci.c
@@ -80,7 +80,7 @@ void pci_setup(void)
 {
     uint8_t is_64bar, using_64bar, bar64_relocate = 0;
     uint32_t devfn, bar_reg, cmd, bar_data, bar_data_upper;
-    uint64_t base, bar_sz, bar_sz_upper, mmio_total = 0;
+    uint64_t base, bar_sz, bar_sz_upper, mmio_total = 0, mmio_64bit_total = 0;
     uint32_t vga_devfn = 256;
     uint16_t class, vendor_id, device_id;
     unsigned int bar, pin, link, isa_irq;
@@ -97,6 +97,7 @@ void pci_setup(void)
         uint32_t devfn;
         uint32_t bar_reg;
         uint64_t bar_sz;
+        bool above_4gb;
     } *bars = (struct bars *)scratch_start;
     unsigned int i, nr_bars = 0;
     uint64_t mmio_hole_size = 0;
@@ -265,11 +266,30 @@ void pci_setup(void)
             bars[i].devfn   = devfn;
             bars[i].bar_reg = bar_reg;
             bars[i].bar_sz  = bar_sz;
+            bars[i].above_4gb = false;
 
             if ( ((bar_data & PCI_BASE_ADDRESS_SPACE) ==
                   PCI_BASE_ADDRESS_SPACE_MEMORY) ||
                  (bar_reg == PCI_ROM_ADDRESS) )
-                mmio_total += bar_sz;
+            {
+                /*
+                 * If bigger than 2GB minus emulated devices BAR space and
+                 * APIC space, then don't try to put under 4GB.
+                 */
+                if ( is_64bar && (mmio_total >= GB(2) || bar_sz >=
+                     (GB(2) - HVM_BELOW_4G_MMIO_LENGTH - mmio_total)) )
+                {
+                    mmio_64bit_total += bar_sz;
+                    bars[i].above_4gb = true;
+                    /*
+                     * As this may not trigger now that mmio_total could be
+                     * less than 2GB, so force it.
+                     */
+                    bar64_relocate = 1;
+                }
+                else
+                    mmio_total += bar_sz;
+            }
 
             nr_bars++;
 
@@ -349,7 +369,7 @@ void pci_setup(void)
             pci_mem_start = hvm_info->low_mem_pgend << PAGE_SHIFT;
     }
 
-    if ( mmio_total > (pci_mem_end - pci_mem_start) )
+    if ( mmio_total > (pci_mem_end - pci_mem_start) || bar64_relocate )
     {
         printf("Low MMIO hole not large enough for all devices,"
                " relocating some BARs to 64-bit\n");
@@ -431,7 +451,7 @@ void pci_setup(void)
          * Should either of those two conditions change, this code will break.
          */
         using_64bar = bars[i].is_64bar && bar64_relocate
-            && (mmio_total > (mem_resource.max - mem_resource.base));
+            && ((mmio_total + mmio_64bit_total) > (mem_resource.max - 
mem_resource.base));
         bar_data = pci_readl(devfn, bar_reg);
 
         if ( (bar_data & PCI_BASE_ADDRESS_SPACE) ==
@@ -451,7 +471,10 @@ void pci_setup(void)
                 resource = &mem_resource;
                 bar_data &= ~PCI_BASE_ADDRESS_MEM_MASK;
             }
-            mmio_total -= bar_sz;
+            if ( bars[i].above_4gb )
+                mmio_64bit_total -= bar_sz;
+            else
+                mmio_total -= bar_sz;
         }
         else
         {
-- 
2.4.11


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel