[PATCH v1 2/3] xen/pci: modify pci_add_device to handle device add by Xen

[Mykyta Poturai <Mykyta_Poturai@xxxxxxxx>]

From: Luca Fancellu <luca.fancellu@xxxxxxx>

Currently pci_add_device is called from hypercalls requested by Dom0
to add pci devices and when the device has no domain associated with
it, it is assumed that hardware_domain is the owner.

On the dom0less scenario, the enumeration is performed by the
firmware and Xen at boot time might want to assign some pci devices
to guests, so it has to firstly add the device and then assign it to
the final guest.

Modify pci_add_device to have the owner domain passed as a parameter
to the function, so that when it is called from the hypercall the
owner would be the hardware domain, otherwise when Xen is calling it,
it would be another domain since hw domain could not be there
(dom0less guests without Dom0 use case).

In pci_add_device there is a call to xsm that doesn't consider the
requester of the function to be Xen itself, so add a check to skip
the call if the owner domain is dom_io, since it means the call is
coming directly from Xen.

Signed-off-by: Luca Fancellu <luca.fancellu@xxxxxxx>
Signed-off-by: Mykyta Poturai <mykyta_poturai@xxxxxxxx>
---
(cherry picked from commit f0c85d9043f7c9402e85b73361c8a13c683428ca from
 the downstream branch poc/pci-passthrough from
 https://gitlab.com/xen-project/people/bmarquis/xen-arm-poc.git)

changes since cherry-pick:
* s/hardware_domain/d/ in some write_unlock calls in 
xen/drivers/passthrough/pci.c
---
 xen/arch/x86/physdev.c        |  9 ++++-----
 xen/drivers/passthrough/pci.c | 32 ++++++++++++++++++++------------
 xen/drivers/pci/physdev.c     |  3 ++-
 xen/include/xen/pci.h         |  2 +-
 4 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/xen/arch/x86/physdev.c b/xen/arch/x86/physdev.c
index 4dfa1c0191..9e894ce665 100644
--- a/xen/arch/x86/physdev.c
+++ b/xen/arch/x86/physdev.c
@@ -446,8 +446,8 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) 
arg)
         if ( copy_from_guest(&manage_pci, arg, 1) != 0 )
             break;
 
-        ret = pci_add_device(0, manage_pci.bus, manage_pci.devfn,
-                             NULL, NUMA_NO_NODE);
+        ret = pci_add_device(hardware_domain, 0, manage_pci.bus,
+                             manage_pci.devfn, NULL, NUMA_NO_NODE);
         break;
     }
 
@@ -477,9 +477,8 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) 
arg)
         pdev_info.is_virtfn = manage_pci_ext.is_virtfn;
         pdev_info.physfn.bus = manage_pci_ext.physfn.bus;
         pdev_info.physfn.devfn = manage_pci_ext.physfn.devfn;
-        ret = pci_add_device(0, manage_pci_ext.bus,
-                             manage_pci_ext.devfn,
-                             &pdev_info, NUMA_NO_NODE);
+        ret = pci_add_device(hardware_domain, 0, manage_pci_ext.bus,
+                             manage_pci_ext.devfn, &pdev_info, NUMA_NO_NODE);
         break;
     }
 
diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c
index fa03dafac7..49a39d69db 100644
--- a/xen/drivers/passthrough/pci.c
+++ b/xen/drivers/passthrough/pci.c
@@ -654,7 +654,7 @@ unsigned int pci_size_mem_bar(pci_sbdf_t sbdf, unsigned int 
pos,
     return is64bits ? 2 : 1;
 }
 
-int pci_add_device(u16 seg, u8 bus, u8 devfn,
+int pci_add_device(struct domain *d, u16 seg, u8 bus, u8 devfn,
                    const struct pci_dev_info *info, nodeid_t node)
 {
     struct pci_seg *pseg;
@@ -672,9 +672,12 @@ int pci_add_device(u16 seg, u8 bus, u8 devfn,
     else
         type = "device";
 
-    ret = xsm_resource_plug_pci(XSM_PRIV, (seg << 16) | (bus << 8) | devfn);
-    if ( ret )
-        return ret;
+    if ( d != dom_io )
+    {
+        ret = xsm_resource_plug_pci(XSM_PRIV, (seg << 16) | (bus << 8) | 
devfn);
+        if ( ret )
+            return ret;
+    }
 
     ret = -ENOMEM;
 
@@ -767,9 +770,9 @@ int pci_add_device(u16 seg, u8 bus, u8 devfn,
     ret = 0;
     if ( !pdev->domain )
     {
-        pdev->domain = hardware_domain;
-        write_lock(&hardware_domain->pci_lock);
-        list_add(&pdev->domain_list, &hardware_domain->pdev_list);
+        pdev->domain = d;
+        write_lock(&d->pci_lock);
+        list_add(&pdev->domain_list, &pdev->domain->pdev_list);
 
         /*
          * For devices not discovered by Xen during boot, add vPCI handlers
@@ -779,25 +782,30 @@ int pci_add_device(u16 seg, u8 bus, u8 devfn,
         if ( ret )
         {
             list_del(&pdev->domain_list);
-            write_unlock(&hardware_domain->pci_lock);
+            write_unlock(&d->pci_lock);
             pdev->domain = NULL;
             printk(XENLOG_ERR "Setup of vPCI failed: %d\n", ret);
             goto out;
         }
-        write_unlock(&hardware_domain->pci_lock);
+        write_unlock(&d->pci_lock);
         ret = iommu_add_device(pdev);
         if ( ret )
         {
-            write_lock(&hardware_domain->pci_lock);
+            write_lock(&d->pci_lock);
             vpci_deassign_device(pdev);
             list_del(&pdev->domain_list);
-            write_unlock(&hardware_domain->pci_lock);
+            write_unlock(&d->pci_lock);
             pdev->domain = NULL;
             goto out;
         }
     }
-    else
+    else if ( pdev->domain == d )
         iommu_enable_device(pdev);
+    else
+    {
+        ret = -EINVAL;
+        goto out;
+    }
 
     pci_enable_acs(pdev);
 
diff --git a/xen/drivers/pci/physdev.c b/xen/drivers/pci/physdev.c
index d46501b884..1b59bf1246 100644
--- a/xen/drivers/pci/physdev.c
+++ b/xen/drivers/pci/physdev.c
@@ -50,7 +50,8 @@ ret_t pci_physdev_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) 
arg)
         }
 #endif
 
-        ret = pci_add_device(add.seg, add.bus, add.devfn, &pdev_info, node);
+        ret = pci_add_device(hardware_domain, add.seg, add.bus, add.devfn,
+                             &pdev_info, node);
         break;
     }
 
diff --git a/xen/include/xen/pci.h b/xen/include/xen/pci.h
index 5c242278b9..77a44aea70 100644
--- a/xen/include/xen/pci.h
+++ b/xen/include/xen/pci.h
@@ -227,7 +227,7 @@ void setup_hwdom_pci_devices(struct domain *d,
 int pci_release_devices(struct domain *d);
 int pci_add_segment(u16 seg);
 const unsigned long *pci_get_ro_map(u16 seg);
-int pci_add_device(u16 seg, u8 bus, u8 devfn,
+int pci_add_device(struct domain *d, u16 seg, u8 bus, u8 devfn,
                    const struct pci_dev_info *info, nodeid_t node);
 int pci_remove_device(u16 seg, u8 bus, u8 devfn);
 int pci_ro_device(int seg, int bus, int devfn);
-- 
2.34.1