[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] libxenlight: implement pci passthrough



Hi all,
this patch implements pci passthrough (hotplug and coldplug) in
libxenlight, it also adds three new commands to xl: pci-attach,
pci-detach and pci-list.
Currently flr on a device is done writing to
/sys/bus/pci/drivers/pciback/do_flr
pciback do_flr is present in both XCI and XCP 2.6.27 kernels and it is
implemented in this patch:

http://xenbits.xen.org/gitweb?p=xenclient/linux-2.6.27-pq.git;a=blob;f=master/pciback-flr;h=e993f18dbda740cef63176b75e114bbbf9a3801d;hb=HEAD


Signed-off-by: Stefano Stabellini <stefano.stabellini@xxxxxxxxxxxxx>

---

diff -r 32a27fe01e07 tools/libxl/libxl.c
--- a/tools/libxl/libxl.c       Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl.c       Thu Nov 12 18:04:19 2009 +0000
@@ -156,6 +156,7 @@
 
     xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/uuid", vm_path), uuid_string, 
strlen(uuid_string));
     xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/name", vm_path), info->name, 
strlen(info->name));
+    xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/image/ostype", vm_path), 
"hvm", strlen("hvm"));
 
     libxl_xs_writev(ctx, t, dom_path, info->xsdata);
     libxl_xs_writev(ctx, t, libxl_sprintf(ctx, "%s/platform", dom_path), 
info->platformdata);
@@ -368,6 +369,8 @@
         XL_LOG(ctx, XL_LOG_ERROR, "failed ot get uuid for %d\n", domid);
         return -1;
     }
+    if (libxl_device_pci_shutdown(ctx, domid) < 0)
+        XL_LOG(ctx, XL_LOG_ERROR, "pci shutdown failed for domid %d\n", domid);
     xs_write(ctx->xsh, XBT_NULL,
              libxl_sprintf(ctx, "/local/domain/0/device-model/%d/command", 
domid),
              "shutdown", strlen("shutdown"));
@@ -375,7 +378,6 @@
         XL_LOG(ctx, XL_LOG_ERROR, "xc_domain_pause failed for %d\n", domid);
         return -1;
     }
-    /* do_FLR */
     if (xc_domain_destroy(ctx->xch, domid) < 0) {
         XL_LOG(ctx, XL_LOG_ERROR, "xc_domain_destroy failed for %d\n", domid);
         return -1;
@@ -746,17 +748,417 @@
 }
 
 
/******************************************************************************/
-int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid)
+
+int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
+                          unsigned int bus, unsigned int dev,
+                          unsigned int func, unsigned int vdevfn)
 {
-    return ERROR_NI;
+    pcidev->domain = domain;
+    pcidev->bus = bus;
+    pcidev->dev = dev;
+    pcidev->func = func;
+    pcidev->vdevfn = vdevfn;
+    return 0;
 }
 
-int libxl_device_pci_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+static int libxl_create_pci_backend(struct libxl_ctx *ctx, uint32_t domid, 
libxl_device_pci *pcidev, int num)
 {
-    return ERROR_NI;
+    flexarray_t *front;
+    flexarray_t *back;
+    unsigned int boffset = 0;
+    unsigned int foffset = 0;
+    libxl_device device;
+    int i;
+
+    front = flexarray_make(16, 1);
+    if (!front)
+        return ERROR_NOMEM;
+    back = flexarray_make(16, 1);
+    if (!back)
+        return ERROR_NOMEM;
+
+    XL_LOG(ctx, XL_LOG_DEBUG, "Creating pci backend\n");
+
+    /* add pci device */
+    device.backend_devid = 0;
+    device.backend_domid = 0;
+    device.backend_kind = DEVICE_PCI;
+    device.devid = 0;
+    device.domid = domid;
+    device.kind = DEVICE_PCI;
+
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "frontend-id"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", domid));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "online"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "1"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "state"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "domain"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%s", 
libxl_domid_to_name(ctx, domid)));
+    for (i = 0; i < num; i++) {
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", i));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, 
pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", i));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, 
pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
+        if (pcidev->vdevfn) {
+            flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", i));
+            flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x", 
pcidev->vdevfn));
+        }
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", i));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, 
"msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", i));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+    }
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "num_devs"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num));
+
+    flexarray_set(front, foffset++, libxl_sprintf(ctx, "backend-id"));
+    flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 0));
+    flexarray_set(front, foffset++, libxl_sprintf(ctx, "state"));
+    flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 1));
+
+    libxl_device_generic_add(ctx, &device,
+                             libxl_xs_kvs_of_flexarray(ctx, back, boffset),
+                             libxl_xs_kvs_of_flexarray(ctx, front, foffset));
+    
+    flexarray_free(back);
+    flexarray_free(front);
+    return 0;
 }
 
-int libxl_device_pci_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+static int libxl_device_pci_add_xenstore(struct libxl_ctx *ctx, uint32_t 
domid, libxl_device_pci *pcidev)
 {
-    return ERROR_NI;
+    flexarray_t *back;
+    char *num_devs, *be_path;
+    int num = 0;
+    unsigned int boffset = 0;
+    xs_transaction_t t;
+
+    be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", 
xs_get_domain_path(ctx->xsh, 0), domid);
+    num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs", 
be_path));
+    if (!num_devs)
+        return libxl_create_pci_backend(ctx, domid, pcidev, 1);
+
+    if (!is_hvm(ctx, domid)) {
+        if (libxl_wait_for_backend(ctx, be_path, "4") < 0)
+            return -1;
+    }
+
+    back = flexarray_make(16, 1);
+    if (!back)
+        return ERROR_NOMEM;
+
+    XL_LOG(ctx, XL_LOG_DEBUG, "Adding new pci device to xenstore\n");
+    num = atoi(num_devs);
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", num));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, 
pcidev->bus, pcidev->dev, pcidev->func));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", num));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, 
pcidev->bus, pcidev->dev, pcidev->func));
+    if (pcidev->vdevfn) {
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", num));
+        flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x", 
pcidev->vdevfn));
+    }
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", num));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, 
"msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", num));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "num_devs"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num + 1));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "state"));
+    flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 7));
+
+retry_transaction:
+    t = xs_transaction_start(ctx->xsh);
+    libxl_xs_writev(ctx, t, be_path,
+                    libxl_xs_kvs_of_flexarray(ctx, back, boffset));
+    if (!xs_transaction_end(ctx->xsh, t, 0))
+        if (errno == EAGAIN)
+            goto retry_transaction;
+
+    flexarray_free(back);
+    return 0;
 }
+
+static int libxl_device_pci_remove_xenstore(struct libxl_ctx *ctx, uint32_t 
domid, libxl_device_pci *pcidev)
+{
+    char *be_path, *num_devs_path, *num_devs, *xsdev;
+    int num, i;
+    xs_transaction_t t;
+    unsigned int domain = 0, bus = 0, dev = 0, func = 0;
+
+    be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", 
xs_get_domain_path(ctx->xsh, 0), domid);
+    num_devs_path = libxl_sprintf(ctx, "%s/num_devs", be_path);
+    num_devs = libxl_xs_read(ctx, XBT_NULL, num_devs_path);
+    if (!num_devs)
+        return -1;
+    num = atoi(num_devs);
+    if (num == 1) {
+        libxl_device_destroy(ctx, be_path, 1);
+        xs_rm(ctx->xsh, XBT_NULL, be_path);
+        return 0;
+    }
+
+    if (!is_hvm(ctx, domid)) {
+        if (libxl_wait_for_backend(ctx, be_path, "4") < 0) {
+            XL_LOG(ctx, XL_LOG_DEBUG, "pci backend at %s is not ready\n");
+            return -1;
+        }
+    }
+
+    for (i = 0; i < num; i++) {
+        xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d", 
be_path, i));
+        sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
+        if (domain == pcidev->domain && bus == pcidev->bus &&
+            pcidev->dev == dev && pcidev->func == func) {
+            break;
+        }
+    }
+    if (i == num) {
+        XL_LOG(ctx, XL_LOG_ERROR, "Couldn't find the device on xenstore\n");
+        return -1;
+    }
+
+retry_transaction:
+    t = xs_transaction_start(ctx->xsh);
+    libxl_xs_write(ctx, t, num_devs_path, "%d", num - 1);
+    xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state-%d", be_path, i), "6", 
strlen("6"));
+    xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state", be_path), "7", 
strlen("7"));
+    if (!xs_transaction_end(ctx->xsh, t, 0))
+        if (errno == EAGAIN)
+            goto retry_transaction;
+    return 0;
+}
+
+int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid, 
libxl_device_pci *pcidev)
+{
+    char path[50];
+    char *state, *vdevfn;
+    int rc, hvm;
+
+    /* TODO: check if the device can be assigned */
+
+    libxl_device_pci_flr(ctx, pcidev->domain, pcidev->bus, pcidev->dev, 
pcidev->func);
+
+    hvm = is_hvm(ctx, domid);
+    if (hvm) {
+        if (libxl_wait_for_device_model(ctx, domid, "running") < 0) {
+            return -1;
+        }
+        snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state", 
domid);
+        state = libxl_xs_read(ctx, XBT_NULL, path);
+        snprintf(path, sizeof(path), 
"/local/domain/0/device-model/%d/parameter", domid);
+        if (pcidev->vdevfn)
+            libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF_VDEVFN, pcidev->domain,
+                           pcidev->bus, pcidev->dev, pcidev->func, 
pcidev->vdevfn);
+        else
+            libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
+                           pcidev->bus, pcidev->dev, pcidev->func);
+        snprintf(path, sizeof(path), 
"/local/domain/0/device-model/%d/command", domid);
+        xs_write(ctx->xsh, XBT_NULL, path, "pci-ins", strlen("pci-ins"));
+        if (libxl_wait_for_device_model(ctx, domid, "pci-inserted") < 0)
+            XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time\n");
+        snprintf(path, sizeof(path), 
"/local/domain/0/device-model/%d/parameter", domid);
+        vdevfn = libxl_xs_read(ctx, XBT_NULL, path);
+        sscanf(vdevfn + 2, "%x", &pcidev->vdevfn);
+        snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state", 
domid);
+        xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
+    } else {
+        char *sysfs_path = libxl_sprintf(ctx, 
"SYSFS_PCI_DEV/"PCI_BDF"/resource", pcidev->domain,
+                                         pcidev->bus, pcidev->dev, 
pcidev->func);
+        FILE *f = fopen(sysfs_path, "r");
+        unsigned int start = 0, end = 0, flags = 0, size = 0;
+        int irq = 0;
+        int i;
+
+        if (f == NULL) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+            return -1;
+        }
+        for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
+            fscanf(f, "0x%x 0x%x 0x%x\n", &start, &end, &flags);
+            size = end - start + 1;
+            if (start) {
+                if (flags & PCI_BAR_IO) {
+                    rc = xc_domain_ioport_permission(ctx->xch, domid, start, 
size, 1);
+                    if (rc < 0)
+                        XL_LOG(ctx, XL_LOG_ERROR, "Error: 
xc_domain_ioport_permission error 0x%x/0x%x:  %d\n", start, size, rc);
+                } else {
+                    rc = xc_domain_iomem_permission(ctx->xch, domid, 
start>>XC_PAGE_SHIFT,
+                                                    
(size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 1);
+                    if (rc < 0)
+                        XL_LOG(ctx, XL_LOG_ERROR, "Error: 
xc_domain_iomem_permission error 0x%x/0x%x:  %d\n", start, size, rc);
+                }
+            }
+        }
+        fclose(f);
+        sysfs_path = libxl_sprintf(ctx, "SYSFS_PCI_DEV/"PCI_BDF"/irq", 
pcidev->domain,
+                                   pcidev->bus, pcidev->dev, pcidev->func);
+        f = fopen(sysfs_path, "r");
+        if (f == NULL) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+            goto out;
+        }
+        fscanf(f, "%u", &irq);
+        if (irq) {
+            rc = xc_physdev_map_pirq(ctx->xch, domid, irq, &irq);
+            if (rc < 0) {
+                XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_physdev_map_pirq irq=%d: 
%d\n", irq, rc);
+            }
+            rc = xc_domain_irq_permission(ctx->xch, domid, irq, 1);
+            if (rc < 0) {
+                XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_domain_irq_permission 
irq=%d: %d\n", irq, rc);
+            }
+        }
+        fclose(f);
+    }
+out:
+    if ((rc = xc_assign_device(ctx->xch, domid, pcidev->value)) < 0)
+        XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_assign_device error %d\n", rc);
+
+    libxl_device_pci_add_xenstore(ctx, domid, pcidev);
+    return 0;
+}
+
+int libxl_device_pci_remove(struct libxl_ctx *ctx, uint32_t domid, 
libxl_device_pci *pcidev)
+{
+    char path[50];
+    char *state;
+    int hvm, rc;
+
+    /* TODO: check if the device can be detached */
+
+    hvm = is_hvm(ctx, domid);
+    if (hvm) {
+        if (libxl_wait_for_device_model(ctx, domid, "running") < 0) {
+            return -1;
+        }
+        snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state", 
domid);
+        state = libxl_xs_read(ctx, XBT_NULL, path);
+        snprintf(path, sizeof(path), 
"/local/domain/0/device-model/%d/parameter", domid);
+        libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
+                       pcidev->bus, pcidev->dev, pcidev->func);
+        snprintf(path, sizeof(path), 
"/local/domain/0/device-model/%d/command", domid);
+        xs_write(ctx->xsh, XBT_NULL, path, "pci-rem", strlen("pci-rem"));
+        if (libxl_wait_for_device_model(ctx, domid, "pci-removed") < 0) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time\n");
+            return -1;
+        }
+        snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state", 
domid);
+        xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
+    } else {
+        char *sysfs_path = libxl_sprintf(ctx, 
"SYSFS_PCI_DEV/"PCI_BDF"/resource", pcidev->domain,
+                                         pcidev->bus, pcidev->dev, 
pcidev->func);
+        FILE *f = fopen(sysfs_path, "r");
+        unsigned int start = 0, end = 0, flags = 0, size = 0;
+        int irq = 0;
+        int i;
+
+        if (f == NULL) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+            goto skip1;
+        }
+        for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
+            fscanf(f, "0x%x 0x%x 0x%x\n", &start, &end, &flags);
+            size = end - start + 1;
+            if (start) {
+                if (flags & PCI_BAR_IO) {
+                    rc = xc_domain_ioport_permission(ctx->xch, domid, start, 
size, 0);
+                    if (rc < 0)
+                        XL_LOG(ctx, XL_LOG_ERROR, "Error: 
xc_domain_ioport_permission error 0x%x/0x%x:  %d\n", start, size, rc);
+                } else {
+                    rc = xc_domain_iomem_permission(ctx->xch, domid, 
start>>XC_PAGE_SHIFT,
+                                                    
(size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 0);
+                    if (rc < 0)
+                        XL_LOG(ctx, XL_LOG_ERROR, "Error: 
xc_domain_iomem_permission error 0x%x/0x%x:  %d\n", start, size, rc);
+                }
+            }
+        }
+        fclose(f);
+skip1:
+        sysfs_path = libxl_sprintf(ctx, "SYSFS_PCI_DEV/"PCI_BDF"/irq", 
pcidev->domain,
+                                   pcidev->bus, pcidev->dev, pcidev->func);
+        f = fopen(sysfs_path, "r");
+        if (f == NULL) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+            goto out;
+        }
+        fscanf(f, "%u", &irq);
+        if (irq) {
+            rc = xc_physdev_unmap_pirq(ctx->xch, domid, irq);
+            if (rc < 0) {
+                XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_physdev_map_pirq irq=%d: 
%d\n", irq, rc);
+            }
+            rc = xc_domain_irq_permission(ctx->xch, domid, irq, 0);
+            if (rc < 0) {
+                XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_domain_irq_permission 
irq=%d: %d\n", irq, rc);
+            }
+        }
+        fclose(f);
+    }
+out:
+    libxl_device_pci_remove_xenstore(ctx, domid, pcidev);
+
+    libxl_device_pci_flr(ctx, pcidev->domain, pcidev->bus, pcidev->dev, 
pcidev->func);
+
+    if ((rc = xc_deassign_device(ctx->xch, domid, pcidev->value)) < 0)
+        XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_deassign_device error %d\n", rc);
+    return 0;
+}
+
+libxl_device_pci *libxl_device_pci_list(struct libxl_ctx *ctx, uint32_t domid, 
int *num)
+{
+    char *be_path, *num_devs, *xsdev, *xsvdevfn, *xsopts;
+    int n, i;
+    unsigned int domain = 0, bus = 0, dev = 0, func = 0, vdevfn = 0;
+    libxl_device_pci *pcidevs;
+
+    be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", 
xs_get_domain_path(ctx->xsh, 0), domid);
+    num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs", 
be_path));
+    if (!num_devs) {
+        *num = 0;
+        return NULL;
+    }
+    n = atoi(num_devs);
+    pcidevs = (libxl_device_pci *) libxl_calloc(ctx, n, 
sizeof(libxl_device_pci));
+    *num = n;
+
+    for (i = 0; i < n; i++) {
+        xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d", 
be_path, i));
+        sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
+        xsvdevfn = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, 
"%s/vdevfn-%d", be_path, i));
+        if (xsvdevfn)
+            vdevfn = strtol(xsvdevfn, (char **) NULL, 16);
+        libxl_device_pci_init(pcidevs + i, domain, bus, dev, func, vdevfn);
+        xsopts = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/opts-%d", 
be_path, i));
+        if (xsopts) {
+            char *saveptr;
+            char *p = strtok_r(xsopts, ",=", &saveptr);
+            do {
+                while (*p == ' ')
+                    p++;
+                if (!strcmp(p, "msitranslate")) {
+                    p = strtok_r(NULL, ",=", &saveptr);
+                    pcidevs[i].msitranslate = atoi(p);
+                } else if (!strcmp(p, "power_mgmt")) {
+                    p = strtok_r(NULL, ",=", &saveptr);
+                    pcidevs[i].power_mgmt = atoi(p);
+                }
+            } while ((p = strtok_r(NULL, ",=", &saveptr)) != NULL);
+        }
+    }
+    return pcidevs;
+}
+
+int libxl_device_pci_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+{
+    libxl_device_pci *pcidevs;
+    int num, i;
+
+    pcidevs = libxl_device_pci_list(ctx, domid, &num);
+    for (i = 0; i < num; i++) {
+        if (libxl_device_pci_remove(ctx, domid, pcidevs + i) < 0)
+            return -1;
+    }
+    return 0;
+}
+
diff -r 32a27fe01e07 tools/libxl/libxl.h
--- a/tools/libxl/libxl.h       Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl.h       Thu Nov 12 18:04:19 2009 +0000
@@ -148,6 +148,25 @@
     libxl_nic_type nictype;
 } libxl_device_nic;
 
+typedef struct  {
+    union {
+        unsigned int value;
+        struct {
+            unsigned int reserved1:2;
+            unsigned int reg:6;
+            unsigned int func:3;
+            unsigned int dev:5;
+            unsigned int bus:8;
+            unsigned int reserved2:7;
+            unsigned int enable:1;
+        };
+    };
+    unsigned int domain;
+    unsigned int vdevfn;
+    bool msitranslate;
+    bool power_mgmt;
+} libxl_device_pci;
+
 #define ERROR_FAIL (-2)
 #define ERROR_NI (-101)
 #define ERROR_NOMEM (-1032)
@@ -194,8 +213,14 @@
 int libxl_device_vfb_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid);
 int libxl_device_vfb_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid);
 
-int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid);
-int libxl_device_pci_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid);
-int libxl_device_pci_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid);
+#define PCI_BDF                "%04x:%02x:%02x.%01x"
+#define PCI_BDF_VDEVFN         "%04x:%02x:%02x.%01x@%02x"
+int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid, 
libxl_device_pci *pcidev);
+int libxl_device_pci_remove(struct libxl_ctx *ctx, uint32_t domid, 
libxl_device_pci *pcidev);
+int libxl_device_pci_shutdown(struct libxl_ctx *ctx, uint32_t domid);
+libxl_device_pci *libxl_device_pci_list(struct libxl_ctx *ctx, uint32_t domid, 
int *num);
+int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
+                          unsigned int bus, unsigned int dev,
+                          unsigned int func, unsigned int vdevfn);
 
 #endif
diff -r 32a27fe01e07 tools/libxl/libxl_device.c
--- a/tools/libxl/libxl_device.c        Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl_device.c        Thu Nov 12 18:04:19 2009 +0000
@@ -15,6 +15,7 @@
  */
 
 #include <string.h>
+#include <stdio.h>
 #include "libxl.h"
 #include "libxl_internal.h"
 #include <sys/time.h> /* for struct timeval */
@@ -83,9 +84,12 @@
     libxl_xs_writev(ctx, t, backend_path, bents);
     libxl_xs_writev(ctx, t, frontend_path, fents);
 
-    if (!xs_transaction_end(ctx->xsh, t, 0))
+    if (!xs_transaction_end(ctx->xsh, t, 0)) {
         if (errno == EAGAIN)
             goto retry_transaction;
+        else
+            XL_LOG(ctx, XL_LOG_ERROR, "xs transaction failed errno=%d\n", 
errno);
+    }
     return 0;
 }
 
@@ -154,7 +158,7 @@
     char *state = libxl_xs_read(ctx, XBT_NULL, state_path);
     if (!state)
         return 0;
-    if (atoi(state) <= 3) {
+    if (atoi(state) != 4) {
         xs_rm(ctx->xsh, XBT_NULL, be_path);
         return 0;
     }
@@ -240,3 +244,72 @@
     flexarray_free(toremove);
     return 0;
 }
+
+int libxl_device_pci_flr(struct libxl_ctx *ctx, unsigned int domain, unsigned 
int bus,
+                         unsigned int dev, unsigned int func)
+{
+    FILE *fd;
+
+    fd = fopen("/sys/bus/pci/drivers/pciback/do_flr", "w");
+    if (fd != NULL) {
+        fprintf(fd, PCI_BDF, domain, bus, dev, func);
+        fclose(fd);
+        return 0;
+    }
+    XL_LOG(ctx, XL_LOG_ERROR, "Pciback doesn't support do_flr, cannot flr the 
device\n");
+    return -1;
+}
+
+int libxl_wait_for_device_model(struct libxl_ctx *ctx, uint32_t domid, char 
*state)
+{
+    char path[50];
+    char *p;
+    int watchdog = 100;
+    unsigned int len;
+
+    snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state", 
domid);
+    while (watchdog > 0) {
+        p = xs_read(ctx->xsh, XBT_NULL, path, &len);
+        if (p == NULL) {
+            usleep(100000);
+            watchdog--;
+        } else {
+            if (state == NULL || !strcmp(state, p)) {
+                free(p);
+                return 0;
+            } else {
+                free(p);
+                usleep(100000);
+                watchdog--;
+            }
+        }
+    }
+    XL_LOG(ctx, XL_LOG_ERROR, "Device Model not ready\n");
+    return -1;
+}
+
+int libxl_wait_for_backend(struct libxl_ctx *ctx, char *be_path, char *state)
+{
+    int watchdog = 100;
+    unsigned int len;
+    char *p;
+    char *path = libxl_sprintf(ctx, "%s/state", be_path);
+
+    while (watchdog > 0) {
+        p = xs_read(ctx->xsh, XBT_NULL, path, &len);
+        if (p == NULL) {
+            XL_LOG(ctx, XL_LOG_ERROR, "Backend %s does not exist\n", be_path);
+            return -1;
+        } else {
+            if (!strcmp(p, state)) {
+                return 0;
+            } else {
+                usleep(100000);
+                watchdog--;
+            }
+        }
+    }
+    XL_LOG(ctx, XL_LOG_ERROR, "Backend %s not ready\n", be_path);
+    return -1;
+}
+
diff -r 32a27fe01e07 tools/libxl/libxl_internal.h
--- a/tools/libxl/libxl_internal.h      Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl_internal.h      Thu Nov 12 18:04:19 2009 +0000
@@ -72,6 +72,12 @@
     libxl_device_kinds kind;
 } libxl_device;
 
+#define XC_PCI_BDF             "0x%x, 0x%x, 0x%x, 0x%x"
+#define AUTO_PHP_SLOT          0x100
+#define SYSFS_PCI_DEV          /sys/bus/pci/devices
+#define PROC_PCI_NUM_RESOURCES 7
+#define PCI_BAR_IO             0x01
+
 #define PRINTF_ATTRIBUTE(x, y) __attribute__((format(printf, x, y)))
 
 /* memory allocation tracking/helpers */
@@ -92,7 +98,7 @@
 char *libxl_xs_read(struct libxl_ctx *ctx, xs_transaction_t t, char *path);
 char **libxl_xs_directory(struct libxl_ctx *ctx, xs_transaction_t t, char 
*path, unsigned int *nb);
 
-/* from xd_dom */
+/* from xl_dom */
 int is_hvm(struct libxl_ctx *ctx, uint32_t domid);
 int build_pre(struct libxl_ctx *ctx, uint32_t domid,
               libxl_domain_build_info *info, libxl_domain_build_state *state);
@@ -109,7 +115,7 @@
                    libxl_domain_build_info *info, libxl_domain_build_state 
*state, int fd);
 int core_suspend(struct libxl_ctx *ctx, uint32_t domid, int fd, int hvm, int 
live, int debug);
 
-/* from xd_device */
+/* from xl_device */
 char *device_disk_backend_type_of_phystype(libxl_disk_phystype phystype);
 char *device_disk_string_of_phystype(libxl_disk_phystype phystype);
 
@@ -120,13 +126,17 @@
                              char **bents, char **fents);
 int libxl_device_destroy(struct libxl_ctx *ctx, char *be_path, int force);
 int libxl_devices_destroy(struct libxl_ctx *ctx, uint32_t domid, int force);
+int libxl_wait_for_device_model(struct libxl_ctx *ctx, uint32_t domid, char 
*state);
+int libxl_wait_for_backend(struct libxl_ctx *ctx, char *be_path, char *state);
+int libxl_device_pci_flr(struct libxl_ctx *ctx, unsigned int domain, unsigned 
int bus,
+                         unsigned int dev, unsigned int func);
 
 /* from xenguest (helper */
 int hvm_build_set_params(int handle, uint32_t domid,
                          int apic, int acpi, int pae, int nx, int viridian,
                          int vcpus, int store_evtchn, unsigned long 
*store_mfn);
 
-/* xd_exec */
+/* xl_exec */
 int libxl_exec(struct libxl_ctx *ctx, int stdinfd, int stdoutfd, int stderrfd,
                char *arg0, char **args);
 
diff -r 32a27fe01e07 tools/libxl/xl.c
--- a/tools/libxl/xl.c  Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/xl.c  Thu Nov 12 18:04:19 2009 +0000
@@ -40,6 +40,8 @@
                         int num_disks,
                         libxl_device_nic *vifs,
                         int num_vifs,
+                        libxl_device_pci *pcidevs,
+                        int num_pcidevs,
                         libxl_device_model_info *dm_info)
 {
     int i;
@@ -104,6 +106,12 @@
         printf("model %s\n", vifs[i].model);
         printf("mac %02x:%02x:%02x:%02x:%02x:%02x\n", vifs[i].mac[0], 
vifs[i].mac[1], vifs[i].mac[2], vifs[i].mac[3], vifs[i].mac[4], vifs[i].mac[5]);
         printf("smac %s\n", vifs[i].mac);
+    }
+
+    for (i = 0; i < num_pcidevs; i++) {
+        printf("\n\n\n*** pcidevs_info: %d ***\n", i);
+        printf("pci dev "PCI_BDF_VDEVFN"\n", pcidevs[i].domain, 
pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func, pcidevs[i].vdevfn);
+        printf("opts msitranslate %d power_mgmt %d\n", 
pcidevs[i].msitranslate, pcidevs[i].power_mgmt);
     }
 
     printf("\n\n\n*** device_model_info ***\n");
@@ -286,13 +294,17 @@
                               int *num_disks,
                               libxl_device_nic **vifs,
                               int *num_vifs,
+                              libxl_device_pci **pcidevs,
+                              int *num_pcidevs,
                               libxl_device_model_info *dm_info)
 {
     const char *buf;
     xen_uuid_t uuid[16];
     long l;
     struct config_t config;
-    struct config_setting_t *vbds, *nics;
+    struct config_setting_t *vbds, *nics, *pcis;
+    int pci_power_mgmt = 0;
+    int pci_msitranslate = 1;
 
     config_init (&config);
 
@@ -482,6 +494,48 @@
         }
     }
 
+    if (config_lookup_int (&config, "pci_msitranslate", &l) == CONFIG_TRUE)
+        pci_msitranslate = l;
+
+    if (config_lookup_int (&config, "pci_power_mgmt", &l) == CONFIG_TRUE)
+        pci_power_mgmt = l;
+
+    if ((pcis = config_lookup (&config, "pci")) != NULL) {
+        *num_pcidevs = 0;
+        *pcidevs = NULL;
+        while ((buf = config_setting_get_string_elem (pcis, *num_pcidevs)) != 
NULL) {
+            unsigned int domain = 0, bus = 0, dev = 0, func = 0, vdevfn = 0;
+            char *buf2 = strdup(buf);
+            char *p;
+            *pcidevs = (libxl_device_pci *) realloc(*pcidevs, sizeof 
(libxl_device_pci) * ((*num_pcidevs) + 1));
+            memset(*pcidevs + *num_pcidevs, 0x00, sizeof(libxl_device_pci));
+            p = strtok(buf2, ",");
+            if (!p)
+                goto skip_pci;
+            if (!sscanf(p, PCI_BDF_VDEVFN, &domain, &bus, &dev, &func, 
&vdevfn)) {
+                sscanf(p, "%02x:%02x.%01x@%02x", &bus, &dev, &func, &vdevfn);
+                domain = 0;
+            }
+            libxl_device_pci_init(*pcidevs + *num_pcidevs, domain, bus, dev, 
func, vdevfn);
+            (*pcidevs)[*num_pcidevs].msitranslate = pci_msitranslate;
+            (*pcidevs)[*num_pcidevs].power_mgmt = pci_power_mgmt;
+            while ((p = strtok(NULL, ",=")) != NULL) {
+                while (*p == ' ')
+                    p++;
+                if (!strcmp(p, "msitranslate")) {
+                    p = strtok(NULL, ",=");
+                    (*pcidevs)[*num_pcidevs].msitranslate = atoi(p);
+                } else if (!strcmp(p, "power_mgmt")) {
+                    p = strtok(NULL, ",=");
+                    (*pcidevs)[*num_pcidevs].power_mgmt = atoi(p);
+                }
+            }
+            *num_pcidevs = (*num_pcidevs) + 1;
+skip_pci:
+            free(buf2);
+        }
+    }
+
     /* init dm from c and b */
     init_dm_info(dm_info, c_info, b_info);
 
@@ -527,13 +581,14 @@
     libxl_device_model_info dm_info;
     libxl_device_disk *disks = NULL;
     libxl_device_nic *vifs = NULL;
-    int num_disks = 0, num_vifs = 0;
+    libxl_device_pci *pcidevs = NULL;
+    int num_disks = 0, num_vifs = 0, num_pcidevs = 0;
     int i;
 
     printf("Parsing config file %s\n", filename);
-    parse_config_file(filename, &info1, &info2, &disks, &num_disks, &vifs, 
&num_vifs, &dm_info);
+    parse_config_file(filename, &info1, &info2, &disks, &num_disks, &vifs, 
&num_vifs, &pcidevs, &num_pcidevs, &dm_info);
     if (debug)
-        printf_info(&info1, &info2, disks, num_disks, vifs, num_vifs, 
&dm_info);
+        printf_info(&info1, &info2, disks, num_disks, vifs, num_vifs, pcidevs, 
num_pcidevs, &dm_info);
 
     libxl_ctx_init(&ctx);
     libxl_ctx_set_log(&ctx, log_callback, NULL);
@@ -551,6 +606,8 @@
         libxl_device_nic_add(&ctx, domid, &vifs[i]);
     }
     libxl_create_device_model(&ctx, &dm_info, vifs, num_vifs);
+    for (i = 0; i < num_pcidevs; i++)
+        libxl_device_pci_add(&ctx, domid, &pcidevs[i]);
     libxl_domain_unpause(&ctx, domid);
 
 }
@@ -560,9 +617,12 @@
     if (!command || !strcmp(command, "help")) {
         printf("Usage xl <subcommand> [args]\n\n");
         printf("xl full list of subcommands:\n\n");
-        printf(" create                                create a domain from 
config file <filename>\n\n");
+        printf(" create                        create a domain from config 
file <filename>\n\n");
         printf(" list                          list information about all 
domains\n\n");
         printf(" destroy                       terminate a domain 
immediately\n\n");
+        printf(" pci-attach                    insert a new pass-through pci 
device\n\n");
+        printf(" pci-detach                    remove a domain's pass-through 
pci device\n\n");
+        printf(" pci-list                      list pass-through pci devices 
for a domain\n\n");
     } else if(!strcmp(command, "create")) {
         printf("Usage: xl create <ConfigFile> [options] [vars]\n\n");
         printf("Create a domain based on <ConfigFile>.\n\n");
@@ -572,11 +632,166 @@
     } else if(!strcmp(command, "list")) {
         printf("Usage: xl list [Domain]\n\n");
         printf("List information about all/some domains.\n\n");
+    } else if(!strcmp(command, "pci-attach")) {
+        printf("Usage: xl pci-attach <Domain> <BDF> [Virtual Slot]\n\n");
+        printf("Insert a new pass-through pci device.\n\n");
+    } else if(!strcmp(command, "pci-detach")) {
+        printf("Usage: xl pci-detach <Domain> <BDF>\n\n");
+        printf("Remove a domain's pass-through pci device.\n\n");
+    } else if(!strcmp(command, "pci-list")) {
+        printf("Usage: xl pci-list <Domain>\n\n");
+        printf("List pass-through pci devices for a domain.\n\n");
     } else if(!strcmp(command, "destroy")) {
         printf("Usage: xl destroy <Domain>\n\n");
         printf("Terminate a domain immediately.\n\n");
     }
 }
+
+void pcilist(char *dom)
+{
+    struct libxl_ctx ctx;
+    uint32_t domid;
+    libxl_device_pci *pcidevs;
+    int num, i;
+
+    libxl_ctx_init(&ctx);
+    libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+    if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+        fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+        exit(2);
+    }
+    pcidevs = libxl_device_pci_list(&ctx, domid, &num);
+    if (!num)
+        return;
+    printf("VFn  domain bus  slot func\n");
+    for (i = 0; i < num; i++) {
+        printf("0x%02x 0x%04x 0x%02x 0x%02x 0x%01x\n", pcidevs[i].vdevfn, 
pcidevs[i].domain, pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func);
+    }
+}
+
+int main_pcilist(int argc, char **argv)
+{
+    int opt;
+    char *domname = NULL;
+
+    while ((opt = getopt(argc, argv, "h")) != -1) {
+        switch (opt) {
+        case 'h':
+            help("pci-list");
+            exit(0);
+        default:
+            fprintf(stderr, "option not supported\n");
+            break;
+        }
+    }
+    if (optind >= argc) {
+        help("pci-list");
+        exit(2);
+    }
+
+    domname = argv[optind];
+
+    pcilist(domname);
+    exit(0);
+}
+
+void pcidetach(char *dom, char *bdf)
+{
+    struct libxl_ctx ctx;
+    uint32_t domid;
+    libxl_device_pci pcidev;
+    unsigned int domain, bus, dev, func;
+
+    libxl_ctx_init(&ctx);
+    libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+    if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+        fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+        exit(2);
+    }
+    memset(&pcidev, 0x00, sizeof(pcidev));
+    sscanf(bdf, PCI_BDF, &domain, &bus, &dev, &func);
+    libxl_device_pci_init(&pcidev, domain, bus, dev, func, 0);
+    libxl_device_pci_remove(&ctx, domid, &pcidev);
+}
+
+int main_pcidetach(int argc, char **argv)
+{
+    int opt;
+    char *domname = NULL, *bdf = NULL;
+
+    while ((opt = getopt(argc, argv, "h")) != -1) {
+        switch (opt) {
+        case 'h':
+            help("pci-attach");
+            exit(0);
+        default:
+            fprintf(stderr, "option not supported\n");
+            break;
+        }
+    }
+    if (optind >= argc - 1) {
+        help("pci-detach");
+        exit(2);
+    }
+
+    domname = argv[optind];
+    bdf = argv[optind + 1];
+
+    pcidetach(domname, bdf);
+    exit(0);
+}
+void pciattach(char *dom, char *bdf, char *vs)
+{
+    struct libxl_ctx ctx;
+    uint32_t domid;
+    libxl_device_pci pcidev;
+    unsigned int domain, bus, dev, func;
+
+    libxl_ctx_init(&ctx);
+    libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+    if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+        fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+        exit(2);
+    }
+    memset(&pcidev, 0x00, sizeof(pcidev));
+    sscanf(bdf, PCI_BDF, &domain, &bus, &dev, &func);
+    libxl_device_pci_init(&pcidev, domain, bus, dev, func, 0);
+    libxl_device_pci_add(&ctx, domid, &pcidev);
+}
+
+int main_pciattach(int argc, char **argv)
+{
+    int opt;
+    char *domname = NULL, *bdf = NULL, *vs = NULL;
+
+    while ((opt = getopt(argc, argv, "h")) != -1) {
+        switch (opt) {
+        case 'h':
+            help("pci-attach");
+            exit(0);
+        default:
+            fprintf(stderr, "option not supported\n");
+            break;
+        }
+    }
+    if (optind >= argc - 1) {
+        help("pci-attach");
+        exit(2);
+    }
+
+    domname = argv[optind];
+    bdf = argv[optind + 1];
+
+    if (optind + 1 < argc)
+        vs = argv[optind + 2];
+
+    pciattach(domname, bdf, vs);
+    exit(0);
+}
+
 
 void destroy_domain(char *p)
 {
@@ -713,6 +928,12 @@
         main_list(argc - 1, argv + 1);
     } else if (!strcmp(argv[1], "destroy")) {
         main_destroy(argc - 1, argv + 1);
+    } else if (!strcmp(argv[1], "pci-attach")) {
+        main_pciattach(argc - 1, argv + 1);
+    } else if (!strcmp(argv[1], "pci-detach")) {
+        main_pcidetach(argc - 1, argv + 1);
+    } else if (!strcmp(argv[1], "pci-list")) {
+        main_pcilist(argc - 1, argv + 1);
     } else if (!strcmp(argv[1], "help")) {
         if (argc > 2)
             help(argv[2]);

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.