commit 24b8be890d2e3f2d309c7f55ffd5b1125f63d056
Author: Osier Yang <jyang@redhat.com>
Date:   Thu Oct 13 12:05:04 2011 +0800

    qemu: Do not reattach PCI device used by other domain when shutdown
    
    When failing on starting a domain, it tries to reattach all the PCI
    devices defined in the domain conf, regardless of whether the devices
    are still used by other domain. This will cause the devices to be deleted
    from the list qemu_driver->activePciHostdevs, thus the devices will be
    thought as usable even if it's not true. And following commands
    nodedev-{reattach,reset} will be successful.
    
    How to reproduce:
      1) Define two domains with same PCI device defined in the confs.
      2) # virsh start domain1
      3) # virsh start domain2
      4) # virsh nodedev-reattach $pci_device
    
    You will see the device will be reattached to host successfully.
    As pciDeviceReattach just check if the device is still used by
    other domain via checking if the device is in list driver->activePciHostdevs,
    however, the device is deleted from the list by step 2).
    
    This patch is to prohibit the bug by:
      1) Prohibit a domain starting or device attachment right at
         preparation period (qemuPrepareHostdevPCIDevices) if the
         device is in list driver->activePciHostdevs, which means
         it's used by other domain.
    
      2) Introduces a new field for struct _pciDevice, (const char *used_by),
         it will be set as the domain name at preparation period,
         (qemuPrepareHostdevPCIDevices). Thus we can prohibit deleting
         the device from driver->activePciHostdevs if it's still used by
         other domain when stopping the domain process.
    
    * src/pci.h (define two internal functions, pciDeviceSetUsedBy and
        pciDevceGetUsedBy)
    * src/pci.c (new field "const char *used_by" for struct _pciDevice,
        implementations for the two new functions)
    * src/libvirt_private.syms (Add the two new internal functions)
    * src/qemu_hostdev.h (Modify the definition of functions
        qemuPrepareHostdevPCIDevices, and qemuDomainReAttachHostdevDevices)
    * src/qemu_hostdev.c (Prohibit preparation and don't delete the
        device from activePciHostdevs list if it's still used by other domain)
    * src/qemu_hotplug.c (Update function usage, as the definitions are
        changed)
    
    Signed-off-by: Eric Blake <eblake@redhat.com>

Index: libvirt-0.9.6/src/libvirt_private.syms
===================================================================
--- libvirt-0.9.6.orig/src/libvirt_private.syms
+++ libvirt-0.9.6/src/libvirt_private.syms
@@ -872,6 +872,8 @@ virNWFilterHashTableRemoveEntry;
 pciDettachDevice;
 pciDeviceFileIterate;
 pciDeviceGetManaged;
+pciDeviceGetName;
+pciDeviceGetUsedBy;
 pciDeviceIsAssignable;
 pciDeviceIsVirtualFunction;
 pciDeviceListAdd;
@@ -884,6 +886,7 @@ pciDeviceListSteal;
 pciDeviceNetName;
 pciDeviceReAttachInit;
 pciDeviceSetManaged;
+pciDeviceSetUsedBy;
 pciFreeDevice;
 pciGetDevice;
 pciGetPhysicalFunction;
Index: libvirt-0.9.6/src/qemu/qemu_hostdev.c
===================================================================
--- libvirt-0.9.6.orig/src/qemu/qemu_hostdev.c
+++ libvirt-0.9.6/src/qemu/qemu_hostdev.c
@@ -1,7 +1,7 @@
 /*
  * qemu_hostdev.c: QEMU hostdev management
  *
- * Copyright (C) 2006-2007, 2009-2010 Red Hat, Inc.
+ * Copyright (C) 2006-2007, 2009-2011 Red Hat, Inc.
  * Copyright (C) 2006 Daniel P. Berrange
  *
  * This library is free software; you can redistribute it and/or
@@ -101,6 +101,7 @@ cleanup:
 
 
 int qemuPrepareHostdevPCIDevices(struct qemud_driver *driver,
+                                 const char *name,
                                  virDomainHostdevDefPtr *hostdevs,
                                  int nhostdevs)
 {
@@ -111,37 +112,63 @@ int qemuPrepareHostdevPCIDevices(struct
     if (!(pcidevs = qemuGetPciHostDeviceList(hostdevs, nhostdevs)))
         return -1;
 
-    /* We have to use 4 loops here. *All* devices must
+    /* We have to use 6 loops here. *All* devices must
      * be detached before we reset any of them, because
      * in some cases you have to reset the whole PCI,
      * which impacts all devices on it. Also, all devices
      * must be reset before being marked as active.
      */
 
-    /* XXX validate that non-managed device isn't in use, eg
+    /* Loop 1: validate that non-managed device isn't in use, eg
      * by checking that device is either un-bound, or bound
      * to pci-stub.ko
      */
 
     for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
         pciDevice *dev = pciDeviceListGet(pcidevs, i);
-        if (!pciDeviceIsAssignable(dev, !driver->relaxedACS))
-            goto reattachdevs;
+        pciDevice *other;
 
+        if (!pciDeviceIsAssignable(dev, !driver->relaxedACS)) {
+            qemuReportError(VIR_ERR_OPERATION_INVALID,
+                            _("PCI device %s is not assignable"),
+                            pciDeviceGetName(dev));
+            goto cleanup;
+        }
+        /* The device is in use by other active domain if
+         * the dev is in list driver->activePciHostdevs.
+         */
+        if ((other = pciDeviceListFind(driver->activePciHostdevs, dev))) {
+            const char *other_name = pciDeviceGetUsedBy(other);
+
+            if (other_name)
+                qemuReportError(VIR_ERR_OPERATION_INVALID,
+                                _("PCI device %s is in use by domain %s"),
+                                pciDeviceGetName(dev), other_name);
+            else
+                qemuReportError(VIR_ERR_OPERATION_INVALID,
+                                _("PCI device %s is already in use"),
+                                pciDeviceGetName(dev));
+            goto cleanup;
+        }
+    }
+
+    /* Loop 2: detach managed devices */
+    for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
+        pciDevice *dev = pciDeviceListGet(pcidevs, i);
         if (pciDeviceGetManaged(dev) &&
             pciDettachDevice(dev, driver->activePciHostdevs) < 0)
             goto reattachdevs;
     }
 
-    /* Now that all the PCI hostdevs have be dettached, we can safely
-     * reset them */
+    /* Loop 3: Now that all the PCI hostdevs have been detached, we
+     * can safely reset them */
     for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
         pciDevice *dev = pciDeviceListGet(pcidevs, i);
         if (pciResetDevice(dev, driver->activePciHostdevs, pcidevs) < 0)
             goto reattachdevs;
     }
 
-    /* Now mark all the devices as active */
+    /* Loop 4: Now mark all the devices as active */
     for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
         pciDevice *dev = pciDeviceListGet(pcidevs, i);
         if (pciDeviceListAdd(driver->activePciHostdevs, dev) < 0) {
@@ -150,7 +177,19 @@ int qemuPrepareHostdevPCIDevices(struct
         }
     }
 
-    /* Now steal all the devices from pcidevs */
+    /* Loop 5: Now set the used_by_domain of the device in
+     * driver->activePciHostdevs as domain name.
+     */
+    for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
+        pciDevice *dev, *activeDev;
+
+        dev = pciDeviceListGet(pcidevs, i);
+        activeDev = pciDeviceListFind(driver->activePciHostdevs, dev);
+
+        pciDeviceSetUsedBy(activeDev, name);
+    }
+
+    /* Loop 6: Now steal all the devices from pcidevs */
     while (pciDeviceListCount(pcidevs) > 0) {
         pciDevice *dev = pciDeviceListGet(pcidevs, 0);
         pciDeviceListSteal(pcidevs, dev);
@@ -183,7 +222,7 @@ static int
 qemuPrepareHostPCIDevices(struct qemud_driver *driver,
                           virDomainDefPtr def)
 {
-    return qemuPrepareHostdevPCIDevices(driver, def->hostdevs, def->nhostdevs);
+    return qemuPrepareHostdevPCIDevices(driver, def->name, def->hostdevs, def->nhostdevs);
 }
 
 
@@ -258,6 +297,7 @@ void qemuReattachPciDevice(pciDevice *de
 
 
 void qemuDomainReAttachHostdevDevices(struct qemud_driver *driver,
+                                      const char *name,
                                       virDomainHostdevDefPtr *hostdevs,
                                       int nhostdevs)
 {
@@ -277,6 +317,16 @@ void qemuDomainReAttachHostdevDevices(st
 
     for (i = 0; i < pciDeviceListCount(pcidevs); i++) {
         pciDevice *dev = pciDeviceListGet(pcidevs, i);
+        pciDevice *activeDev = NULL;
+
+        /* Never delete the dev from list driver->activePciHostdevs
+         * if it's used by other domain.
+         */
+        activeDev = pciDeviceListFind(driver->activePciHostdevs, dev);
+        if (activeDev &&
+            STRNEQ_NULLABLE(name, pciDeviceGetUsedBy(activeDev)))
+            continue;
+
         pciDeviceListDel(driver->activePciHostdevs, dev);
     }
 
@@ -305,5 +355,5 @@ void qemuDomainReAttachHostDevices(struc
     if (!def->nhostdevs)
         return;
 
-    qemuDomainReAttachHostdevDevices(driver, def->hostdevs, def->nhostdevs);
+    qemuDomainReAttachHostdevDevices(driver, def->name, def->hostdevs, def->nhostdevs);
 }
Index: libvirt-0.9.6/src/qemu/qemu_hostdev.h
===================================================================
--- libvirt-0.9.6.orig/src/qemu/qemu_hostdev.h
+++ libvirt-0.9.6/src/qemu/qemu_hostdev.h
@@ -30,12 +30,14 @@
 int qemuUpdateActivePciHostdevs(struct qemud_driver *driver,
                                 virDomainDefPtr def);
 int qemuPrepareHostdevPCIDevices(struct qemud_driver *driver,
+                                 const char *name,
                                  virDomainHostdevDefPtr *hostdevs,
                                  int nhostdevs);
 int qemuPrepareHostDevices(struct qemud_driver *driver,
                            virDomainDefPtr def);
 void qemuReattachPciDevice(pciDevice *dev, struct qemud_driver *driver);
 void qemuDomainReAttachHostdevDevices(struct qemud_driver *driver,
+                                      const char *name,
                                       virDomainHostdevDefPtr *hostdevs,
                                       int nhostdevs);
 void qemuDomainReAttachHostDevices(struct qemud_driver *driver,
Index: libvirt-0.9.6/src/qemu/qemu_hotplug.c
===================================================================
--- libvirt-0.9.6.orig/src/qemu/qemu_hotplug.c
+++ libvirt-0.9.6/src/qemu/qemu_hotplug.c
@@ -859,7 +859,7 @@ int qemuDomainAttachHostPciDevice(struct
         return -1;
     }
 
-    if (qemuPrepareHostdevPCIDevices(driver, &hostdev, 1) < 0)
+    if (qemuPrepareHostdevPCIDevices(driver, vm->def->name, &hostdev, 1) < 0)
         return -1;
 
     if (qemuCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE)) {
@@ -925,7 +925,7 @@ error:
                                         hostdev->info.addr.pci.slot) < 0)
         VIR_WARN("Unable to release PCI address on host device");
 
-    qemuDomainReAttachHostdevDevices(driver, &hostdev, 1);
+    qemuDomainReAttachHostdevDevices(driver, vm->def->name, &hostdev, 1);
 
     VIR_FREE(devstr);
     VIR_FREE(configfd_name);
Index: libvirt-0.9.6/src/util/pci.c
===================================================================
--- libvirt-0.9.6.orig/src/util/pci.c
+++ libvirt-0.9.6/src/util/pci.c
@@ -62,6 +62,7 @@ struct _pciDevice {
     char          name[PCI_ADDR_LEN]; /* domain:bus:slot.function */
     char          id[PCI_ID_LEN];     /* product vendor */
     char          *path;
+    const char    *used_by;           /* The domain which uses the device */
     int           fd;
 
     unsigned      initted;
@@ -1377,6 +1378,12 @@ pciFreeDevice(pciDevice *dev)
     VIR_FREE(dev);
 }
 
+const char *
+pciDeviceGetName(pciDevice *dev)
+{
+    return dev->name;
+}
+
 void pciDeviceSetManaged(pciDevice *dev, unsigned managed)
 {
     dev->managed = !!managed;
@@ -1387,6 +1394,18 @@ unsigned pciDeviceGetManaged(pciDevice *
     return dev->managed;
 }
 
+void
+pciDeviceSetUsedBy(pciDevice *dev, const char *name)
+{
+    dev->used_by = name;
+}
+
+const char *
+pciDeviceGetUsedBy(pciDevice *dev)
+{
+    return dev->used_by;
+}
+
 void pciDeviceReAttachInit(pciDevice *pci)
 {
     pci->unbind_from_stub = 1;
Index: libvirt-0.9.6/src/util/pci.h
===================================================================
--- libvirt-0.9.6.orig/src/util/pci.h
+++ libvirt-0.9.6/src/util/pci.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2009, 2011 Red Hat, Inc.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -39,6 +39,7 @@ pciDevice *pciGetDevice      (unsigned
                               unsigned       slot,
                               unsigned       function);
 void       pciFreeDevice     (pciDevice     *dev);
+const char *pciDeviceGetName (pciDevice     *dev);
 int        pciDettachDevice  (pciDevice     *dev, pciDeviceList *activeDevs);
 int        pciReAttachDevice (pciDevice     *dev, pciDeviceList *activeDevs);
 int        pciResetDevice    (pciDevice     *dev,
@@ -47,6 +48,9 @@ int        pciResetDevice    (pciDevice
 void      pciDeviceSetManaged(pciDevice     *dev,
                               unsigned       managed);
 unsigned  pciDeviceGetManaged(pciDevice     *dev);
+void      pciDeviceSetUsedBy(pciDevice     *dev,
+                             const char *used_by);
+const char *pciDeviceGetUsedBy(pciDevice   *dev);
 void      pciDeviceReAttachInit(pciDevice   *dev);
 
 pciDeviceList *pciDeviceListNew  (void);
