devices: use device manager to manage all devices

Fixes #50

Previously the devices are created with device manager and laterly
attached to hypervisor with "device.Attach()", this could work, but
there's no way to remember the reference count for every device, which
means if we plug one device to hypervisor twice, it's truly inserted
twice, but actually we only need to insert once but use it in many
places.

Use device manager as a consolidated entrypoint of device management can
give us a way to handle many "references" to single device, because it
can save all devices and remember it's use count.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>

virtcontainers/container.go

@@ -440,8 +440,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 		if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
 			// TODO: remove dependency of package drivers
 			b := &drivers.BlockDevice{
-				DevType: config.DeviceBlock,
-				DeviceInfo: config.DeviceInfo{
+				DeviceInfo: &config.DeviceInfo{
 					HostPath:      m.Source,
 					ContainerPath: m.Destination,
 					DevType:       "b",
@@ -574,12 +573,13 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 	} else {
 		// If devices were not found in storage, create Device implementations
 		// from the configuration. This should happen at create.
-
-		devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos)
-		if err != nil {
-			return &Container{}, err
+		for _, info := range contConfig.DeviceInfos {
+			dev, err := sandbox.devManager.NewDevice(info)
+			if err != nil {
+				return &Container{}, err
+			}
+			c.devices = append(c.devices, dev)
 		}
-		c.devices = devices
 	}
 	return c, nil
 }
@@ -1022,7 +1022,7 @@ func (c *Container) hotplugDrive() error {
 
 	// Add drive with id as container id
 	devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
-	drive := drivers.Drive{
+	drive := config.BlockDrive{
 		File:   devicePath,
 		Format: "raw",
 		ID:     devID,
@@ -1059,7 +1059,7 @@ func (c *Container) removeDrive() (err error) {
 		c.Logger().Info("unplugging block device")
 
 		devID := utils.MakeNameID("drive", c.id, maxDevIDSize)
-		drive := &drivers.Drive{
+		drive := &config.BlockDrive{
 			ID: devID,
 		}
 

virtcontainers/device/api/interface.go

@@ -27,6 +27,7 @@ func DeviceLogger() *logrus.Entry {
 // DeviceReceiver is an interface used for accepting devices
 // a device should be attached/added/plugged to a DeviceReceiver
 type DeviceReceiver interface {
+	// these are for hotplug/hot-unplug devices to/from hypervisor
 	HotplugAddDevice(Device, config.DeviceType) error
 	HotplugRemoveDevice(Device, config.DeviceType) error
 
@@ -51,11 +52,29 @@ type VhostUserDevice interface {
 type Device interface {
 	Attach(DeviceReceiver) error
 	Detach(DeviceReceiver) error
+	// ID returns device identifier
+	DeviceID() string
+	// DeviceType indicates which kind of device it is
+	// e.g. block, vfio or vhost user
 	DeviceType() config.DeviceType
+	// GetDeviceInfo returns device information that the device is created based on
+	GetDeviceInfo() *config.DeviceInfo
+	// GetDeviceDrive returns device specific data used for hotplugging by hypervisor
+	// Caller could cast the return value to device specific struct
+	// e.g. Block device returns *config.BlockDrive and
+	// vfio device returns *config.VFIODrive
+	GetDeviceDrive() interface{}
+	// IsAttached checks if the device is attached
+	IsAttached() bool
 }
 
 // DeviceManager can be used to create a new device, this can be used as single
 // device management object.
 type DeviceManager interface {
-	NewDevices(devInfos []config.DeviceInfo) ([]Device, error)
+	NewDevice(config.DeviceInfo) (Device, error)
+	AttachDevice(string, DeviceReceiver) error
+	DetachDevice(string, DeviceReceiver) error
+	IsDeviceAttached(string) bool
+	GetDeviceByID(string) Device
+	GetAllDevices() []Device
 }

virtcontainers/device/config/config.go

@@ -49,10 +49,10 @@ var SysIOMMUPath = "/sys/kernel/iommu_groups"
 
 // DeviceInfo is an embedded type that contains device data common to all types of devices.
 type DeviceInfo struct {
-	// Device path on host
+	// Hostpath is device path on host
 	HostPath string
 
-	// Device path inside the container
+	// ContainerPath is device path inside container
 	ContainerPath string
 
 	// Type of device: c, b, u or p
@@ -87,6 +87,40 @@ type DeviceInfo struct {
 	DriverOptions map[string]string
 }
 
+// BlockDrive represents a block storage drive which may be used in case the storage
+// driver has an underlying block storage device.
+type BlockDrive struct {
+	// File is the path to the disk-image/device which will be used with this drive
+	File string
+
+	// Format of the drive
+	Format string
+
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+
+	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
+	Index int
+
+	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
+	PCIAddr string
+
+	// SCSI Address of the block device, in case the device is attached using SCSI driver
+	// SCSI address is in the format SCSI-Id:LUN
+	SCSIAddr string
+
+	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
+	VirtPath string
+}
+
+// VFIODrive represents a VFIO drive used for hotplugging
+type VFIODrive struct {
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+	// BDF (Bus:Device.Function) of the PCI address
+	BDF string
+}
+
 // VhostUserDeviceAttrs represents data shared by most vhost-user devices
 type VhostUserDeviceAttrs struct {
 	DevType    DeviceType

virtcontainers/device/drivers/block.go

@@ -7,7 +7,6 @@
 package drivers
 
 import (
-	"encoding/hex"
 	"path/filepath"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
@@ -17,62 +16,24 @@ import (
 
 const maxDevIDSize = 31
 
-// Drive represents a block storage drive which may be used in case the storage
-// driver has an underlying block storage device.
-type Drive struct {
-
-	// Path to the disk-image/device which will be used with this drive
-	File string
-
-	// Format of the drive
-	Format string
-
-	// ID is used to identify this drive in the hypervisor options.
-	ID string
-
-	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
-	Index int
-
-	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
-	PCIAddr string
-}
-
 // BlockDevice refers to a block storage device implementation.
 type BlockDevice struct {
-	DevType    config.DeviceType
-	DeviceInfo config.DeviceInfo
-
-	// SCSI Address of the block device, in case the device is attached using SCSI driver
-	// SCSI address is in the format SCSI-Id:LUN
-	SCSIAddr string
-
-	// Path at which the device appears inside the VM, outside of the container mount namespace
-	VirtPath string
-
-	// PCI Slot of the block device
-	PCIAddr string
-
-	BlockDrive *Drive
+	ID         string
+	DeviceInfo *config.DeviceInfo
+	BlockDrive *config.BlockDrive
 }
 
 // NewBlockDevice creates a new block device based on DeviceInfo
-func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice {
+func NewBlockDevice(devInfo *config.DeviceInfo) *BlockDevice {
 	return &BlockDevice{
-		DevType:    config.DeviceBlock,
+		ID:         devInfo.ID,
 		DeviceInfo: devInfo,
 	}
 }
 
 // Attach is standard interface of api.Device, it's used to add device to some
 // DeviceReceiver
 func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
-	randBytes, err := utils.GenerateRandomBytes(8)
-	if err != nil {
-		return err
-	}
-
-	device.DeviceInfo.ID = hex.EncodeToString(randBytes)
-
 	// Increment the block index for the sandbox. This is used to determine the name
 	// for the block device in the case where the block device is used as container
 	// rootfs and the predicted block device name needs to be provided to the agent.
@@ -88,58 +49,75 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 		return err
 	}
 
-	drive := Drive{
+	drive := &config.BlockDrive{
 		File:   device.DeviceInfo.HostPath,
 		Format: "raw",
 		ID:     utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
 		Index:  index,
 	}
 
-	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
-	device.BlockDrive = &drive
-	if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
-		return err
-	}
-
-	device.DeviceInfo.Hotplugged = true
-
 	driveName, err := utils.GetVirtDriveName(index)
 	if err != nil {
 		return err
 	}
 
 	customOptions := device.DeviceInfo.DriverOptions
 	if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
-		device.VirtPath = filepath.Join("/dev", driveName)
-		device.PCIAddr = drive.PCIAddr
+		drive.VirtPath = filepath.Join("/dev", driveName)
 	} else {
 		scsiAddr, err := utils.GetSCSIAddress(index)
 		if err != nil {
 			return err
 		}
 
-		device.SCSIAddr = scsiAddr
+		drive.SCSIAddr = scsiAddr
 	}
 
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
+	device.BlockDrive = drive
+	if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
+		return err
+	}
+
+	device.DeviceInfo.Hotplugged = true
+
 	return nil
 }
 
 // Detach is standard interface of api.Device, it's used to remove device from some
 // DeviceReceiver
 func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error {
-	if device.DeviceInfo.Hotplugged {
-		deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
-
-		if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
-			deviceLogger().WithError(err).Error("Failed to unplug block device")
-			return err
-		}
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
 
+	if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil {
+		deviceLogger().WithError(err).Error("Failed to unplug block device")
+		return err
 	}
+	device.DeviceInfo.Hotplugged = false
 	return nil
 }
 
+// IsAttached checks if the device is attached
+func (device *BlockDevice) IsAttached() bool {
+	return device.DeviceInfo.Hotplugged
+}
+
 // DeviceType is standard interface of api.Device, it returns device type
 func (device *BlockDevice) DeviceType() config.DeviceType {
-	return device.DevType
+	return config.DeviceBlock
+}
+
+// DeviceID returns device ID
+func (device *BlockDevice) DeviceID() string {
+	return device.ID
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *BlockDevice) GetDeviceInfo() *config.DeviceInfo {
+	return device.DeviceInfo
+}
+
+// GetDeviceDrive returns device information used for creating
+func (device *BlockDevice) GetDeviceDrive() interface{} {
+	return device.BlockDrive
 }

virtcontainers/device/drivers/generic.go

@@ -13,14 +13,14 @@ import (
 
 // GenericDevice refers to a device that is neither a VFIO device or block device.
 type GenericDevice struct {
-	DevType    config.DeviceType
-	DeviceInfo config.DeviceInfo
+	ID         string
+	DeviceInfo *config.DeviceInfo
 }
 
 // NewGenericDevice creates a new GenericDevice
-func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice {
+func NewGenericDevice(devInfo *config.DeviceInfo) *GenericDevice {
 	return &GenericDevice{
-		DevType:    config.DeviceGeneric,
+		ID:         devInfo.ID,
 		DeviceInfo: devInfo,
 	}
 }
@@ -35,7 +35,27 @@ func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error {
 	return nil
 }
 
+// IsAttached checks if the device is attached
+func (device *GenericDevice) IsAttached() bool {
+	return device.DeviceInfo.Hotplugged
+}
+
+// DeviceID returns device ID
+func (device *GenericDevice) DeviceID() string {
+	return device.ID
+}
+
 // DeviceType is standard interface of api.Device, it returns device type
 func (device *GenericDevice) DeviceType() config.DeviceType {
-	return device.DevType
+	return config.DeviceGeneric
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *GenericDevice) GetDeviceInfo() *config.DeviceInfo {
+	return device.DeviceInfo
+}
+
+// GetDeviceDrive returns device information used for creating
+func (device *GenericDevice) GetDeviceDrive() interface{} {
+	return device.DeviceInfo
 }