devices: add vhost-user storage configuration

Two parameters are used to set in toml file:

1. Set "enable_vhost_user_store = true" to indicate
that vhost-user storage device feature is enabled.

2. Set "vhost_user_store_path = <Vhost-User-Dir>".
vhost-user socket files will be under
"<Vhost-User-Dir>/block/sockets/"; and device node
for vhost-user device will be under
"<Vhost-User-Dir>/block/devices/"
The default value of "vhost_user_store_path" is
"/var/run/kata-containers/vhost-user/".

Fixes: #2380

Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>

Makefile

@@ -181,6 +181,8 @@ DEFVIRTIOFSEXTRAARGS := []
 DEFENABLEIOTHREADS := false
 DEFENABLEMEMPREALLOC := false
 DEFENABLEHUGEPAGES := false
+DEFENABLEVHOSTUSERSTORE := false
+DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
 DEFENABLESWAP := false
 DEFENABLEDEBUG := false
 DEFDISABLENESTINGCHECKS := false
@@ -440,6 +442,8 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEIOTHREADS
 USER_VARS += DEFENABLEMEMPREALLOC
 USER_VARS += DEFENABLEHUGEPAGES
+USER_VARS += DEFENABLEVHOSTUSERSTORE
+USER_VARS += DEFVHOSTUSERSTOREPATH
 USER_VARS += DEFENABLESWAP
 USER_VARS += DEFENABLEDEBUG
 USER_VARS += DEFDISABLENESTINGCHECKS
@@ -622,6 +626,8 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \
 		-e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \
 		-e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \
+		-e "s|@DEFENABLEVHOSTUSERSTORE@|$(DEFENABLEVHOSTUSERSTORE)|g" \
+		-e "s|@DEFVHOSTUSERSTOREPATH@|$(DEFVHOSTUSERSTOREPATH)|g" \
 		-e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \
 		-e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \
 		-e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \

cli/config/configuration-qemu-virtiofs.toml.in

@@ -172,6 +172,17 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # result in memory pre allocation
 #enable_hugepages = true
 
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

cli/config/configuration-qemu.toml.in

@@ -179,6 +179,17 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # result in memory pre allocation
 #enable_hugepages = true
 
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

pkg/katautils/config-settings.go

@@ -49,6 +49,7 @@ const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "none"
 const defaultDisableImageNvdimm = false
+const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/"
 
 const defaultTemplatePath string = "/run/vc/vm/template"
 const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"

pkg/katautils/config.go

@@ -104,6 +104,8 @@ type hypervisor struct {
 	BlockDeviceCacheSet     bool     `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect  bool     `toml:"block_device_cache_direct"`
 	BlockDeviceCacheNoflush bool     `toml:"block_device_cache_noflush"`
+	EnableVhostUserStore    bool     `toml:"enable_vhost_user_store"`
+	VhostUserStorePath      string   `toml:"vhost_user_store_path"`
 	NumVCPUs                int32    `toml:"default_vcpus"`
 	DefaultMaxVCPUs         uint32   `toml:"default_maxvcpus"`
 	MemorySize              uint32   `toml:"default_memory"`
@@ -404,6 +406,13 @@ func (h hypervisor) guestHookPath() string {
 	return h.GuestHookPath
 }
 
+func (h hypervisor) vhostUserStorePath() string {
+	if h.VhostUserStorePath == "" {
+		return defaultVhostUserStorePath
+	}
+	return h.VhostUserStorePath
+}
+
 func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) {
 	initrd, errInitrd := h.initrd()
 
@@ -651,6 +660,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
+		EnableVhostUserStore:    h.EnableVhostUserStore,
+		VhostUserStorePath:      h.vhostUserStorePath(),
 		GuestHookPath:           h.guestHookPath(),
 	}, nil
 }
@@ -1078,6 +1089,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		HotplugVFIOOnRootBus:    defaultHotplugVFIOOnRootBus,
 		PCIeRootPort:            defaultPCIeRootPort,
 		GuestHookPath:           defaultGuestHookPath,
+		VhostUserStorePath:      defaultVhostUserStorePath,
 		VirtioFSCache:           defaultVirtioFSCacheMode,
 		DisableImageNvdimm:      defaultDisableImageNvdimm,
 	}

pkg/katautils/config_test.go

@@ -165,6 +165,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		MemSlots:              defaultMemSlots,
 		EntropySource:         defaultEntropySource,
 		GuestHookPath:         defaultGuestHookPath,
+		VhostUserStorePath:    defaultVhostUserStorePath,
 		SharedFS:              sharedFS,
 		VirtioFSDaemon:        "/path/to/virtiofsd",
 		VirtioFSCache:         defaultVirtioFSCacheMode,
@@ -628,6 +629,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
 		Msize9p:               defaultMsize9p,
 		GuestHookPath:         defaultGuestHookPath,
+		VhostUserStorePath:    defaultVhostUserStorePath,
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 	}
 
@@ -1198,6 +1200,21 @@ func TestHypervisorDefaultsGuestHookPath(t *testing.T) {
 	assert.Equal(guestHookPath, testGuestHookPath, "custom guest hook path wrong")
 }
 
+func TestHypervisorDefaultsVhostUserStorePath(t *testing.T) {
+	assert := assert.New(t)
+
+	h := hypervisor{}
+	vhostUserStorePath := h.vhostUserStorePath()
+	assert.Equal(vhostUserStorePath, defaultVhostUserStorePath, "default vhost-user store path wrong")
+
+	testVhostUserStorePath := "/test/vhost/user/store/path"
+	h = hypervisor{
+		VhostUserStorePath: testVhostUserStorePath,
+	}
+	vhostUserStorePath = h.vhostUserStorePath()
+	assert.Equal(vhostUserStorePath, testVhostUserStorePath, "custom vhost-user store path wrong")
+}
+
 func TestProxyDefaults(t *testing.T) {
 	assert := assert.New(t)
 

virtcontainers/hypervisor.go

@@ -393,6 +393,13 @@ type HypervisorConfig struct {
 	// DisableVhostNet is used to indicate if host supports vhost_net
 	DisableVhostNet bool
 
+	// EnableVhostUserStore is used to indicate if host supports vhost-user-blk/scsi
+	EnableVhostUserStore bool
+
+	// VhostUserStorePath is the directory path where vhost-user devices
+	// related folders, sockets and device nodes should be.
+	VhostUserStorePath string
+
 	// GuestHookPath is the path within the VM that will be used for 'drop-in' hooks
 	GuestHookPath string
 

virtcontainers/persist.go

@@ -252,6 +252,8 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		BootToBeTemplate:        sconfig.HypervisorConfig.BootToBeTemplate,
 		BootFromTemplate:        sconfig.HypervisorConfig.BootFromTemplate,
 		DisableVhostNet:         sconfig.HypervisorConfig.DisableVhostNet,
+		EnableVhostUserStore:    sconfig.HypervisorConfig.EnableVhostUserStore,
+		VhostUserStorePath:      sconfig.HypervisorConfig.VhostUserStorePath,
 		GuestHookPath:           sconfig.HypervisorConfig.GuestHookPath,
 		VMid:                    sconfig.HypervisorConfig.VMid,
 	}
@@ -539,6 +541,8 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		BootToBeTemplate:        hconf.BootToBeTemplate,
 		BootFromTemplate:        hconf.BootFromTemplate,
 		DisableVhostNet:         hconf.DisableVhostNet,
+		EnableVhostUserStore:    hconf.EnableVhostUserStore,
+		VhostUserStorePath:      hconf.VhostUserStorePath,
 		GuestHookPath:           hconf.GuestHookPath,
 		VMid:                    hconf.VMid,
 	}

virtcontainers/persist/api/config.go

@@ -166,6 +166,13 @@ type HypervisorConfig struct {
 	// DisableVhostNet is used to indicate if host supports vhost_net
 	DisableVhostNet bool
 
+	// EnableVhostUserStore is used to indicate if host supports vhost-user-blk/scsi
+	EnableVhostUserStore bool
+
+	// VhostUserStorePath is the directory path where vhost-user devices
+	// related folders, sockets and device nodes should be.
+	VhostUserStorePath string
+
 	// GuestHookPath is the path within the VM that will be used for 'drop-in' hooks
 	GuestHookPath string
 

virtcontainers/pkg/annotations/annotations.go

@@ -84,6 +84,13 @@ const (
 	// DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host.
 	DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net"
 
+	// EnableVhostUserStore is a sandbox annotation to specify if vhost-user-blk/scsi is abailable on the host
+	EnableVhostUserStore = kataAnnotHypervisorPrefix + "enable_vhost_user_store"
+
+	// VhostUserStorePath is a sandbox annotation to specify the directory path where vhost-user devices
+	// related folders, sockets and device nodes should be.
+	VhostUserStorePath = kataAnnotHypervisorPrefix + "vhost_user_store_path"
+
 	// GuestHookPath is a sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks.
 	GuestHookPath = kataAnnotHypervisorPrefix + "guest_hook_path"
 

virtcontainers/qemu.go

@@ -522,6 +522,15 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 		}
 	}
 
+	// Vhost-user-blk/scsi process which can improve performance, like SPDK,
+	// requires shared-on hugepage to work with Qemu.
+	if q.config.EnableVhostUserStore {
+		if !q.config.HugePages {
+			return errors.New("Vhost-user-blk/scsi is enabled without HugePages. This configuration will not work")
+		}
+		knobs.MemShared = true
+	}
+
 	rtc := govmmQemu.RTC{
 		Base:     "utc",
 		DriftFix: "slew",
@@ -678,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
 	return err
 }
 
-func (q *qemu) getMemArgs() (bool, string, string) {
+func (q *qemu) getMemArgs() (bool, string, string, error) {
 	share := false
 	target := ""
 	memoryBack := "memory-backend-ram"
@@ -689,15 +698,24 @@ func (q *qemu) getMemArgs() (bool, string, string) {
 		target = "/dev/hugepages"
 		memoryBack = "memory-backend-file"
 		share = true
-	} else if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" {
-		target = q.qemuConfig.Memory.Path
-		memoryBack = "memory-backend-file"
+	} else {
+		if q.config.EnableVhostUserStore {
+			// Vhost-user-blk/scsi process which can improve performance, like SPDK,
+			// requires shared-on hugepage to work with Qemu.
+			return share, target, "", fmt.Errorf("Vhost-user-blk/scsi requires hugepage memory")
+		}
+
+		if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" {
+			target = q.qemuConfig.Memory.Path
+			memoryBack = "memory-backend-file"
+		}
 	}
+
 	if q.qemuConfig.Knobs.MemShared {
 		share = true
 	}
 
-	return share, target, memoryBack
+	return share, target, memoryBack, nil
 }
 
 func (q *qemu) setupVirtioMem() error {
@@ -708,7 +726,11 @@ func (q *qemu) setupVirtioMem() error {
 	// 1024 is size for nvdimm
 	sizeMB := int(maxMem) - int(q.config.MemorySize)
 
-	share, target, memoryBack := q.getMemArgs()
+	share, target, memoryBack, err := q.getMemArgs()
+	if err != nil {
+		return err
+	}
+
 	err = q.qmpSetup()
 	if err != nil {
 		return err
@@ -1551,7 +1573,11 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
 		memDev.slot = maxSlot + 1
 	}
 
-	share, target, memoryBack := q.getMemArgs()
+	share, target, memoryBack, err := q.getMemArgs()
+	if err != nil {
+		return 0, err
+	}
+
 	err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, memoryBack, "mem"+strconv.Itoa(memDev.slot), target, memDev.sizeMB, share)
 	if err != nil {
 		q.Logger().WithError(err).Error("hotplug memory")

virtcontainers/qemu_test.go

@@ -465,6 +465,33 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false)
 	assert.Equal(q.qemuConfig.Knobs.MemShared, false)
 	assert.Equal(q.qemuConfig.Memory.Path, "")
+
+	// Check setting vhost-user storage with Hugepages
+	sandbox, err = createQemuSandboxConfig()
+	assert.NoError(err)
+
+	q = &qemu{
+		store: sandbox.newStore,
+	}
+	sandbox.config.HypervisorConfig.EnableVhostUserStore = true
+	sandbox.config.HypervisorConfig.HugePages = true
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, false)
+	assert.NoError(err)
+	assert.Equal(q.qemuConfig.Knobs.MemShared, true)
+
+	// Check failure for vhost-user storage
+	sandbox, err = createQemuSandboxConfig()
+	assert.NoError(err)
+
+	q = &qemu{
+		store: sandbox.newStore,
+	}
+	sandbox.config.HypervisorConfig.EnableVhostUserStore = true
+	sandbox.config.HypervisorConfig.HugePages = false
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, false)
+
+	expectErr = errors.New("Vhost-user-blk/scsi is enabled without HugePages. This configuration will not work")
+	assert.Equal(expectErr.Error(), err.Error())
 }
 
 func createQemuSandboxConfig() (*Sandbox, error) {