Merge pull request kata-containers#2410 from Jimmy-Xu/fix-q35-hotplug…

…-pcie

Support hotplug PCIe in q35

Gopkg.toml

@@ -48,7 +48,7 @@
 
 [[constraint]]
   name = "github.com/intel/govmm"
-  revision = "ee21903287393441c7bb53a0b0d39b8fa4075221"
+  revision = "cab47093760f36ef5a5da880bb70c4bf128abbbf"
 
 [[constraint]]
   name = "github.com/kata-containers/agent"

Makefile

@@ -186,6 +186,7 @@ DEFENABLEDEBUG := false
 DEFDISABLENESTINGCHECKS := false
 DEFMSIZE9P := 8192
 DEFHOTPLUGVFIOONROOTBUS := false
+DEFPCIEROOTPORT := 0
 
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
@@ -444,6 +445,7 @@ USER_VARS += DEFENABLEDEBUG
 USER_VARS += DEFDISABLENESTINGCHECKS
 USER_VARS += DEFMSIZE9P
 USER_VARS += DEFHOTPLUGVFIOONROOTBUS
+USER_VARS += DEFPCIEROOTPORT
 USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += BUILDFLAGS
@@ -619,7 +621,8 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \
 		-e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \
 		-e "s|@DEFMSIZE9P@|$(DEFMSIZE9P)|g" \
-		-e "s|@DEFHOTPLUGONROOTBUS@|$(DEFHOTPLUGVFIOONROOTBUS)|g" \
+		-e "s|@DEFHOTPLUGVFIOONROOTBUS@|$(DEFHOTPLUGVFIOONROOTBUS)|g" \
+		-e "s|@DEFPCIEROOTPORT@|$(DEFPCIEROOTPORT)|g" \
 		-e "s|@DEFENTROPYSOURCE@|$(DEFENTROPYSOURCE)|g" \
 		-e "s|@DEFSANDBOXCGROUPONLY@|$(DEFSANDBOXCGROUPONLY)|g" \
 		$< > $@

cli/config/configuration-qemu.toml.in

@@ -224,6 +224,13 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #hotplug_vfio_on_root_bus = true
 
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
 # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
 # security (vhost-net runs ring0) for network I/O performance. 
 #disable_vhost_net = true

cli/kata-env.go

@@ -83,16 +83,18 @@ type RuntimeVersionInfo struct {
 
 // HypervisorInfo stores hypervisor details
 type HypervisorInfo struct {
-	MachineType       string
-	Version           string
-	Path              string
-	BlockDeviceDriver string
-	EntropySource     string
-	Msize9p           uint32
-	MemorySlots       uint32
-	Debug             bool
-	UseVSock          bool
-	SharedFS          string
+	MachineType          string
+	Version              string
+	Path                 string
+	BlockDeviceDriver    string
+	EntropySource        string
+	SharedFS             string
+	Msize9p              uint32
+	MemorySlots          uint32
+	PCIeRootPort         uint32
+	HotplugVFIOOnRootBus bool
+	Debug                bool
+	UseVSock             bool
 }
 
 // ProxyInfo stores proxy details
@@ -355,6 +357,9 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 		MemorySlots:       config.HypervisorConfig.MemSlots,
 		EntropySource:     config.HypervisorConfig.EntropySource,
 		SharedFS:          config.HypervisorConfig.SharedFS,
+
+		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
+		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 	}
 }
 

cli/kata-env_test.go

@@ -91,6 +91,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 
@@ -150,6 +151,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,
 		DefaultMaxVCPUCount:  hypConfig.DefaultMaxVCPUs,
@@ -329,6 +331,9 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		Debug:             config.HypervisorConfig.Debug,
 		EntropySource:     config.HypervisorConfig.EntropySource,
 		SharedFS:          config.HypervisorConfig.SharedFS,
+
+		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
+		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 	}
 }
 

containerd-shim-v2/create_test.go

@@ -398,6 +398,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	blockDeviceDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 
@@ -416,6 +417,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 		BlockDeviceDriver:    blockDeviceDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		SharedFS:             sharedFS,
 	}

pkg/katatestutils/utils.go

@@ -28,6 +28,7 @@ type RuntimeConfigOptions struct {
 	AgentTraceMode       string
 	AgentTraceType       string
 	SharedFS             string
+	PCIeRootPort         uint32
 	DisableBlock         bool
 	EnableIOThreads      bool
 	HotplugVFIOOnRootBus bool
@@ -59,6 +60,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	disable_block_device_use =  ` + strconv.FormatBool(config.DisableBlock) + `
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
 	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
+	pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"

pkg/katautils/config-settings.go

@@ -44,6 +44,7 @@ const defaultEnableDebug bool = false
 const defaultDisableNestingChecks bool = false
 const defaultMsize9p uint32 = 8192
 const defaultHotplugVFIOOnRootBus bool = false
+const defaultPCIeRootPort = 0
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "none"

pkg/katautils/config.go

@@ -111,6 +111,7 @@ type hypervisor struct {
 	MemOffset               uint32   `toml:"memory_offset"`
 	DefaultBridges          uint32   `toml:"default_bridges"`
 	Msize9p                 uint32   `toml:"msize_9p"`
+	PCIeRootPort            uint32   `toml:"pcie_root_port"`
 	DisableBlockDeviceUse   bool     `toml:"disable_block_device_use"`
 	MemPrealloc             bool     `toml:"enable_mem_prealloc"`
 	HugePages               bool     `toml:"enable_hugepages"`
@@ -648,6 +649,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		UseVSock:                useVSock,
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
+		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
 		GuestHookPath:           h.guestHookPath(),
 	}, nil
@@ -796,6 +798,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:         h.EnableIOThreads,
 		Msize9p:                 h.msize9p(),
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
+		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         true,
 		UseVSock:                true,
 	}, nil
@@ -1073,6 +1076,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		EnableIOThreads:         defaultEnableIOThreads,
 		Msize9p:                 defaultMsize9p,
 		HotplugVFIOOnRootBus:    defaultHotplugVFIOOnRootBus,
+		PCIeRootPort:            defaultPCIeRootPort,
 		GuestHookPath:           defaultGuestHookPath,
 		VirtioFSCache:           defaultVirtioFSCacheMode,
 		DisableImageNvdimm:      defaultDisableImageNvdimm,

pkg/katautils/config_test.go

@@ -82,6 +82,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	blockDeviceDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 
@@ -101,6 +102,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		BlockDeviceDriver:    blockDeviceDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     defaultVCPUCount,
 		DefaultMaxVCPUCount:  defaultMaxVCPUCount,
@@ -158,6 +160,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		Mlock:                 !defaultEnableSwap,
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
+		PCIeRootPort:          pcieRootPort,
 		Msize9p:               defaultMsize9p,
 		MemSlots:              defaultMemSlots,
 		EntropySource:         defaultEntropySource,
@@ -775,6 +778,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	disableBlock := true
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	pcieRootPort := uint32(2)
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	defer func() {
 		utils.VHostVSockDevicePath = orgVHostVSockDevicePath
@@ -789,6 +793,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
+		PCIeRootPort:          pcieRootPort,
 		UseVSock:              true,
 	}
 
@@ -846,6 +851,10 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	if config.HotplugVFIOOnRootBus != hotplugVFIOOnRootBus {
 		t.Errorf("Expected value for HotplugVFIOOnRootBus %v, got %v", hotplugVFIOOnRootBus, config.HotplugVFIOOnRootBus)
 	}
+
+	if config.PCIeRootPort != pcieRootPort {
+		t.Errorf("Expected value for PCIeRootPort %v, got %v", pcieRootPort, config.PCIeRootPort)
+	}
 }
 
 func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
@@ -869,6 +878,7 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 	disableBlock := true
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	pcieRootPort := uint32(2)
 
 	hypervisor := hypervisor{
 		Path:                  hypervisorPath,
@@ -879,6 +889,7 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
+		PCIeRootPort:          pcieRootPort,
 	}
 
 	_, err = newQemuHypervisorConfig(hypervisor)