virtcontainers/qemu: reduce memory footprint

There is a relation between the maximum number of vCPUs and the
memory footprint, if QEMU maxcpus option and kernel nr_cpus
cmdline argument are big, then memory footprint is big, this
issue only occurs if CPU hotplug support is enabled in the kernel,
might be because of kernel needs to allocate resources to watch all
sockets waiting for a CPU to be connected (ACPI event).

For example

```
+---------------+-------------------------+
|               | Memory Footprint (KB)   |
+---------------+-------------------------+
| NR_CPUS=240   | 186501                  |
+---------------+-------------------------+
| NR_CPUS=8     | 110684                  |
+---------------+-------------------------+
```

In order to do not affect CPU hotplug and allow to users to have containers
with the same number of physical CPUs, this patch tries to mitigate the
big memory footprint by using the actual number of physical CPUs as the
maximum number of vCPUs for each container if `default_maxvcpus` is <= 0 in
the runtime configuration file,  otherwise `default_maxvcpus` is used as the
maximum number of vCPUs.

Before this patch a container with 256MB of RAM

```
              total        used        free      shared  buff/cache   available
Mem:           195M         40M        113M         26M         41M        112M
Swap:            0B          0B          0B
```

With this patch

```
              total        used        free      shared  buff/cache   available
Mem:           236M         11M        188M         26M         36M        186M
Swap:            0B          0B          0B
```

fixes kata-containers#295

Signed-off-by: Julio Montes <julio.montes@intel.com>

Makefile

@@ -92,6 +92,8 @@ PROXYPATH := $(PKGLIBEXECDIR)/$(PROXYCMD)
 
 # Default number of vCPUs
 DEFVCPUS := 1
+# Default maximum number of vCPUs
+DEFMAXVCPUS := 0
 # Default memory size in MiB
 DEFMEMSZ := 2048
 #Default number of bridges
@@ -169,6 +171,7 @@ USER_VARS += SHAREDIR
 USER_VARS += SHIMPATH
 USER_VARS += SYSCONFDIR
 USER_VARS += DEFVCPUS
+USER_VARS += DEFMAXVCPUS
 USER_VARS += DEFMEMSZ
 USER_VARS += DEFBRIDGES
 USER_VARS += DEFNETWORKMODEL
@@ -262,6 +265,7 @@ const defaultMachineType = "$(MACHINETYPE)"
 const defaultRootDirectory = "$(PKGRUNDIR)"
 
 const defaultVCPUCount uint32 = $(DEFVCPUS)
+const defaultMaxVCPUCount uint32 = $(DEFMAXVCPUS)
 const defaultMemSize uint32 = $(DEFMEMSZ) // MiB
 const defaultBridgesCount uint32 = $(DEFBRIDGES)
 const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)"
@@ -347,6 +351,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION
 		-e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \
 		-e "s|@SHIMPATH@|$(SHIMPATH)|g" \
 		-e "s|@DEFVCPUS@|$(DEFVCPUS)|g" \
+		-e "s|@DEFMAXVCPUS@|$(DEFMAXVCPUS)|g" \
 		-e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \
 		-e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \
 		-e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \

cli/config.go

@@ -74,6 +74,7 @@ type hypervisor struct {
 	KernelParams          string `toml:"kernel_params"`
 	MachineType           string `toml:"machine_type"`
 	DefaultVCPUs          int32  `toml:"default_vcpus"`
+	DefaultMaxVCPUs       uint32 `toml:"default_maxvcpus"`
 	DefaultMemSz          uint32 `toml:"default_memory"`
 	DefaultBridges        uint32 `toml:"default_bridges"`
 	Msize9p               uint32 `toml:"msize_9p"`
@@ -202,6 +203,25 @@ func (h hypervisor) defaultVCPUs() uint32 {
 	return uint32(h.DefaultVCPUs)
 }
 
+func (h hypervisor) defaultMaxVCPUs() uint32 {
+	numcpus := goruntime.NumCPU()
+	maxvcpus := vc.MaxQemuVCPUs()
+
+	// Don't exceed the maximum number of vCPUs supported by hypervisor
+	if h.DefaultMaxVCPUs >= maxvcpus {
+		return maxvcpus
+	}
+
+	if h.DefaultMaxVCPUs == 0 || h.DefaultMaxVCPUs > uint32(numcpus) {
+		if numcpus > int(maxvcpus) {
+			return maxvcpus
+		}
+		return uint32(numcpus)
+	}
+
+	return h.DefaultMaxVCPUs
+}
+
 func (h hypervisor) defaultMemSz() uint32 {
 	if h.DefaultMemSz < 8 {
 		return defaultMemSize // MiB
@@ -313,6 +333,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType: machineType,
 		DefaultVCPUs:          h.defaultVCPUs(),
+		DefaultMaxVCPUs:       h.defaultMaxVCPUs(),
 		DefaultMemSz:          h.defaultMemSz(),
 		DefaultBridges:        h.defaultBridges(),
 		DisableBlockDeviceUse: h.DisableBlockDeviceUse,
@@ -418,6 +439,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		MachineAccelerators:   defaultMachineAccelerators,
 		HypervisorMachineType: defaultMachineType,
 		DefaultVCPUs:          defaultVCPUCount,
+		DefaultMaxVCPUs:       defaultMaxVCPUCount,
 		DefaultMemSz:          defaultMemSize,
 		DefaultBridges:        defaultBridgesCount,
 		MemPrealloc:           defaultEnableMemPrealloc,

cli/config/configuration.toml.in

@@ -45,6 +45,21 @@ machine_accelerators="@MACHINEACCELERATORS@"
 # > number of physical cores      --> will be set to the actual number of physical cores
 default_vcpus = 1
 
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
 
 # Bridges can be used to hot plug devices.
 # Limitations:

cli/config_test.go

@@ -44,6 +44,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath
 	image = "` + imagePath + `"
 	machine_type = "` + machineType + `"
 	default_vcpus = ` + strconv.FormatUint(uint64(defaultVCPUCount), 10) + `
+	default_maxvcpus = ` + strconv.FormatUint(uint64(defaultMaxVCPUCount), 10) + `
 	default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + `
 	disable_block_device_use =  ` + strconv.FormatBool(disableBlock) + `
 	enable_iothreads =  ` + strconv.FormatBool(enableIOThreads) + `
@@ -129,6 +130,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		KernelParams:          vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType: machineType,
 		DefaultVCPUs:          defaultVCPUCount,
+		DefaultMaxVCPUs:       uint32(goruntime.NumCPU()),
 		DefaultMemSz:          defaultMemSize,
 		DisableBlockDeviceUse: disableBlockDevice,
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
@@ -513,6 +515,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		InitrdPath:            defaultInitrdPath,
 		HypervisorMachineType: defaultMachineType,
 		DefaultVCPUs:          defaultVCPUCount,
+		DefaultMaxVCPUs:       defaultMaxVCPUCount,
 		DefaultMemSz:          defaultMemSize,
 		DisableBlockDeviceUse: defaultDisableBlockDeviceUse,
 		DefaultBridges:        defaultBridgesCount,
@@ -658,10 +661,13 @@ func TestNewShimConfig(t *testing.T) {
 func TestHypervisorDefaults(t *testing.T) {
 	assert := assert.New(t)
 
+	numCPUs := goruntime.NumCPU()
+
 	h := hypervisor{}
 
 	assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong")
 	assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong")
+	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
 	assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong")
 
 	machineType := "foo"
@@ -670,15 +676,24 @@ func TestHypervisorDefaults(t *testing.T) {
 
 	// auto inferring
 	h.DefaultVCPUs = -1
-	assert.Equal(h.defaultVCPUs(), uint32(goruntime.NumCPU()), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
 
 	h.DefaultVCPUs = 2
 	assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong")
 
-	numCPUs := goruntime.NumCPU()
 	h.DefaultVCPUs = int32(numCPUs) + 1
 	assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong")
 
+	h.DefaultMaxVCPUs = 2
+	assert.Equal(h.defaultMaxVCPUs(), uint32(h.DefaultMaxVCPUs), "default max vCPU number is wrong")
+
+	h.DefaultMaxVCPUs = uint32(numCPUs) + 1
+	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+
+	maxvcpus := vc.MaxQemuVCPUs()
+	h.DefaultMaxVCPUs = uint32(maxvcpus) + 1
+	assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong")
+
 	h.DefaultMemSz = 1024
 	assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong")
 }

virtcontainers/hypervisor.go

@@ -41,7 +41,7 @@ const (
 )
 
 // In some architectures the maximum number of vCPUs depends on the number of physical cores.
-var defaultMaxQemuVCPUs = maxQemuVCPUs()
+var defaultMaxQemuVCPUs = MaxQemuVCPUs()
 
 // deviceType describes a virtualized device type.
 type deviceType int

virtcontainers/qemu.go

@@ -122,6 +122,9 @@ func (q *qemu) kernelParameters() string {
 	// use default parameters
 	params = append(params, defaultKernelParameters...)
 
+	// set the maximum number of vCPUs
+	params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)})
+
 	// add the params specified by the provided config. As the kernel
 	// honours the last parameter value set and since the config-provided
 	// params are added here, they will take priority over the defaults.
@@ -197,7 +200,7 @@ func (q *qemu) init(sandbox *Sandbox) error {
 }
 
 func (q *qemu) cpuTopology() govmmQemu.SMP {
-	return q.arch.cpuTopology(q.config.DefaultVCPUs)
+	return q.arch.cpuTopology(q.config.DefaultVCPUs, q.config.DefaultMaxVCPUs)
 }
 
 func (q *qemu) memoryTopology(sandboxConfig SandboxConfig) (govmmQemu.Memory, error) {

virtcontainers/qemu_amd64.go

@@ -71,8 +71,8 @@ var supportedQemuMachines = []govmmQemu.Machine{
 	},
 }
 
-// returns the maximum number of vCPUs supported
-func maxQemuVCPUs() uint32 {
+// MaxQemuVCPUs returns the maximum number of vCPUs supported
+func MaxQemuVCPUs() uint32 {
 	return uint32(240)
 }
 

virtcontainers/qemu_arch_base.go

@@ -42,7 +42,7 @@ type qemuArch interface {
 	bridges(number uint32) []Bridge
 
 	// cpuTopology returns the CPU topology for the given amount of vcpus
-	cpuTopology(vcpus uint32) govmmQemu.SMP
+	cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP
 
 	// cpuModel returns the CPU model for the machine type
 	cpuModel() string
@@ -219,13 +219,13 @@ func (q *qemuArchBase) bridges(number uint32) []Bridge {
 	return bridges
 }
 
-func (q *qemuArchBase) cpuTopology(vcpus uint32) govmmQemu.SMP {
+func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP {
 	smp := govmmQemu.SMP{
 		CPUs:    vcpus,
 		Sockets: vcpus,
 		Cores:   defaultCores,
 		Threads: defaultThreads,
-		MaxCPUs: defaultMaxQemuVCPUs,
+		MaxCPUs: maxvcpus,
 	}
 
 	return smp

virtcontainers/qemu_arch_base_test.go

@@ -169,7 +169,7 @@ func TestQemuArchBaseCPUTopology(t *testing.T) {
 		MaxCPUs: defaultMaxQemuVCPUs,
 	}
 
-	smp := qemuArchBase.cpuTopology(vcpus)
+	smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs)
 	assert.Equal(expectedSMP, smp)
 }
 

virtcontainers/qemu_arm64.go

@@ -42,8 +42,8 @@ var supportedQemuMachines = []govmmQemu.Machine{
 	},
 }
 
-// returns the maximum number of vCPUs supported
-func maxQemuVCPUs() uint32 {
+// MaxQemuVCPUs returns the maximum number of vCPUs supported
+func MaxQemuVCPUs() uint32 {
 	return uint32(runtime.NumCPU())
 }
 

virtcontainers/qemu_test.go

@@ -52,7 +52,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin
 }
 
 func TestQemuKernelParameters(t *testing.T) {
-	expectedOut := "panic=1 initcall_debug foo=foo bar=bar"
+	expectedOut := fmt.Sprintf("panic=1 initcall_debug nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs())
 	params := []Param{
 		{
 			Key:   "foo",
@@ -128,7 +128,8 @@ func TestQemuCPUTopology(t *testing.T) {
 	q := &qemu{
 		arch: &qemuArchBase{},
 		config: HypervisorConfig{
-			DefaultVCPUs: uint32(vcpus),
+			DefaultVCPUs:    uint32(vcpus),
+			DefaultMaxVCPUs: uint32(vcpus),
 		},
 	}
 
@@ -137,7 +138,7 @@ func TestQemuCPUTopology(t *testing.T) {
 		Sockets: uint32(vcpus),
 		Cores:   defaultCores,
 		Threads: defaultThreads,
-		MaxCPUs: defaultMaxQemuVCPUs,
+		MaxCPUs: uint32(vcpus),
 	}
 
 	smp := q.cpuTopology()