From 84e1d7f379fe6d0d22fe091490e15c5c2a1be36d Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 9 May 2018 13:34:20 -0500 Subject: [PATCH] virtcontainers/qemu: reduce memory footprint There is a relation between the maximum number of vCPUs and the memory footprint, if QEMU maxcpus option and kernel nr_cpus cmdline argument are big, then memory footprint is big, this issue only occurs if CPU hotplug support is enabled in the kernel, might be because of kernel needs to allocate resources to watch all sockets waiting for a CPU to be connected (ACPI event). For example ``` +---------------+-------------------------+ | | Memory Footprint (KB) | +---------------+-------------------------+ | NR_CPUS=240 | 186501 | +---------------+-------------------------+ | NR_CPUS=8 | 110684 | +---------------+-------------------------+ ``` In order to do not affect CPU hotplug and allow to users to have containers with the same number of physical CPUs, this patch tries to mitigate the big memory footprint by using the actual number of physical CPUs as the maximum number of vCPUs for each container if `default_maxvcpus` is <= 0 in the runtime configuration file, otherwise `default_maxvcpus` is used as the maximum number of vCPUs. Before this patch a container with 256MB of RAM ``` total used free shared buff/cache available Mem: 195M 40M 113M 26M 41M 112M Swap: 0B 0B 0B ``` With this patch ``` total used free shared buff/cache available Mem: 236M 11M 188M 26M 36M 186M Swap: 0B 0B 0B ``` fixes #295 Signed-off-by: Julio Montes --- Makefile | 5 +++++ cli/config.go | 22 ++++++++++++++++++++++ cli/config/configuration.toml.in | 15 +++++++++++++++ cli/config_test.go | 19 +++++++++++++++++-- virtcontainers/hypervisor.go | 2 +- virtcontainers/qemu.go | 5 ++++- virtcontainers/qemu_amd64.go | 4 ++-- virtcontainers/qemu_arch_base.go | 6 +++--- virtcontainers/qemu_arch_base_test.go | 2 +- virtcontainers/qemu_arm64.go | 4 ++-- virtcontainers/qemu_test.go | 7 ++++--- 11 files changed, 76 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index a9055ec236..a27ec3127c 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,8 @@ PROXYPATH := $(PKGLIBEXECDIR)/$(PROXYCMD) # Default number of vCPUs DEFVCPUS := 1 +# Default maximum number of vCPUs +DEFMAXVCPUS := 0 # Default memory size in MiB DEFMEMSZ := 2048 #Default number of bridges @@ -169,6 +171,7 @@ USER_VARS += SHAREDIR USER_VARS += SHIMPATH USER_VARS += SYSCONFDIR USER_VARS += DEFVCPUS +USER_VARS += DEFMAXVCPUS USER_VARS += DEFMEMSZ USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL @@ -262,6 +265,7 @@ const defaultMachineType = "$(MACHINETYPE)" const defaultRootDirectory = "$(PKGRUNDIR)" const defaultVCPUCount uint32 = $(DEFVCPUS) +const defaultMaxVCPUCount uint32 = $(DEFMAXVCPUS) const defaultMemSize uint32 = $(DEFMEMSZ) // MiB const defaultBridgesCount uint32 = $(DEFBRIDGES) const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)" @@ -347,6 +351,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ -e "s|@DEFVCPUS@|$(DEFVCPUS)|g" \ + -e "s|@DEFMAXVCPUS@|$(DEFMAXVCPUS)|g" \ -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ -e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \ diff --git a/cli/config.go b/cli/config.go index 9950d0ae40..2e9fb88a15 100644 --- a/cli/config.go +++ b/cli/config.go @@ -74,6 +74,7 @@ type hypervisor struct { KernelParams string `toml:"kernel_params"` MachineType string `toml:"machine_type"` DefaultVCPUs int32 `toml:"default_vcpus"` + DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` DefaultMemSz uint32 `toml:"default_memory"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` @@ -202,6 +203,25 @@ func (h hypervisor) defaultVCPUs() uint32 { return uint32(h.DefaultVCPUs) } +func (h hypervisor) defaultMaxVCPUs() uint32 { + numcpus := goruntime.NumCPU() + maxvcpus := vc.MaxQemuVCPUs() + + // Don't exceed the maximum number of vCPUs supported by hypervisor + if h.DefaultMaxVCPUs >= maxvcpus { + return maxvcpus + } + + if h.DefaultMaxVCPUs == 0 || h.DefaultMaxVCPUs > uint32(numcpus) { + if numcpus > int(maxvcpus) { + return maxvcpus + } + return uint32(numcpus) + } + + return h.DefaultMaxVCPUs +} + func (h hypervisor) defaultMemSz() uint32 { if h.DefaultMemSz < 8 { return defaultMemSize // MiB @@ -313,6 +333,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, DefaultVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), DefaultMemSz: h.defaultMemSz(), DefaultBridges: h.defaultBridges(), DisableBlockDeviceUse: h.DisableBlockDeviceUse, @@ -418,6 +439,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat MachineAccelerators: defaultMachineAccelerators, HypervisorMachineType: defaultMachineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: defaultMaxVCPUCount, DefaultMemSz: defaultMemSize, DefaultBridges: defaultBridgesCount, MemPrealloc: defaultEnableMemPrealloc, diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index 4b310aa208..b6e91a31ac 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -45,6 +45,21 @@ machine_accelerators="@MACHINEACCELERATORS@" # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = 1 +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ # Bridges can be used to hot plug devices. # Limitations: diff --git a/cli/config_test.go b/cli/config_test.go index fc24137a42..5dbdf7524d 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -44,6 +44,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath image = "` + imagePath + `" machine_type = "` + machineType + `" default_vcpus = ` + strconv.FormatUint(uint64(defaultVCPUCount), 10) + ` + default_maxvcpus = ` + strconv.FormatUint(uint64(defaultMaxVCPUCount), 10) + ` default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + ` disable_block_device_use = ` + strconv.FormatBool(disableBlock) + ` enable_iothreads = ` + strconv.FormatBool(enableIOThreads) + ` @@ -129,6 +130,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: uint32(goruntime.NumCPU()), DefaultMemSz: defaultMemSize, DisableBlockDeviceUse: disableBlockDevice, BlockDeviceDriver: defaultBlockDeviceDriver, @@ -513,6 +515,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { InitrdPath: defaultInitrdPath, HypervisorMachineType: defaultMachineType, DefaultVCPUs: defaultVCPUCount, + DefaultMaxVCPUs: defaultMaxVCPUCount, DefaultMemSz: defaultMemSize, DisableBlockDeviceUse: defaultDisableBlockDeviceUse, DefaultBridges: defaultBridgesCount, @@ -658,10 +661,13 @@ func TestNewShimConfig(t *testing.T) { func TestHypervisorDefaults(t *testing.T) { assert := assert.New(t) + numCPUs := goruntime.NumCPU() + h := hypervisor{} assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong") assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong") + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong") machineType := "foo" @@ -670,15 +676,24 @@ func TestHypervisorDefaults(t *testing.T) { // auto inferring h.DefaultVCPUs = -1 - assert.Equal(h.defaultVCPUs(), uint32(goruntime.NumCPU()), "default vCPU number is wrong") + assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") h.DefaultVCPUs = 2 assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong") - numCPUs := goruntime.NumCPU() h.DefaultVCPUs = int32(numCPUs) + 1 assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") + h.DefaultMaxVCPUs = 2 + assert.Equal(h.defaultMaxVCPUs(), uint32(h.DefaultMaxVCPUs), "default max vCPU number is wrong") + + h.DefaultMaxVCPUs = uint32(numCPUs) + 1 + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + + maxvcpus := vc.MaxQemuVCPUs() + h.DefaultMaxVCPUs = uint32(maxvcpus) + 1 + assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + h.DefaultMemSz = 1024 assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong") } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 2e0907ca88..0d6631192f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -41,7 +41,7 @@ const ( ) // In some architectures the maximum number of vCPUs depends on the number of physical cores. -var defaultMaxQemuVCPUs = maxQemuVCPUs() +var defaultMaxQemuVCPUs = MaxQemuVCPUs() // deviceType describes a virtualized device type. type deviceType int diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 95ae5c8260..b233f0925b 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -122,6 +122,9 @@ func (q *qemu) kernelParameters() string { // use default parameters params = append(params, defaultKernelParameters...) + // set the maximum number of vCPUs + params = append(params, Param{"nr_cpus", fmt.Sprintf("%d", q.config.DefaultMaxVCPUs)}) + // add the params specified by the provided config. As the kernel // honours the last parameter value set and since the config-provided // params are added here, they will take priority over the defaults. @@ -197,7 +200,7 @@ func (q *qemu) init(sandbox *Sandbox) error { } func (q *qemu) cpuTopology() govmmQemu.SMP { - return q.arch.cpuTopology(q.config.DefaultVCPUs) + return q.arch.cpuTopology(q.config.DefaultVCPUs, q.config.DefaultMaxVCPUs) } func (q *qemu) memoryTopology(sandboxConfig SandboxConfig) (govmmQemu.Memory, error) { diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 2e3406a921..6099ade1e7 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -71,8 +71,8 @@ var supportedQemuMachines = []govmmQemu.Machine{ }, } -// returns the maximum number of vCPUs supported -func maxQemuVCPUs() uint32 { +// MaxQemuVCPUs returns the maximum number of vCPUs supported +func MaxQemuVCPUs() uint32 { return uint32(240) } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 28a06d2a00..795c63cc5f 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -42,7 +42,7 @@ type qemuArch interface { bridges(number uint32) []Bridge // cpuTopology returns the CPU topology for the given amount of vcpus - cpuTopology(vcpus uint32) govmmQemu.SMP + cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP // cpuModel returns the CPU model for the machine type cpuModel() string @@ -219,13 +219,13 @@ func (q *qemuArchBase) bridges(number uint32) []Bridge { return bridges } -func (q *qemuArchBase) cpuTopology(vcpus uint32) govmmQemu.SMP { +func (q *qemuArchBase) cpuTopology(vcpus, maxvcpus uint32) govmmQemu.SMP { smp := govmmQemu.SMP{ CPUs: vcpus, Sockets: vcpus, Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: maxvcpus, } return smp diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 24100d1e84..906d354cc6 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -169,7 +169,7 @@ func TestQemuArchBaseCPUTopology(t *testing.T) { MaxCPUs: defaultMaxQemuVCPUs, } - smp := qemuArchBase.cpuTopology(vcpus) + smp := qemuArchBase.cpuTopology(vcpus, defaultMaxQemuVCPUs) assert.Equal(expectedSMP, smp) } diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go index 2b80440b5f..42ae767189 100644 --- a/virtcontainers/qemu_arm64.go +++ b/virtcontainers/qemu_arm64.go @@ -42,8 +42,8 @@ var supportedQemuMachines = []govmmQemu.Machine{ }, } -// returns the maximum number of vCPUs supported -func maxQemuVCPUs() uint32 { +// MaxQemuVCPUs returns the maximum number of vCPUs supported +func MaxQemuVCPUs() uint32 { return uint32(runtime.NumCPU()) } diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 14751db5e1..4470e8b3ac 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -52,7 +52,7 @@ func testQemuKernelParameters(t *testing.T, kernelParams []Param, expected strin } func TestQemuKernelParameters(t *testing.T) { - expectedOut := "panic=1 initcall_debug foo=foo bar=bar" + expectedOut := fmt.Sprintf("panic=1 initcall_debug nr_cpus=%d foo=foo bar=bar", MaxQemuVCPUs()) params := []Param{ { Key: "foo", @@ -128,7 +128,8 @@ func TestQemuCPUTopology(t *testing.T) { q := &qemu{ arch: &qemuArchBase{}, config: HypervisorConfig{ - DefaultVCPUs: uint32(vcpus), + DefaultVCPUs: uint32(vcpus), + DefaultMaxVCPUs: uint32(vcpus), }, } @@ -137,7 +138,7 @@ func TestQemuCPUTopology(t *testing.T) { Sockets: uint32(vcpus), Cores: defaultCores, Threads: defaultThreads, - MaxCPUs: defaultMaxQemuVCPUs, + MaxCPUs: uint32(vcpus), } smp := q.cpuTopology()