From f0391fbe2af0ec606afee244b195079ec434c570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=81=A5=E4=BF=9E?= Date: Thu, 11 Jul 2024 16:42:12 +0800 Subject: [PATCH 1/3] feat(qrm): to refresh state RequestQuantity --- .../qrm-plugins/cpu/dynamicpolicy/policy.go | 27 ++++++++++--------- .../dynamicpolicy/policy_advisor_handler.go | 4 +-- .../policy_allocation_handlers.go | 4 +-- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go index 2ad5dd29c..9408707bd 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go @@ -1094,22 +1094,25 @@ func (p *DynamicPolicy) getContainerRequestedCores(allocationInfo *state.Allocat return 0 } - if allocationInfo.RequestQuantity == 0 { - if p.metaServer == nil { - general.Errorf("got nil metaServer") - return 0 - } + if p.metaServer == nil { + general.Errorf("got nil metaServer") + return allocationInfo.RequestQuantity + } - container, err := p.metaServer.GetContainerSpec(allocationInfo.PodUid, allocationInfo.ContainerName) - if err != nil || container == nil { - general.Errorf("get container failed with error: %v", err) - return 0 - } + container, err := p.metaServer.GetContainerSpec(allocationInfo.PodUid, allocationInfo.ContainerName) + if err != nil || container == nil { + general.Errorf("get container failed with error: %v", err) + return allocationInfo.RequestQuantity + } + + cpuQuantity := native.CPUQuantityGetter()(container.Resources.Requests) + metaValue := general.MaxFloat64(float64(cpuQuantity.MilliValue())/1000.0, 0) - cpuQuantity := native.CPUQuantityGetter()(container.Resources.Requests) - allocationInfo.RequestQuantity = general.MaxFloat64(float64(cpuQuantity.MilliValue())/1000.0, 0) + if metaValue != allocationInfo.RequestQuantity { + allocationInfo.RequestQuantity = metaValue general.Infof("get cpu request quantity: %.3f for pod: %s/%s container: %s from podWatcher", allocationInfo.RequestQuantity, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) } + return allocationInfo.RequestQuantity } diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go index b9af80bf8..baef8e7e4 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_advisor_handler.go @@ -574,8 +574,6 @@ func (p *DynamicPolicy) applyBlocks(blockCPUSet advisorapi.BlockCPUSet, resp *ad } if newEntries[podUID][containerName] != nil { - // adapt to old checkpoint without RequestQuantity property - newEntries[podUID][containerName].RequestQuantity = state.GetContainerRequestedCores()(allocationInfo) continue } @@ -583,6 +581,8 @@ func (p *DynamicPolicy) applyBlocks(blockCPUSet advisorapi.BlockCPUSet, resp *ad newEntries[podUID] = make(state.ContainerEntries) } newEntries[podUID][containerName] = allocationInfo.Clone() + // adapt to old checkpoint without RequestQuantity property + newEntries[podUID][containerName].RequestQuantity = state.GetContainerRequestedCores()(allocationInfo) switch allocationInfo.QoSLevel { case consts.PodAnnotationQoSLevelDedicatedCores: diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go index b2c6d30a3..27d07a79a 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go @@ -910,8 +910,6 @@ func (p *DynamicPolicy) applyPoolsAndIsolatedInfo(poolsCPUSet map[string]machine } if newPodEntries[podUID][containerName] != nil { - // adapt to old checkpoint without RequestQuantity property - newPodEntries[podUID][containerName].RequestQuantity = state.GetContainerRequestedCores()(allocationInfo) general.Infof("pod: %s/%s, container: %s, qosLevel: %s is isolated, ignore original allocationInfo", allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, allocationInfo.QoSLevel) continue @@ -922,6 +920,8 @@ func (p *DynamicPolicy) applyPoolsAndIsolatedInfo(poolsCPUSet map[string]machine } newPodEntries[podUID][containerName] = allocationInfo.Clone() + // adapt to old checkpoint without RequestQuantity property + newPodEntries[podUID][containerName].RequestQuantity = state.GetContainerRequestedCores()(allocationInfo) switch allocationInfo.QoSLevel { case apiconsts.PodAnnotationQoSLevelDedicatedCores: newPodEntries[podUID][containerName].OwnerPoolName = allocationInfo.GetPoolName() From 09706ed5badc2685f16c48fd577c90e05df09545 Mon Sep 17 00:00:00 2001 From: lilianrong Date: Tue, 13 Aug 2024 13:29:56 +0800 Subject: [PATCH 2/3] feat(qrm): to refresh specified qos cpu/memory state from pod spec feat(qrm): remove unused line --- .../qrm-plugins/cpu/dynamicpolicy/policy.go | 28 +- pkg/agent/qrm-plugins/memory/consts/consts.go | 1 + .../memory/dynamicpolicy/policy.go | 118 ++++- .../dynamicpolicy/policy_advisor_handler.go | 1 - .../policy_allocation_handlers.go | 41 +- .../memory/dynamicpolicy/policy_test.go | 486 +++++++++++++++++- 6 files changed, 656 insertions(+), 19 deletions(-) diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go index 9408707bd..b04cba98d 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go @@ -1108,10 +1108,30 @@ func (p *DynamicPolicy) getContainerRequestedCores(allocationInfo *state.Allocat cpuQuantity := native.CPUQuantityGetter()(container.Resources.Requests) metaValue := general.MaxFloat64(float64(cpuQuantity.MilliValue())/1000.0, 0) - if metaValue != allocationInfo.RequestQuantity { - allocationInfo.RequestQuantity = metaValue - general.Infof("get cpu request quantity: %.3f for pod: %s/%s container: %s from podWatcher", - allocationInfo.RequestQuantity, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + if state.CheckShared(allocationInfo) { + // if there is these two annotations in memory state, it is a new pod, + // we don't need to check the pod request from podWatcher + if allocationInfo.Annotations[consts.PodAnnotationAggregatedRequestsKey] != "" || + allocationInfo.Annotations[consts.PodAnnotationVPAResizingKey] != "" { + return allocationInfo.RequestQuantity + } + if state.CheckNUMABinding(allocationInfo) { + if metaValue < allocationInfo.RequestQuantity { + general.Infof("[snb] get cpu request quantity: (%.3f->%.3f) for pod: %s/%s container: %s from podWatcher", + allocationInfo.RequestQuantity, metaValue, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return metaValue + } + } else { + if metaValue != allocationInfo.RequestQuantity { + general.Infof("[share] get cpu request quantity: (%.3f->%.3f) for pod: %s/%s container: %s from podWatcher", + allocationInfo.RequestQuantity, metaValue, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return metaValue + } + } + } else if allocationInfo.RequestQuantity == 0 { + general.Infof("[other] get cpu request quantity: (%.3f->%.3f) for pod: %s/%s container: %s from podWatcher", + allocationInfo.RequestQuantity, metaValue, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return metaValue } return allocationInfo.RequestQuantity diff --git a/pkg/agent/qrm-plugins/memory/consts/consts.go b/pkg/agent/qrm-plugins/memory/consts/consts.go index c2cfba521..0374f0b4e 100644 --- a/pkg/agent/qrm-plugins/memory/consts/consts.go +++ b/pkg/agent/qrm-plugins/memory/consts/consts.go @@ -25,6 +25,7 @@ const ( MemoryPluginDynamicPolicyName = "qrm_memory_plugin_" + MemoryResourcePluginPolicyNameDynamic ClearResidualState = MemoryPluginDynamicPolicyName + "_clear_residual_state" + SyncMemoryStateFromSpec = MemoryPluginDynamicPolicyName + "_sync_memory_state_form_spec" CheckMemSet = MemoryPluginDynamicPolicyName + "_check_mem_set" ApplyExternalCGParams = MemoryPluginDynamicPolicyName + "_apply_external_cg_params" SetExtraControlKnob = MemoryPluginDynamicPolicyName + "_set_extra_control_knob" diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go index 44a599c3e..3793f5b24 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go @@ -18,6 +18,7 @@ package dynamicpolicy import ( "context" + "errors" "fmt" "sync" "time" @@ -44,6 +45,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" "github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler" "github.com/kubewharf/katalyst-core/pkg/config" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" "github.com/kubewharf/katalyst-core/pkg/config/generic" "github.com/kubewharf/katalyst-core/pkg/metaserver" "github.com/kubewharf/katalyst-core/pkg/metrics" @@ -301,6 +303,29 @@ func (p *DynamicPolicy) Start() (err error) { general.Errorf("start %v failed, err: %v", memconsts.ClearResidualState, err) } + // we should remove this healthy check when we support vpa in all clusters. + syncMemoryStatusFromSpec := func(_ *config.Configuration, + _ interface{}, + _ *dynamicconfig.DynamicAgentConfiguration, + _ metrics.MetricEmitter, + _ *metaserver.MetaServer, + ) { + p.Lock() + defer func() { + p.Unlock() + }() + if err := p.adjustAllocationEntries(); err != nil { + general.Warningf("failed to sync memory state from pod spec: %q", err) + } else { + general.Warningf("sync memory state from pod spec successfully") + } + } + err = periodicalhandler.RegisterPeriodicalHandler(memconsts.SyncMemoryStateFromSpec, qrm.QRMMemoryPluginPeriodicalHandlerGroupName, + syncMemoryStatusFromSpec, stateCheckPeriod) + if err != nil { + general.Errorf("start %v failed, err: %v", memconsts.SyncMemoryStateFromSpec, err) + } + err = periodicalhandler.RegisterPeriodicalHandlerWithHealthz(memconsts.CheckMemSet, general.HealthzCheckStateNotReady, qrm.QRMMemoryPluginPeriodicalHandlerGroupName, p.checkMemorySet, memsetCheckPeriod, healthCheckTolerationTimes) if err != nil { @@ -938,29 +963,102 @@ func (p *DynamicPolicy) removeContainer(podUID, containerName string) error { } // getContainerRequestedMemoryBytes parses and returns requested memory bytes for the given container -func (p *DynamicPolicy) getContainerRequestedMemoryBytes(allocationInfo *state.AllocationInfo) int { +func (p *DynamicPolicy) getContainerRequestedMemoryBytes(allocationInfo *state.AllocationInfo) uint64 { if allocationInfo == nil { general.Errorf("got nil allocationInfo") return 0 } if p.metaServer == nil { - general.Errorf("got nil metaServer") - return 0 + general.Errorf("got nil metaServer, return the origin value") + return allocationInfo.AggregatedQuantity } - container, err := p.metaServer.GetContainerSpec(allocationInfo.PodUid, allocationInfo.ContainerName) - if err != nil || container == nil { + // check request for VPA + if allocationInfo.QoSLevel == apiconsts.PodAnnotationQoSLevelSharedCores { + // if there is these two annotations in memory state, it is a new pod, + // we don't need to check the pod request from podWatcher + if allocationInfo.Annotations[apiconsts.PodAnnotationAggregatedRequestsKey] != "" || + allocationInfo.Annotations[apiconsts.PodAnnotationVPAResizingKey] != "" { + return allocationInfo.AggregatedQuantity + } + + if allocationInfo.CheckNumaBinding() { + // snb count all memory into main container, sidecar is zero + if allocationInfo.CheckSideCar() { + // sidecar container always is zero + general.Infof("[snb] get memory request quantity: (%d -> 0) for pod: %s/%s container(sidecar): %s", + allocationInfo.AggregatedQuantity, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return 0 + } else { + // main container contains the whole pod request (both sidecars and main container). + // check pod memory aggregated quantity from podWatcher + podAggregatedMemoryRequestBytes, err := p.getPodSpecAggregatedMemoryRequestBytes(allocationInfo.PodUid) + if err != nil { + general.Errorf("[snb] get container failed with error: %v, return the origin value", err) + return allocationInfo.AggregatedQuantity + } + + // only handle scale in case + if podAggregatedMemoryRequestBytes < allocationInfo.AggregatedQuantity { + general.Infof("[snb] get memory request quantity: (%d->%d) for pod: %s/%s container: %s from podWatcher", + allocationInfo.AggregatedQuantity, podAggregatedMemoryRequestBytes, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return podAggregatedMemoryRequestBytes + } + } + } else { + // normal share cores pod + requestBytes, err := p.getContainerSpecMemoryRequestBytes(allocationInfo.PodUid, allocationInfo.ContainerName) + if err != nil { + general.Errorf("[other] get container failed with error: %v, return the origin value", err) + return allocationInfo.AggregatedQuantity + } + if requestBytes != allocationInfo.AggregatedQuantity { + general.Infof("[share] get memory request quantity: (%d->%d) for pod: %s/%s container: %s from podWatcher", + allocationInfo.AggregatedQuantity, requestBytes, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return requestBytes + } + } + } + + general.Infof("get memory request bytes: %d for pod: %s/%s container: %s from podWatcher", + allocationInfo.AggregatedQuantity, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + return allocationInfo.AggregatedQuantity +} + +func (p *DynamicPolicy) getPodSpecAggregatedMemoryRequestBytes(podUID string) (uint64, error) { + pod, err := p.metaServer.GetPod(context.Background(), podUID) + if err != nil { + general.Errorf("get pod failed with error: %v", err) + return 0, err + } else if pod == nil { + general.Errorf("get pod failed with not found") + return 0, errors.New("pod not found") + } + + requestBytes := uint64(0) + for _, container := range pod.Spec.Containers { + containerMemoryQuantity := native.MemoryQuantityGetter()(container.Resources.Requests) + requestBytes += uint64(general.Max(int(containerMemoryQuantity.Value()), 0)) + } + + return requestBytes, nil +} + +func (p *DynamicPolicy) getContainerSpecMemoryRequestBytes(podUID, containerName string) (uint64, error) { + container, err := p.metaServer.GetContainerSpec(podUID, containerName) + if err != nil { general.Errorf("get container failed with error: %v", err) - return 0 + return 0, err + } else if container == nil { + general.Errorf("get container failed with not found") + return 0, errors.New("container not found") } memoryQuantity := native.MemoryQuantityGetter()(container.Resources.Requests) - requestBytes := general.Max(int(memoryQuantity.Value()), 0) + requestBytes := uint64(general.Max(int(memoryQuantity.Value()), 0)) - general.Infof("get memory request bytes: %d for pod: %s/%s container: %s from podWatcher", - requestBytes, allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) - return requestBytes + return requestBytes, nil } // hasLastLevelEnhancementKey check if the pod with the given UID has the corresponding last level enhancement key diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go index 4ad19b52e..cf4c92d62 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go @@ -413,7 +413,6 @@ func handleAdvisorCPUSetMems( allocationInfo.NumaAllocationResult = cpusetMems allocationInfo.TopologyAwareAllocations = nil - allocationInfo.AggregatedQuantity = 0 _ = emitter.StoreInt64(util.MetricNameMemoryHandleAdvisorCPUSetMems, 1, metrics.MetricTypeNameRaw, metrics.ConvertMapToTags(map[string]string{ diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go index 266283fb0..fb17be1da 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go @@ -427,12 +427,51 @@ func (p *DynamicPolicy) adjustAllocationEntries() error { allocationInfo.NumaAllocationResult.String(), numaWithoutNUMABindingPods.String()) } - allocationInfo.AggregatedQuantity = 0 allocationInfo.NumaAllocationResult = numaWithoutNUMABindingPods.Clone() allocationInfo.TopologyAwareAllocations = nil } } + // we should remove it someday + // adjust container request for old VPA pods + for podUID, containerEntries := range podEntries { + for containerName, allocationInfo := range containerEntries { + if allocationInfo == nil { + general.Errorf("pod: %s, container: %s has nil allocationInfo", podUID, containerName) + continue + } else if containerName == "" { + general.Errorf("pod: %s has empty containerName entry", podUID) + continue + } else if allocationInfo.QoSLevel != apiconsts.PodAnnotationQoSLevelSharedCores { + continue + } + if allocationInfo.QoSLevel == apiconsts.PodAnnotationQoSLevelSharedCores { + if allocationInfo.CheckNumaBinding() { + if allocationInfo.CheckSideCar() { + continue + } + + if len(allocationInfo.TopologyAwareAllocations) != 1 { + general.Errorf("pod: %s/%s, container: %s topologyAwareAllocations length is not 1: %v", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, allocationInfo.TopologyAwareAllocations) + continue + } + + // update AggregatedQuantity && TopologyAwareAllocations for snb + allocationInfo.AggregatedQuantity = p.getContainerRequestedMemoryBytes(allocationInfo) + for numaId, quantity := range allocationInfo.TopologyAwareAllocations { + if quantity != allocationInfo.AggregatedQuantity { + allocationInfo.TopologyAwareAllocations[numaId] = allocationInfo.AggregatedQuantity + } + } + } else { + // update AggregatedQuantity for normal share cores + allocationInfo.AggregatedQuantity = p.getContainerRequestedMemoryBytes(allocationInfo) + } + } + } + } + resourcesMachineState, err := state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) if err != nil { return fmt.Errorf("calculate machineState by updated pod entries failed with error: %v", err) diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go index d999cdb45..314f82581 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - /* Copyright 2022 The Katalyst Authors. @@ -3770,3 +3767,486 @@ func BenchmarkGetTopologyHints(b *testing.B) { _ = os.RemoveAll(tmpDir) } } + +func Test_getContainerRequestedMemoryBytes(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-Test_getContainerRequestedMemoryBytes") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + metaServer := makeMetaServer() + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + assert.NoError(t, err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + assert.NoError(t, err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + + dynamicPolicy.metaServer = metaServer + + memorySize1G := resource.MustParse("1Gi") + + // check normal share cores + pod1Allocation := &state.AllocationInfo{ + PodUid: "test-pod-1-uid", + PodName: "test-pod-1", + ContainerName: "test-container-1", + AggregatedQuantity: 512, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + } + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-1-uid", "test-pod-1", pod1Allocation) + // case 1. pod spec not found, return the current allocated memory + as.Equal(uint64(512), dynamicPolicy.getContainerRequestedMemoryBytes(pod1Allocation)) + + podFetcher := &pod.PodFetcherStub{ + PodList: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-1-uid", + Name: "test-pod-1", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }, + }, + } + metaServer.PodFetcher = podFetcher + + // case 2. check normal share cores scale out success + as.Equal(uint64(memorySize1G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod1Allocation)) + + // case 3. check normal share cores scale in + memorySize2G := resource.MustParse("2Gi") + pod1Allocation.AggregatedQuantity = uint64(memorySize2G.Value()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-1-uid", "test-pod-1", pod1Allocation) + as.Equal(uint64(memorySize1G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod1Allocation)) + + // check normal snb + pod2Allocation := &state.AllocationInfo{ + PodUid: "test-pod-2-uid", + PodName: "test-pod-2", + ContainerName: "test-container-2", + AggregatedQuantity: 512, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + } + as.Equalf(true, pod2Allocation.CheckNumaBinding(), "check numa binding failed") + // case 1. check pod spec not found + as.Equal(uint64(512), dynamicPolicy.getContainerRequestedMemoryBytes(pod2Allocation)) + + podFetcher.PodList = append(podFetcher.PodList, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-2-uid", + Name: "test-pod-2", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-2", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }) + + // case 2. snb scale out (expect ignore) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-2-uid", "test-pod-2", pod2Allocation) + as.Equal(uint64(512), dynamicPolicy.getContainerRequestedMemoryBytes(pod2Allocation)) + + // case 3. snb scale in (expect sync from pod spec) + pod2Allocation.AggregatedQuantity = uint64(memorySize2G.Value()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-2-uid", "test-pod-2", pod2Allocation) + as.Equal(uint64(memorySize1G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod2Allocation)) + + // snb with sidecar + podFetcher.PodList = append(podFetcher.PodList, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-3-uid", + Name: "test-pod-3", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-3", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + { + Name: "test-container-4", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }) + + pod3Container3Allocation := &state.AllocationInfo{ + PodUid: "test-pod-3-uid", + PodName: "test-pod-3", + ContainerName: "test-container-3", + AggregatedQuantity: 512, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + ContainerType: pluginapi.ContainerType_MAIN.String(), + } + as.Equal(true, pod3Container3Allocation.CheckMainContainer()) + pod3Container4Allocation := &state.AllocationInfo{ + PodUid: "test-pod-3-uid", + PodName: "test-pod-3", + ContainerName: "test-container-4", + AggregatedQuantity: 512, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + ContainerType: pluginapi.ContainerType_SIDECAR.String(), + } + as.Equalf(true, pod3Container4Allocation.CheckSideCar(), "check sidecar container failed") + + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3", pod3Container3Allocation) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-4", pod3Container4Allocation) + + // case 1. check snb with sidecar scale out (expect ignore) + as.Equal(uint64(512), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container3Allocation)) + as.Equal(uint64(0), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container4Allocation)) + + // case 2. check snb with sidecar scale in (expect not sync from pod spec) + pod3Container3Allocation.AggregatedQuantity = uint64(memorySize2G.Value()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3", pod3Container3Allocation) + as.Equal(uint64(memorySize2G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container3Allocation)) + as.Equal(uint64(0), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container4Allocation)) + + // case 3. check snb with sidecar scale in (expect not sync from pod spec) + memorySize1_5G := resource.MustParse("1.5Gi") + pod3Container3Allocation.AggregatedQuantity = uint64(memorySize1_5G.Value()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3", pod3Container3Allocation) + as.Equal(uint64(memorySize1_5G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container3Allocation)) + as.Equal(uint64(0), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container4Allocation)) + + // case 4. check snb with sidecar scale in (expect sync from pod spec) + memorySize3G := resource.MustParse("3Gi") + pod3Container3Allocation.AggregatedQuantity = uint64(memorySize3G.Value()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3", pod3Container3Allocation) + as.Equal(uint64(memorySize2G.Value()), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container3Allocation)) + as.Equal(uint64(0), dynamicPolicy.getContainerRequestedMemoryBytes(pod3Container4Allocation)) +} + +func Test_adjustAllocationEntries(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-Test_adjustAllocationEntries") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + metaServer := makeMetaServer() + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + assert.NoError(t, err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + assert.NoError(t, err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + + dynamicPolicy.metaServer = metaServer + + memorySize1G := resource.MustParse("1Gi") + podFetcher := &pod.PodFetcherStub{ + PodList: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-1-uid", + Name: "test-pod-1", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-2-uid", + Name: "test-pod-2", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-3-uid", + Name: "test-pod-3", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-3", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + { + Name: "test-container-4", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-4-uid", + Name: "test-pod-4", + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: memorySize1G, + }, + }, + }, + }, + }, + }, + }, + } + metaServer.PodFetcher = podFetcher + + pod1Allocation := &state.AllocationInfo{ + PodUid: "test-pod-1-uid", + PodName: "test-pod-1", + ContainerName: "test-container-1", + AggregatedQuantity: 512, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + NumaAllocationResult: machine.NewCPUSet(3), + } + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-1-uid", "test-container-1", pod1Allocation) + + pod2Allocation := &state.AllocationInfo{ + PodUid: "test-pod-2-uid", + PodName: "test-pod-2", + ContainerName: "test-container-2", + AggregatedQuantity: 2 * uint64(memorySize1G.Value()), + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + "katalyst.kubewharf.io/qos_level": "shared_cores", + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + NumaAllocationResult: machine.NewCPUSet(0), + TopologyAwareAllocations: map[int]uint64{ + 0: uint64(2 * memorySize1G.Value()), + }, + } + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-2-uid", "test-container-2", pod2Allocation) + + pod3Container3Allocation := &state.AllocationInfo{ + PodUid: "test-pod-3-uid", + PodName: "test-pod-3", + ContainerName: "test-container-3", + AggregatedQuantity: 3 * uint64(memorySize1G.Value()), + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + "katalyst.kubewharf.io/qos_level": "shared_cores", + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + ContainerType: pluginapi.ContainerType_MAIN.String(), + NumaAllocationResult: machine.NewCPUSet(1), + TopologyAwareAllocations: map[int]uint64{ + 1: uint64(3 * memorySize1G.Value()), + }, + } + as.Equal(true, pod3Container3Allocation.CheckMainContainer()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3", pod3Container3Allocation) + + pod3Container4Allocation := &state.AllocationInfo{ + PodUid: "test-pod-3-uid", + PodName: "test-pod-3", + ContainerName: "test-container-4", + AggregatedQuantity: 0, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + "katalyst.kubewharf.io/qos_level": "shared_cores", + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + ContainerType: pluginapi.ContainerType_SIDECAR.String(), + NumaAllocationResult: machine.NewCPUSet(1), + } + as.Equalf(true, pod3Container4Allocation.CheckSideCar(), "check sidecar container failed") + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-4", pod3Container4Allocation) + + pod4Container1Allocation := &state.AllocationInfo{ + PodUid: "test-pod-4-uid", + PodName: "test-pod-4", + ContainerName: "test-container-1", + AggregatedQuantity: uint64(memorySize1G.Value() / 2), + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + Annotations: map[string]string{ + consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, + }, + NumaAllocationResult: machine.NewCPUSet(2), + TopologyAwareAllocations: map[int]uint64{ + 2: uint64(memorySize1G.Value() / 2), + }, + ContainerType: pluginapi.ContainerType_MAIN.String(), + } + as.Equal(true, pod3Container3Allocation.CheckMainContainer()) + dynamicPolicy.state.SetAllocationInfo(v1.ResourceMemory, "test-pod-4-uid", "test-container-1", pod4Container1Allocation) + + podResourceEntries := dynamicPolicy.state.GetPodResourceEntries() + machineState, err := state.GenerateMachineStateFromPodEntries(dynamicPolicy.state.GetMachineInfo(), podResourceEntries, dynamicPolicy.state.GetReservedMemory()) + as.NoError(err) + dynamicPolicy.state.SetMachineState(machineState) + + as.NoError(dynamicPolicy.adjustAllocationEntries()) + + allcation1 := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, "test-pod-1-uid", "test-container-1") + as.Equal(uint64(memorySize1G.Value()), allcation1.AggregatedQuantity) + + allocation2 := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, "test-pod-2-uid", "test-container-2") + as.Equal(uint64(memorySize1G.Value()), allocation2.AggregatedQuantity) + as.Equal(map[int]uint64{0: uint64(memorySize1G.Value())}, allocation2.TopologyAwareAllocations) + + allocation3Container3 := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-3") + as.Equal(2*uint64(memorySize1G.Value()), allocation3Container3.AggregatedQuantity) + as.Equal(map[int]uint64{1: uint64(2 * memorySize1G.Value())}, allocation3Container3.TopologyAwareAllocations) + + allocation3Container4 := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, "test-pod-3-uid", "test-container-4") + as.Equal(uint64(0), allocation3Container4.AggregatedQuantity) + as.Equal(map[int]uint64(nil), allocation3Container4.TopologyAwareAllocations) + + allocation4Container1 := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, "test-pod-4-uid", "test-container-1") + as.Equal(pod4Container1Allocation.AggregatedQuantity, allocation4Container1.AggregatedQuantity) + as.Equal(map[int]uint64{2: pod4Container1Allocation.AggregatedQuantity}, allocation4Container1.TopologyAwareAllocations) + + machineStateNew := dynamicPolicy.state.GetMachineState() + as.Equal(uint64(6442450944), machineStateNew[v1.ResourceMemory][0].Free) + as.Equal(uint64(5368709120), machineStateNew[v1.ResourceMemory][1].Free) + as.Equal(uint64(6979321856), machineStateNew[v1.ResourceMemory][2].Free) +} From 49088303b02fff3b0174e0fa26c1d3dc239259c7 Mon Sep 17 00:00:00 2001 From: lilianrong Date: Tue, 13 Aug 2024 13:36:44 +0800 Subject: [PATCH 3/3] feat(qrm): support qrm pod inplace update resizing admit --- .../app/options/qrm/qrm_base.go | 4 +- go.mod | 2 +- go.sum | 4 +- .../qrm-plugins/cpu/dynamicpolicy/policy.go | 140 ++- .../policy_allocation_handlers.go | 280 ++++- .../cpu/dynamicpolicy/policy_hint_handlers.go | 118 +- .../cpu/dynamicpolicy/policy_test.go | 137 ++- .../cpu/dynamicpolicy/state/state.go | 31 + .../cpu/dynamicpolicy/state/state_test.go | 163 +++ .../cpu/dynamicpolicy/state/util.go | 63 + .../qrm-plugins/cpu/dynamicpolicy/vpa_test.go | 1045 +++++++++++++++++ .../memory/dynamicpolicy/policy.go | 84 +- .../policy_allocation_handlers.go | 102 +- .../dynamicpolicy/policy_hint_handlers.go | 147 ++- .../memory/dynamicpolicy/policy_test.go | 151 ++- .../qrm-plugins/memory/dynamicpolicy/util.go | 9 + .../memory/dynamicpolicy/vpa_test.go | 911 ++++++++++++++ pkg/agent/qrm-plugins/util/consts.go | 3 + pkg/agent/qrm-plugins/util/util.go | 30 + pkg/config/agent/qrm/qrm_base.go | 9 +- 20 files changed, 3262 insertions(+), 171 deletions(-) create mode 100644 pkg/agent/qrm-plugins/cpu/dynamicpolicy/vpa_test.go create mode 100644 pkg/agent/qrm-plugins/memory/dynamicpolicy/vpa_test.go diff --git a/cmd/katalyst-agent/app/options/qrm/qrm_base.go b/cmd/katalyst-agent/app/options/qrm/qrm_base.go index 7bb4feaa1..fe080b3e9 100644 --- a/cmd/katalyst-agent/app/options/qrm/qrm_base.go +++ b/cmd/katalyst-agent/app/options/qrm/qrm_base.go @@ -65,8 +65,8 @@ func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfig conf.ExtraStateFileAbsPath = o.ExtraStateFileAbsPath conf.PodDebugAnnoKeys = o.PodDebugAnnoKeys conf.UseKubeletReservedConfig = o.UseKubeletReservedConfig - conf.PodAnnotationKeptKeys = o.PodAnnotationKeptKeys - conf.PodLabelKeptKeys = o.PodLabelKeptKeys + conf.PodAnnotationKeptKeys = append(conf.PodAnnotationKeptKeys, o.PodAnnotationKeptKeys...) + conf.PodLabelKeptKeys = append(conf.PodLabelKeptKeys, o.PodLabelKeptKeys...) return nil } diff --git a/go.mod b/go.mod index 3d607a621..b2370a3f3 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 github.com/klauspost/cpuid/v2 v2.2.6 - github.com/kubewharf/katalyst-api v0.5.1-0.20240702044746-be552fd7ea7d + github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078 github.com/montanaflynn/stats v0.7.1 github.com/opencontainers/runc v1.1.6 github.com/opencontainers/selinux v1.10.0 diff --git a/go.sum b/go.sum index 637b0ff7a..8e788129c 100644 --- a/go.sum +++ b/go.sum @@ -568,8 +568,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubewharf/katalyst-api v0.5.1-0.20240702044746-be552fd7ea7d h1:6CuK3axf2B63zIkEu5XyxbaC+JArE/3Jo3QHvb+Hn0M= -github.com/kubewharf/katalyst-api v0.5.1-0.20240702044746-be552fd7ea7d/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k= +github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078 h1:CSBXQOe0AzlWcGaww8uqOUDu+/4bL3hVNBz86oziOis= +github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k= github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc= github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c= github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8= diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go index b04cba98d..2948f2280 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go @@ -19,6 +19,7 @@ package dynamicpolicy import ( "context" "fmt" + "math" "sync" "time" @@ -436,24 +437,31 @@ func (p *DynamicPolicy) GetResourcesAllocation(_ context.Context, return nil, fmt.Errorf("GetNumaAwareAssignments err: %v", err) } - podResources := make(map[string]*pluginapi.ContainerResources) var allocationInfosJustFinishRampUp []*state.AllocationInfo + needUpdateMachineState := false for podUID, containerEntries := range podEntries { // if it's a pool, not returning to QRM if containerEntries.IsPoolEntry() { continue } - if podResources[podUID] == nil { - podResources[podUID] = &pluginapi.ContainerResources{} - } - + mainContainerAllocationInfo := podEntries[podUID].GetMainContainerEntry() for containerName, allocationInfo := range containerEntries { if allocationInfo == nil { continue } allocationInfo = allocationInfo.Clone() + // sync allocation info from main container to sidecar + if allocationInfo.CheckSideCar() && mainContainerAllocationInfo != nil { + if p.applySidecarAllocationInfoFromMainContainer(allocationInfo, mainContainerAllocationInfo) { + general.Infof("pod: %s/%s, container: %s sync allocation info from main container", + allocationInfo.PodNamespace, allocationInfo.PodName, containerName) + p.state.SetAllocationInfo(podUID, containerName, allocationInfo) + needUpdateMachineState = true + } + } + initTs, tsErr := time.Parse(util.QRMTimeFormat, allocationInfo.InitTimestamp) if tsErr != nil { if state.CheckShared(allocationInfo) && !state.CheckNUMABinding(allocationInfo) { @@ -484,6 +492,39 @@ func (p *DynamicPolicy) GetResourcesAllocation(_ context.Context, } } + } + } + + if len(allocationInfosJustFinishRampUp) > 0 { + if err = p.putAllocationsAndAdjustAllocationEntries(allocationInfosJustFinishRampUp, true); err != nil { + // not influencing return response to kubelet when putAllocationsAndAdjustAllocationEntries failed + general.Errorf("putAllocationsAndAdjustAllocationEntries failed with error: %v", err) + } + } else if needUpdateMachineState { + // NOTE: we only need update machine state when putAllocationsAndAdjustAllocationEntries is skipped, + // because putAllocationsAndAdjustAllocationEntries will update machine state. + general.Infof("GetResourcesAllocation update machine state") + podEntries = p.state.GetPodEntries() + updatedMachineState, err := generateMachineStateFromPodEntries(p.machineInfo.CPUTopology, podEntries) + if err != nil { + general.Errorf("GetResourcesAllocation GenerateMachineStateFromPodEntries failed with error: %v", err) + return nil, fmt.Errorf("GenerateMachineStateFromPodEntries failed with error: %v", err) + } + p.state.SetMachineState(updatedMachineState) + } + + podEntries = p.state.GetPodEntries() + podResources := make(map[string]*pluginapi.ContainerResources) + for podUID, containerEntries := range podEntries { + if containerEntries.IsPoolEntry() { + continue + } + + if podResources[podUID] == nil { + podResources[podUID] = &pluginapi.ContainerResources{} + } + + for containerName, allocationInfo := range containerEntries { if podResources[podUID].ContainerResources == nil { podResources[podUID].ContainerResources = make(map[string]*pluginapi.ResourceAllocation) } @@ -501,13 +542,6 @@ func (p *DynamicPolicy) GetResourcesAllocation(_ context.Context, } } - if len(allocationInfosJustFinishRampUp) > 0 { - if err = p.putAllocationsAndAdjustAllocationEntries(allocationInfosJustFinishRampUp, true); err != nil { - // not influencing return response to kubelet when putAllocationsAndAdjustAllocationEntries failed - general.Errorf("putAllocationsAndAdjustAllocationEntries failed with error: %v", err) - } - } - return &pluginapi.GetResourcesAllocationResponse{ PodResources: podResources, }, nil @@ -639,7 +673,8 @@ func (p *DynamicPolicy) GetTopologyHints(ctx context.Context, "qosLevel", qosLevel, "numCPUsInt", reqInt, "numCPUsFloat64", reqFloat64, - "isDebugPod", isDebugPod) + "isDebugPod", isDebugPod, + "annotation", req.Annotations) if req.ContainerType == pluginapi.ContainerType_INIT || isDebugPod { general.Infof("there is no NUMA preference, return nil hint") @@ -721,7 +756,8 @@ func (p *DynamicPolicy) Allocate(ctx context.Context, "qosLevel", qosLevel, "numCPUsInt", reqInt, "numCPUsFloat64", reqFloat64, - "isDebugPod", isDebugPod) + "isDebugPod", isDebugPod, + "annotations", req.Annotations) if req.ContainerType == pluginapi.ContainerType_INIT { return &pluginapi.ResourceAllocationResponse{ @@ -794,7 +830,7 @@ func (p *DynamicPolicy) Allocate(ctx context.Context, }() allocationInfo := p.state.GetAllocationInfo(req.PodUid, req.ContainerName) - if allocationInfo != nil && allocationInfo.OriginalAllocationResult.Size() >= reqInt { + if allocationInfo != nil && allocationInfo.OriginalAllocationResult.Size() >= reqInt && !util.PodInplaceUpdateResizing(req) { general.InfoS("already allocated and meet requirement", "podNamespace", req.PodNamespace, "podName", req.PodName, @@ -1108,11 +1144,13 @@ func (p *DynamicPolicy) getContainerRequestedCores(allocationInfo *state.Allocat cpuQuantity := native.CPUQuantityGetter()(container.Resources.Requests) metaValue := general.MaxFloat64(float64(cpuQuantity.MilliValue())/1000.0, 0) + // optimize this logic someday: + // only for refresh cpu request for old pod with cpu ceil and old inplace update resized pods. if state.CheckShared(allocationInfo) { // if there is these two annotations in memory state, it is a new pod, // we don't need to check the pod request from podWatcher if allocationInfo.Annotations[consts.PodAnnotationAggregatedRequestsKey] != "" || - allocationInfo.Annotations[consts.PodAnnotationVPAResizingKey] != "" { + allocationInfo.Annotations[consts.PodAnnotationInplaceUpdateResizingKey] != "" { return allocationInfo.RequestQuantity } if state.CheckNUMABinding(allocationInfo) { @@ -1136,3 +1174,73 @@ func (p *DynamicPolicy) getContainerRequestedCores(allocationInfo *state.Allocat return allocationInfo.RequestQuantity } + +func (p *DynamicPolicy) checkNormalShareCoresCpuResource(req *pluginapi.ResourceRequest) (bool, error) { + _, reqFloat64, err := util.GetPodAggregatedRequestResource(req) + if err != nil { + return false, fmt.Errorf("GetQuantityFromResourceReq failed with error: %v", err) + } + + shareCoresAllocated := reqFloat64 + podEntries := p.state.GetPodEntries() + for podUid, podEntry := range podEntries { + if podEntry.IsPoolEntry() { + continue + } + if podUid == req.PodUid { + continue + } + for _, allocation := range podEntry { + // shareCoresAllocated should involve both main and sidecar containers + if state.CheckShared(allocation) && !state.CheckNUMABinding(allocation) { + shareCoresAllocated += p.getContainerRequestedCores(allocation) + } + } + } + + machineState := p.state.GetMachineState() + pooledCPUs := machineState.GetFilteredAvailableCPUSet(p.reservedCPUs, + state.CheckDedicated, state.CheckNUMABinding) + + shareCoresAllocatedInt := int(math.Ceil(shareCoresAllocated)) + general.Infof("[checkNormalShareCoresCpuResource] node cpu allocated: %d, allocatable: %d", shareCoresAllocatedInt, pooledCPUs.Size()) + if shareCoresAllocatedInt > pooledCPUs.Size() { + general.Warningf("[checkNormalShareCoresCpuResource] no enough cpu resource for normal share cores pod: %s/%s, container: %s (request: %.02f, node allocated: %d, node allocatable: %d)", + req.PodNamespace, req.PodName, req.ContainerName, reqFloat64, shareCoresAllocatedInt, pooledCPUs.Size()) + return false, nil + } + + general.InfoS("checkNormalShareCoresCpuResource memory successfully", + "podNamespace", req.PodNamespace, + "podName", req.PodName, + "containerName", req.ContainerName, + "request", reqFloat64) + + return true, nil +} + +func (p *DynamicPolicy) applySidecarAllocationInfoFromMainContainer(sidecarAllocationInfo, mainAllocationInfo *state.AllocationInfo) bool { + changed := false + if sidecarAllocationInfo.OwnerPoolName != mainAllocationInfo.OwnerPoolName || + !sidecarAllocationInfo.AllocationResult.Equals(mainAllocationInfo.AllocationResult) || + !sidecarAllocationInfo.OriginalAllocationResult.Equals(mainAllocationInfo.OriginalAllocationResult) || + !state.CheckAllocationInfoTopologyAwareAssignments(sidecarAllocationInfo, mainAllocationInfo) || + !state.CheckAllocationInfoOriginTopologyAwareAssignments(sidecarAllocationInfo, mainAllocationInfo) { + + sidecarAllocationInfo.OwnerPoolName = mainAllocationInfo.OwnerPoolName + sidecarAllocationInfo.AllocationResult = mainAllocationInfo.AllocationResult.Clone() + sidecarAllocationInfo.OriginalAllocationResult = mainAllocationInfo.OriginalAllocationResult.Clone() + sidecarAllocationInfo.TopologyAwareAssignments = machine.DeepcopyCPUAssignment(mainAllocationInfo.TopologyAwareAssignments) + sidecarAllocationInfo.OriginalTopologyAwareAssignments = machine.DeepcopyCPUAssignment(mainAllocationInfo.OriginalTopologyAwareAssignments) + + changed = true + } + + request := p.getContainerRequestedCores(sidecarAllocationInfo) + if sidecarAllocationInfo.RequestQuantity != request { + sidecarAllocationInfo.RequestQuantity = request + changed = true + } + + return changed +} diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go index 27d07a79a..c9d78deca 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go @@ -85,6 +85,7 @@ func (p *DynamicPolicy) sharedCoresWithoutNUMABindingAllocationHandler(_ context needSet := true allocationInfo := p.state.GetAllocationInfo(req.PodUid, req.ContainerName) + originAllocationInfo := allocationInfo.Clone() err = updateAllocationInfoByReq(req, allocationInfo) if err != nil { general.Errorf("pod: %s/%s, container: %s updateAllocationInfoByReq failed with error: %v", @@ -139,6 +140,12 @@ func (p *DynamicPolicy) sharedCoresWithoutNUMABindingAllocationHandler(_ context } } } else if allocationInfo.RampUp { + if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s is still in ramp up, not allow to inplace update resize", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("pod is still ramp up, not allow to inplace update resize") + } + general.Infof("pod: %s/%s, container: %s is still in ramp up, allocate pooled cpus: %s", req.PodNamespace, req.PodName, req.ContainerName, pooledCPUs.String()) @@ -147,11 +154,27 @@ func (p *DynamicPolicy) sharedCoresWithoutNUMABindingAllocationHandler(_ context allocationInfo.TopologyAwareAssignments = pooledCPUsTopologyAwareAssignments allocationInfo.OriginalTopologyAwareAssignments = machine.DeepcopyCPUAssignment(pooledCPUsTopologyAwareAssignments) } else { - _, err := p.doAndCheckPutAllocationInfo(allocationInfo, true) - if err != nil { - return nil, err - } + if util.PodInplaceUpdateResizing(req) { + general.Infof("pod: %s/%s, container: %s request to inplace update resize (%.02f->%.02f)", + req.PodNamespace, req.PodName, req.ContainerName, allocationInfo.RequestQuantity, reqFloat64) + allocationInfo.RequestQuantity = reqFloat64 + p.state.SetAllocationInfo(allocationInfo.PodUid, allocationInfo.ContainerName, allocationInfo) + _, err := p.doAndCheckPutAllocationInfoPodResizingAware(originAllocationInfo, allocationInfo, false, true) + if err != nil { + general.Errorf("pod: %s/%s, container: %s doAndCheckPutAllocationInfoPodResizingAware failed: %q", + req.PodNamespace, req.PodName, req.ContainerName, err) + p.state.SetAllocationInfo(originAllocationInfo.PodUid, originAllocationInfo.ContainerName, originAllocationInfo) + return nil, err + } + } else { + _, err := p.doAndCheckPutAllocationInfo(allocationInfo, true) + if err != nil { + general.Errorf("pod: %s/%s, container: %s doAndCheckPutAllocationInfo failed: %q", + req.PodNamespace, req.PodName, req.ContainerName, err) + return nil, err + } + } needSet = false } @@ -187,6 +210,10 @@ func (p *DynamicPolicy) reclaimedCoresAllocationHandler(_ context.Context, return nil, fmt.Errorf("reclaimedCoresAllocationHandler got nil request") } + if util.PodInplaceUpdateResizing(req) { + return nil, fmt.Errorf("not support inplace update resize for reclaimed cores") + } + _, reqFloat64, err := util.GetQuantityFromResourceReq(req) if err != nil { return nil, fmt.Errorf("getReqQuantityFromResourceReq failed with error: %v", err) @@ -274,6 +301,10 @@ func (p *DynamicPolicy) dedicatedCoresAllocationHandler(ctx context.Context, return nil, fmt.Errorf("dedicatedCoresAllocationHandler got nil req") } + if util.PodInplaceUpdateResizing(req) { + return nil, fmt.Errorf("not support inplace update resize for dedicated cores") + } + switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: return p.dedicatedCoresWithNUMABindingAllocationHandler(ctx, req) @@ -425,26 +456,23 @@ func (p *DynamicPolicy) allocationSidecarHandler(_ context.Context, return &pluginapi.ResourceAllocationResponse{}, nil } + // the sidecar container also support inplace update resize, update the allocation and machine state here allocationInfo := &state.AllocationInfo{ - PodUid: req.PodUid, - PodNamespace: req.PodNamespace, - PodName: req.PodName, - ContainerName: req.ContainerName, - ContainerType: req.ContainerType.String(), - ContainerIndex: req.ContainerIndex, - PodRole: req.PodRole, - PodType: req.PodType, - OwnerPoolName: mainContainerAllocationInfo.OwnerPoolName, - AllocationResult: mainContainerAllocationInfo.AllocationResult.Clone(), - OriginalAllocationResult: mainContainerAllocationInfo.OriginalAllocationResult.Clone(), - TopologyAwareAssignments: machine.DeepcopyCPUAssignment(mainContainerAllocationInfo.TopologyAwareAssignments), - OriginalTopologyAwareAssignments: machine.DeepcopyCPUAssignment(mainContainerAllocationInfo.OriginalTopologyAwareAssignments), - InitTimestamp: time.Now().Format(util.QRMTimeFormat), - QoSLevel: qosLevel, - Labels: general.DeepCopyMap(req.Labels), - Annotations: general.DeepCopyMap(req.Annotations), - RequestQuantity: reqFloat64, + PodUid: req.PodUid, + PodNamespace: req.PodNamespace, + PodName: req.PodName, + ContainerName: req.ContainerName, + ContainerType: req.ContainerType.String(), + ContainerIndex: req.ContainerIndex, + PodRole: req.PodRole, + PodType: req.PodType, + InitTimestamp: time.Now().Format(util.QRMTimeFormat), + QoSLevel: qosLevel, + Labels: general.DeepCopyMap(req.Labels), + Annotations: general.DeepCopyMap(req.Annotations), + RequestQuantity: reqFloat64, } + p.applySidecarAllocationInfoFromMainContainer(allocationInfo, mainContainerAllocationInfo) // update pod entries directly. // if one of subsequent steps is failed, we will delete current allocationInfo from podEntries in defer function of allocation function. @@ -604,23 +632,70 @@ func (p *DynamicPolicy) allocateSharedNumaBindingCPUs(req *pluginapi.ResourceReq RequestQuantity: reqFloat64, } - p.state.SetAllocationInfo(allocationInfo.PodUid, allocationInfo.ContainerName, allocationInfo) - checkedAllocationInfo, err := p.doAndCheckPutAllocationInfo(allocationInfo, true) - if err != nil { - return nil, fmt.Errorf("doAndCheckPutAllocationInfo failed with error: %v", err) - } + if util.PodInplaceUpdateResizing(req) { + originAllocationInfo := p.state.GetAllocationInfo(allocationInfo.PodUid, allocationInfo.ContainerName) + if originAllocationInfo == nil { + general.Errorf("pod: %s/%s, container: %s request to cpu inplace update resize alloation, but no origin allocation info, reject it", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("no origion cpu allocation info for inplace update resize") + } - return checkedAllocationInfo, nil + general.Infof("pod: %s/%s, container: %s request to cpu inplace update resize allocation (%.02f->%.02f)", + req.PodNamespace, req.PodName, req.ContainerName, originAllocationInfo.RequestQuantity, allocationInfo.RequestQuantity) + p.state.SetAllocationInfo(allocationInfo.PodUid, allocationInfo.ContainerName, allocationInfo) + checkedAllocationInfo, err := p.doAndCheckPutAllocationInfoPodResizingAware(originAllocationInfo, allocationInfo, false, true) + if err != nil { + general.Errorf("pod: %s/%s, container: %s request to cpu inplace update resize allocation, but doAndCheckPutAllocationInfoPodResizingAware failed: %q", + req.PodNamespace, req.PodName, req.ContainerName, err) + p.state.SetAllocationInfo(originAllocationInfo.PodUid, originAllocationInfo.ContainerName, originAllocationInfo) + return nil, fmt.Errorf("doAndCheckPutAllocationInfo failed with error: %v", err) + } + return checkedAllocationInfo, nil + } else { + p.state.SetAllocationInfo(allocationInfo.PodUid, allocationInfo.ContainerName, allocationInfo) + checkedAllocationInfo, err := p.doAndCheckPutAllocationInfo(allocationInfo, true) + if err != nil { + return nil, fmt.Errorf("doAndCheckPutAllocationInfo failed with error: %v", err) + } + return checkedAllocationInfo, nil + } } // putAllocationsAndAdjustAllocationEntries calculates and generates the latest checkpoint // - unlike adjustAllocationEntries, it will also consider AllocationInfo func (p *DynamicPolicy) putAllocationsAndAdjustAllocationEntries(allocationInfos []*state.AllocationInfo, incrByReq bool) error { + return p.putAllocationsAndAdjustAllocationEntriesResizeAware(nil, allocationInfos, incrByReq, false) +} + +func (p *DynamicPolicy) putAllocationsAndAdjustAllocationEntriesResizeAware(originAllocationInfos, allocationInfos []*state.AllocationInfo, incrByReq, podInplaceUpdateResizing bool) error { if len(allocationInfos) == 0 { return nil } + if podInplaceUpdateResizing { + if len(originAllocationInfos) != 1 && len(allocationInfos) != 1 { + general.Errorf("cannot adjust allocation entries for invalid allocation infos") + return fmt.Errorf("invalid inplace update resize allocation infos length") + } + } entries := p.state.GetPodEntries() + + for _, allocationInfo := range allocationInfos { + if allocationInfo == nil { + return fmt.Errorf("found nil allocationInfo in input parameter") + } else if !state.CheckShared(allocationInfo) { + return fmt.Errorf("put container with invalid qos level: %s into pool", allocationInfo.QoSLevel) + } else if entries[allocationInfo.PodUid][allocationInfo.ContainerName] == nil { + return fmt.Errorf("entry %s/%s, %s isn't found in state", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + } + + poolName := allocationInfo.GetSpecifiedPoolName() + if poolName == state.EmptyOwnerPoolName { + return fmt.Errorf("allocationInfo points to empty poolName") + } + } + machineState := p.state.GetMachineState() var poolsQuantityMap map[string]map[int]int @@ -633,6 +708,29 @@ func (p *DynamicPolicy) putAllocationsAndAdjustAllocationEntries(allocationInfos } poolsQuantityMap = machine.ParseCPUAssignmentQuantityMap(csetMap) + if podInplaceUpdateResizing { + // adjust pool resize + originAllocationInfo := originAllocationInfos[0] + allocationInfo := allocationInfos[0] + + poolName, targetNumaID, resizeReqFloat64, err := p.calcPoolResizeRequest(originAllocationInfo, allocationInfo, entries) + if err != nil { + return fmt.Errorf("calcPoolResizeRequest cannot calc pool resize request: %q", err) + } + + // update the pool size + poolsQuantityMap[poolName][targetNumaID] += int(math.Ceil(resizeReqFloat64)) + // return err will abort the procedure, + // so there is no need to revert modifications made in parameter poolsQuantityMap + if len(poolsQuantityMap[poolName]) > 1 { + return fmt.Errorf("pool %s cross NUMA: %+v", poolName, poolsQuantityMap[poolName]) + } + } else if incrByReq { + err := state.CountAllocationInfosToPoolsQuantityMap(allocationInfos, poolsQuantityMap) + if err != nil { + return fmt.Errorf("CountAllocationInfosToPoolsQuantityMap failed with error: %v", err) + } + } } else { // else we do sum(containers req) for each pool to get pools ratio var err error @@ -640,39 +738,115 @@ func (p *DynamicPolicy) putAllocationsAndAdjustAllocationEntries(allocationInfos if err != nil { return fmt.Errorf("GetSharedQuantityMapFromPodEntries failed with error: %v", err) } + + if incrByReq || podInplaceUpdateResizing { + if podInplaceUpdateResizing { + general.Infof("pod: %s/%s, container: %s request to re-calc pool size for cpu inplace update resize", + allocationInfos[0].PodNamespace, allocationInfos[0].PodName, allocationInfos[0].ContainerName) + } + // if advisor is disabled, qrm can re-calc the pool size exactly. we don't need to adjust the pool size. + err := state.CountAllocationInfosToPoolsQuantityMap(allocationInfos, poolsQuantityMap) + if err != nil { + return fmt.Errorf("CountAllocationInfosToPoolsQuantityMap failed with error: %v", err) + } + } } - for _, allocationInfo := range allocationInfos { - if allocationInfo == nil { - return fmt.Errorf("found nil allocationInfo in input parameter") - } else if !state.CheckShared(allocationInfo) { - return fmt.Errorf("put container with invalid qos level: %s into pool", allocationInfo.QoSLevel) - } else if entries[allocationInfo.PodUid][allocationInfo.ContainerName] == nil { - return fmt.Errorf("entry %s/%s, %s isn't found in state", - allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + isolatedQuantityMap := state.GetIsolatedQuantityMapFromPodEntries(entries, allocationInfos) + err := p.adjustPoolsAndIsolatedEntries(poolsQuantityMap, isolatedQuantityMap, + entries, machineState) + if err != nil { + return fmt.Errorf("adjustPoolsAndIsolatedEntries failed with error: %v", err) + } + + return nil +} + +func (p *DynamicPolicy) calcPoolResizeRequest(originAllocation, allocation *state.AllocationInfo, podEntries state.PodEntries) (string, int, float64, error) { + poolName := allocation.GetPoolName() + targetNumaID := state.FakedNUMAID + + originPodAggregatedRequest, ok := originAllocation.GetPodAggregatedRequest() + if !ok { + containerEntries, ok := podEntries[originAllocation.PodUid] + if !ok { + general.Warningf("pod %s/%s container entries not exist", originAllocation.PodNamespace, originAllocation.PodName) + originPodAggregatedRequest = 0 + } else { + podAggregatedRequestSum := float64(0) + for containerName, containerEntry := range containerEntries { + if containerName == originAllocation.ContainerName { + podAggregatedRequestSum += originAllocation.RequestQuantity + } else { + podAggregatedRequestSum += containerEntry.RequestQuantity + } + } + originPodAggregatedRequest = podAggregatedRequestSum } + } - poolName := allocationInfo.GetSpecifiedPoolName() - if poolName == state.EmptyOwnerPoolName { - return fmt.Errorf("allocationInfo points to empty poolName") + podAggregatedRequest, ok := allocation.GetPodAggregatedRequest() + if !ok { + containerEntries, ok := podEntries[originAllocation.PodUid] + if !ok { + general.Warningf("pod %s/%s container entries not exist", originAllocation.PodNamespace, originAllocation.PodName) + podAggregatedRequest = 0 + } else { + podAggregatedRequestSum := float64(0) + for _, containerEntry := range containerEntries { + podAggregatedRequestSum += containerEntry.RequestQuantity + } + podAggregatedRequest = podAggregatedRequestSum } } - if incrByReq { - err := state.CountAllocationInfosToPoolsQuantityMap(allocationInfos, poolsQuantityMap) + poolResizeQuantity := podAggregatedRequest - originPodAggregatedRequest + if poolResizeQuantity < 0 { + // We don't need to adjust pool size in inplace update scale in mode, wait advisor to adjust the pool size later. + general.Infof("pod: %s/%s, container: %s request cpu inplace update scale in (%.02f->%.02f)", + allocation.PodNamespace, allocation.PodName, allocation.ContainerName, originPodAggregatedRequest, podAggregatedRequest) + poolResizeQuantity = 0 + } else { + // We should adjust pool size in inplace update scale out mode with resizeReqFloat64, and then wait advisor to adjust the pool size later. + general.Infof("pod: %s/%s, container: %s request cpu inplace update scale out (%.02f->%.02f)", + allocation.PodNamespace, allocation.PodName, allocation.ContainerName, originPodAggregatedRequest, podAggregatedRequest) + } + + // only support normal share and snb inplace update resize now + if state.CheckSharedNUMABinding(allocation) { + // check snb numa migrate for inplace update resize + originTargetNumaID, err := state.GetSharedNUMABindingTargetNuma(originAllocation) + if err != nil { + return "", 0, 0, fmt.Errorf("failed to get origin target NUMA") + } + targetNumaID, err = state.GetSharedNUMABindingTargetNuma(allocation) if err != nil { - return fmt.Errorf("CountAllocationInfosToPoolsQuantityMap failed with error: %v", err) + return "", 0, 0, fmt.Errorf("failed to get target NUMA") + } + + // the pod is migrated to a new NUMA if the NUMA changed. + // the new pool should scale out the whole request size. + // the old pool would be adjusted by advisor later. + if originTargetNumaID != targetNumaID { + poolResizeQuantity = podAggregatedRequest + general.Infof("pod %s/%s request inplace update resize and it was migrate to a new NUMA (%d->%d), AggregatedPodRequest(%.02f)", + allocation.PodNamespace, allocation.PodName, originTargetNumaID, targetNumaID, podAggregatedRequest) + } + + // get snb pool name + poolName, err = allocation.GetSpecifiedNUMABindingPoolName() + if err != nil { + return "", 0, 0, fmt.Errorf("GetSpecifiedNUMABindingPoolName for %s/%s/%s failed with error: %v", + allocation.PodNamespace, allocation.PodName, allocation.ContainerName, err) } } - isolatedQuantityMap := state.GetIsolatedQuantityMapFromPodEntries(entries, allocationInfos) - err := p.adjustPoolsAndIsolatedEntries(poolsQuantityMap, isolatedQuantityMap, - entries, machineState) - if err != nil { - return fmt.Errorf("adjustPoolsAndIsolatedEntries failed with error: %v", err) + if poolName == state.EmptyOwnerPoolName { + return "", 0, 0, fmt.Errorf("get poolName failed for %s/%s/%s", + allocation.PodNamespace, allocation.PodName, allocation.ContainerName) } - return nil + return poolName, targetNumaID, poolResizeQuantity, nil } // adjustAllocationEntries calculates and generates the latest checkpoint @@ -1505,15 +1679,15 @@ func (p *DynamicPolicy) shouldSharedCoresRampUp(podUID string) bool { } } -func (p *DynamicPolicy) doAndCheckPutAllocationInfo(allocationInfo *state.AllocationInfo, incrByReq bool) (*state.AllocationInfo, error) { +func (p *DynamicPolicy) doAndCheckPutAllocationInfoPodResizingAware(originAllocationInfo, allocationInfo *state.AllocationInfo, incrByReq, podInplaceUpdateResizing bool) (*state.AllocationInfo, error) { if allocationInfo == nil { return nil, fmt.Errorf("doAndCheckPutAllocationInfo got nil allocationInfo") } // need to adjust pools and putAllocationsAndAdjustAllocationEntries will set the allocationInfo after adjusted - err := p.putAllocationsAndAdjustAllocationEntries([]*state.AllocationInfo{allocationInfo}, incrByReq) + err := p.putAllocationsAndAdjustAllocationEntriesResizeAware([]*state.AllocationInfo{originAllocationInfo}, []*state.AllocationInfo{allocationInfo}, incrByReq, podInplaceUpdateResizing) if err != nil { - general.Errorf("pod: %s/%s, container: %s putAllocationsAndAdjustAllocationEntries failed with error: %v", + general.Errorf("pod: %s/%s, container: %s putAllocationsAndAdjustAllocationEntriesResizeAware failed with error: %v", allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, err) return nil, fmt.Errorf("putAllocationsAndAdjustAllocationEntries failed with error: %v", err) } @@ -1528,6 +1702,10 @@ func (p *DynamicPolicy) doAndCheckPutAllocationInfo(allocationInfo *state.Alloca return checkedAllocationInfo, nil } +func (p *DynamicPolicy) doAndCheckPutAllocationInfo(allocationInfo *state.AllocationInfo, incrByReq bool) (*state.AllocationInfo, error) { + return p.doAndCheckPutAllocationInfoPodResizingAware(nil, allocationInfo, incrByReq, false) +} + func (p *DynamicPolicy) getReclaimOverlapShareRatio(entries state.PodEntries) (map[string]float64, error) { if !p.state.GetAllowSharedCoresOverlapReclaimedCores() { return nil, nil diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go index fe12eac77..7ecb11afc 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go @@ -48,19 +48,42 @@ func (p *DynamicPolicy) sharedCoresHintHandler(ctx context.Context, return nil, fmt.Errorf("got nil request") } - if !qosutil.AnnotationsIndicateNUMABinding(req.Annotations) { - return util.PackResourceHintsResponse(req, string(v1.ResourceCPU), - map[string]*pluginapi.ListOfTopologyHints{ - string(v1.ResourceCPU): nil, // indicates that there is no numa preference - }) + if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) { + return p.sharedCoresWithNUMABindingHintHandler(ctx, req) } - return p.sharedCoresWithNUMABindingHintHandler(ctx, req) + // TODO: support sidecar follow main container for normal share cores in future + if req.ContainerType == pluginapi.ContainerType_MAIN { + ok, err := p.checkNormalShareCoresCpuResource(req) + if err != nil { + general.Errorf("failed to check share cores cpu resource for pod: %s/%s, container: %s", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("failed to check share cores cpu resource: %q", err) + } + + if !ok { + _ = p.emitter.StoreInt64(util.MetricNameShareCoresNoEnoughResourceFailed, 1, metrics.MetricTypeNameCount, metrics.ConvertMapToTags(map[string]string{ + "resource": v1.ResourceCPU.String(), + "podNamespace": req.PodNamespace, + "podName": req.PodName, + "containerName": req.ContainerName, + })...) + return nil, fmt.Errorf("no enough cpu resource") + } + } + + return util.PackResourceHintsResponse(req, string(v1.ResourceCPU), + map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceCPU): nil, // indicates that there is no numa preference + }) } func (p *DynamicPolicy) reclaimedCoresHintHandler(ctx context.Context, req *pluginapi.ResourceRequest, ) (*pluginapi.ResourceHintsResponse, error) { + if util.PodInplaceUpdateResizing(req) { + return nil, fmt.Errorf("not support inplace update resize for reclaimed cores") + } return p.sharedCoresHintHandler(ctx, req) } @@ -71,6 +94,10 @@ func (p *DynamicPolicy) dedicatedCoresHintHandler(ctx context.Context, return nil, fmt.Errorf("dedicatedCoresHintHandler got nil req") } + if util.PodInplaceUpdateResizing(req) { + return nil, fmt.Errorf("not support inplace update resize for dedicated cores") + } + switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: return p.dedicatedCoresWithNUMABindingHintHandler(ctx, req) @@ -503,7 +530,8 @@ func (p *DynamicPolicy) sharedCoresWithNUMABindingHintHandler(_ context.Context, }) } - reqInt, _, err := util.GetQuantityFromResourceReq(req) + // calc the hints with the pod aggregated request + reqInt, _, err := util.GetPodAggregatedRequestResource(req) if err != nil { return nil, fmt.Errorf("getReqQuantityFromResourceReq failed with error: %v", err) } @@ -517,25 +545,62 @@ func (p *DynamicPolicy) sharedCoresWithNUMABindingHintHandler(_ context.Context, if allocationInfo != nil { hints = cpuutil.RegenerateHints(allocationInfo, reqInt) - // regenerateHints failed. need to clear container record and re-calculate. - if hints == nil { - delete(podEntries[req.PodUid], req.ContainerName) - if len(podEntries[req.PodUid]) == 0 { - delete(podEntries, req.PodUid) - } - - var err error - // [TODO]: generateMachineStateFromPodEntries adapts to shared_cores with numa_binding - machineState, err = generateMachineStateFromPodEntries(p.machineInfo.CPUTopology, podEntries) + // clear the current container and regenerate machine state in follow cases: + // 1. regenerateHints failed. + // 2. the container is inplace update resizing. + // hints it as a new container + if hints == nil || util.PodInplaceUpdateResizing(req) { + machineState, err = p.clearContainerAndRegenerateMachineState(podEntries, req) if err != nil { - general.Errorf("pod: %s/%s, container: %s GenerateMachineStateFromPodEntries failed with error: %v", + general.Errorf("pod: %s/%s, container: %s clearContainerAndRegenerateMachineState failed with error: %v", req.PodNamespace, req.PodName, req.ContainerName, err) return nil, fmt.Errorf("GenerateMachineStateFromPodEntries failed with error: %v", err) } } + } else if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to cpu inplace update resize, but no origin allocation info", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("no origin allocation info") } - if hints == nil { + general.Infof("pod: %s/%s, container: %s, inplace update resize: %v", req.PodNamespace, req.PodName, req.ContainerName, util.PodInplaceUpdateResizing(req)) + if util.PodInplaceUpdateResizing(req) { + numaset := allocationInfo.GetAllocationResultNUMASet() + if numaset.Size() != 1 { + general.Errorf("pod: %s/%s, container: %s is snb, but its numa set size is %d", + req.PodNamespace, req.PodName, req.ContainerName, numaset.Size()) + return nil, fmt.Errorf("snb port not support cross numa") + } + nodeID := numaset.ToSliceInt()[0] + availableCPUQuantity := machineState[nodeID].GetAvailableCPUQuantity(p.reservedCPUs) + + general.Infof("pod: %s/%s, container: %s request cpu inplace update resize on numa %d (available: %d, request: %d)", + req.PodNamespace, req.PodName, req.ContainerName, nodeID, availableCPUQuantity, reqInt) + if reqInt > availableCPUQuantity { // no left resource to scale out + general.Infof("pod: %s/%s, container: %s request cpu inplace update resize, but no enough resource for it in current NUMA, checking migratable", + req.PodNamespace, req.PodName, req.ContainerName) + // TODO move this var to config + isInplaceUpdateResizeNumaMigratable := false + if isInplaceUpdateResizeNumaMigratable { + general.Infof("pod: %s/%s, container: %s request inplace update resize and no enough resource in current NUMA, try to migrate it to new NUMA", + req.PodNamespace, req.PodName, req.ContainerName) + var calculateErr error + hints, calculateErr = p.calculateHintsForNUMABindingSharedCores(reqInt, podEntries, machineState, req.Annotations) + if calculateErr != nil { + general.Errorf("pod: %s/%s, container: %s request inplace update resize and no enough resource in current NUMA, failed to migrate it to new NUMA", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("calculateHintsForNUMABindingSharedCores failed in inplace update resize mode with error: %v", calculateErr) + } + } else { + general.Errorf("pod: %s/%s, container: %s request inplace update resize, but no enough resource for it in current NUMA", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("inplace update resize scale out failed with no enough resource") + } + } else { + general.Infof("pod: %s/%s, container: %s request inplace update resize, there is enough resource for it in current NUMA", + req.PodNamespace, req.PodName, req.ContainerName) + } + } else if hints == nil { var calculateErr error hints, calculateErr = p.calculateHintsForNUMABindingSharedCores(reqInt, podEntries, machineState, req.Annotations) if calculateErr != nil { @@ -546,6 +611,21 @@ func (p *DynamicPolicy) sharedCoresWithNUMABindingHintHandler(_ context.Context, return util.PackResourceHintsResponse(req, string(v1.ResourceCPU), hints) } +func (p *DynamicPolicy) clearContainerAndRegenerateMachineState(podEntries state.PodEntries, req *pluginapi.ResourceRequest) (state.NUMANodeMap, error) { + delete(podEntries[req.PodUid], req.ContainerName) + if len(podEntries[req.PodUid]) == 0 { + delete(podEntries, req.PodUid) + } + + var err error + machineState, err := generateMachineStateFromPodEntries(p.machineInfo.CPUTopology, podEntries) + if err != nil { + return nil, fmt.Errorf("GenerateMachineStateFromPodEntries failed with error: %v", err) + } + + return machineState, nil +} + func (p *DynamicPolicy) populateHintsByPreferPolicy(numaNodes []int, preferPolicy string, hints map[string]*pluginapi.ListOfTopologyHints, machineState state.NUMANodeMap, reqInt int, ) { diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go index 709d6ba6e..605ac564e 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - /* Copyright 2022 The Katalyst Authors. @@ -2812,6 +2809,7 @@ func TestGetResourcesAllocation(t *testing.T) { dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) as.Nil(err) + dynamicPolicy.transitionPeriod = time.Millisecond * 10 testName := "test" @@ -2841,6 +2839,9 @@ func TestGetResourcesAllocation(t *testing.T) { resp1, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) as.Nil(err) + reclaim := dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + as.NotNil(resp1.PodResources[req.PodUid]) as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) @@ -2848,8 +2849,8 @@ func TestGetResourcesAllocation(t *testing.T) { OciPropertyName: util.OCIPropertyNameCPUSetCPUs, IsNodeResource: false, IsScalarResource: true, - AllocatedQuantity: 14, - AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).String(), + AllocatedQuantity: 10, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), }, resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) // test after ramping up @@ -5178,3 +5179,129 @@ func Test_getNUMAAllocatedMemBW(t *testing.T) { }) } } + +func TestSNBAdmitWithSidecarReallocate(t *testing.T) { + t.Parallel() + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBAdmit") + as.Nil(err) + defer func() { _ = os.RemoveAll(tmpDir) }() + + cpuTopology, err := machine.GenerateDummyCPUTopology(12, 1, 1) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) + as.Nil(err) + + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + sidecarName := "sidecar" + podUID := string(uuid.NewUUID()) + // admit sidecar container + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"cpu": 8}`, + }, + } + + res, err := dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(res.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + // admit main container + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 6, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"cpu": 8}`, + }, + } + + res, err = dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + hints := res.ResourceHints[string(v1.ResourceCPU)].Hints + as.NotZero(len(hints)) + req.Hint = hints[0] + + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + // we cannot GetResourceAllocation here, it will + // not sidecar container is wait for reallocate, only main container is in state file + sidecarAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, sidecarName) + as.Nil(sidecarAllocation) + + mainAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, testName) + as.NotNil(mainAllocation) + + // another container before reallocate with 4 core (because share-NUMA0 has only 11 core and 6 cores is allocated to podUID/test). + anotherReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: "test1", + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 4, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"cpu": 4}`, + }, + } + + // pod aggregated size is 8, the new container request is 4, 8 + 4 > 11 (share-NUMA0 size) + res, err = dynamicPolicy.GetTopologyHints(context.Background(), anotherReq) + as.Nil(err) + as.NotNil(res.ResourceHints[string(v1.ResourceCPU)]) + as.Equal(0, len(res.ResourceHints[string(v1.ResourceCPU)].Hints)) + + // reallocate sidecar + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + sidecarAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, sidecarName) + as.NotNil(sidecarAllocation) +} diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go index b134a19a1..c8576e3c1 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state.go @@ -21,6 +21,7 @@ import ( "fmt" "math" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" @@ -231,6 +232,23 @@ func (ai *AllocationInfo) CheckSideCar() bool { return ai.ContainerType == pluginapi.ContainerType_SIDECAR.String() } +func (ai *AllocationInfo) GetPodAggregatedRequest() (float64, bool) { + if ai.Annotations == nil { + return 0, false + } + value, ok := ai.Annotations[apiconsts.PodAnnotationAggregatedRequestsKey] + if !ok { + return 0, false + } + var resourceList v1.ResourceList + if err := json.Unmarshal([]byte(value), &resourceList); err != nil { + general.Errorf("failed to unmarshal pod aggregated request list: %q", err) + return 0, false + } + + return float64(resourceList.Cpu().MilliValue()) / 1000, true +} + // CheckDedicated returns true if the AllocationInfo is for pod with dedicated-qos func CheckDedicated(ai *AllocationInfo) bool { if ai == nil { @@ -526,6 +544,19 @@ func (ns *NUMANodeState) GetAvailableCPUQuantity(reservedCPUs machine.CPUSet) in continue } + // if there is pod aggregated resource key in main container annotations, use pod aggregated resource instead. + mainContainerEntry := containerEntries.GetMainContainerEntry() + if mainContainerEntry == nil || !CheckSharedNUMABinding(mainContainerEntry) { + continue + } + + aggregatedPodResource, ok := mainContainerEntry.GetPodAggregatedRequest() + if ok { + preciseAllocatedQuantity += aggregatedPodResource + continue + } + + // calc pod aggregated resource request by container entries. for _, allocationInfo := range containerEntries { if allocationInfo == nil || !CheckSharedNUMABinding(allocationInfo) { diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go index b1622936a..763b4a557 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/state_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/require" "github.com/kubewharf/katalyst-api/pkg/consts" + apiconsts "github.com/kubewharf/katalyst-api/pkg/consts" cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts" "github.com/kubewharf/katalyst-core/pkg/util/machine" ) @@ -3110,3 +3111,165 @@ func TestPodEntries_GetFilteredPoolsCPUSetMap(t *testing.T) { }) } } + +func TestGetAggregatedRequest(t *testing.T) { + t.Parallel() + + allocation := &AllocationInfo{} + _, ok := allocation.GetPodAggregatedRequest() + require.Equal(t, false, ok) + + allocation.Annotations = map[string]string{} + _, ok = allocation.GetPodAggregatedRequest() + require.Equal(t, false, ok) + + allocation.Annotations = map[string]string{ + apiconsts.PodAnnotationAggregatedRequestsKey: "", + } + _, ok = allocation.GetPodAggregatedRequest() + require.Equal(t, false, ok) + + require.Equal(t, false, ok) + allocation.Annotations = map[string]string{ + apiconsts.PodAnnotationAggregatedRequestsKey: "{\"cpu\": \"\"}", + } + _, ok = allocation.GetPodAggregatedRequest() + require.Equal(t, false, ok) + + allocation.Annotations = map[string]string{ + apiconsts.PodAnnotationAggregatedRequestsKey: "{\"cpu\": \"6\"}", + } + req, ok := allocation.GetPodAggregatedRequest() + require.Equal(t, true, ok) + require.Equal(t, req, float64(6)) +} + +func TestGetAvailableCPUQuantity(t *testing.T) { + t.Parallel() + + cpuTopology, err := machine.GenerateDummyCPUTopology(48, 1, 2) + require.NoError(t, err) + + testName := "test" + sidecarName := "sidecar" + nodeState := &NUMANodeState{ + DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(1).Clone(), + AllocatedCPUSet: machine.NewCPUSet(), + PodEntries: PodEntries{ + "373d08e4-7a6b-4293-aaaf-b135ff8123bf": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "373d08e4-7a6b-4293-aaaf-b135ff8123bf", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("3,11"), + OriginalAllocationResult: machine.MustParse("3,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + consts.PodAnnotationMemoryEnhancementNumaBinding: "true", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + sidecarName: &AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("3,11"), + OriginalAllocationResult: machine.MustParse("3,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: "true", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + "ec6e2f30-c78a-4bc4-9576-c916db5281a3": ContainerEntries{ + testName: &AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("3,11"), + OriginalAllocationResult: machine.MustParse("3,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: "true", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + sidecarName: &AllocationInfo{ + PodUid: "ec6e2f30-c78a-4bc4-9576-c916db5281a3", + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR.String(), + ContainerIndex: 0, + RampUp: false, + OwnerPoolName: PoolNameShare, + AllocationResult: machine.MustParse("3,11"), + OriginalAllocationResult: machine.MustParse("3,11"), + TopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ + 1: machine.NewCPUSet(3, 11), + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementNumaBinding: "true", + }, + QoSLevel: consts.PodAnnotationQoSLevelSharedCores, + RequestQuantity: 2, + }, + }, + }, + } + require.Equal(t, 15, nodeState.GetAvailableCPUQuantity(machine.NewCPUSet())) +} diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util.go index 9e3fdd8b4..06cb1bd55 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state/util.go @@ -420,3 +420,66 @@ func CountAllocationInfosToPoolsQuantityMap(allocationInfos []*AllocationInfo, return nil } + +func GetSharedNUMABindingTargetNuma(allocationInfo *AllocationInfo) (int, error) { + var numaSet machine.CPUSet + poolName := allocationInfo.GetOwnerPoolName() + + if poolName == EmptyOwnerPoolName { + var pErr error + poolName, pErr = allocationInfo.GetSpecifiedNUMABindingPoolName() + if pErr != nil { + return FakedNUMAID, fmt.Errorf("GetSpecifiedNUMABindingPoolName for %s/%s/%s failed with error: %v", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, pErr) + } + + numaSet, pErr = machine.Parse(allocationInfo.Annotations[cpuconsts.CPUStateAnnotationKeyNUMAHint]) + if pErr != nil { + return FakedNUMAID, fmt.Errorf("parse numaHintStr: %s failed with error: %v", + allocationInfo.Annotations[cpuconsts.CPUStateAnnotationKeyNUMAHint], pErr) + } + + general.Infof(" %s/%s/%s count to specified NUMA binding pool name: %s, numaSet: %s", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, poolName, numaSet.String()) + } else { + // already in a valid pool (numa aware pool or isolation pool) + numaSet = allocationInfo.GetAllocationResultNUMASet() + + general.Infof(" %s/%s/%s count to non-empty owner pool name: %s, numaSet: %s", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName, poolName, numaSet.String()) + } + + if numaSet.Size() != 1 { + return FakedNUMAID, fmt.Errorf("numaHintStr: %s indicates invalid numaSet size for numa_binding shared_cores", + allocationInfo.Annotations[cpuconsts.CPUStateAnnotationKeyNUMAHint]) + } + + targetNUMAID := numaSet.ToSliceNoSortInt()[0] + + if targetNUMAID < 0 { + return FakedNUMAID, fmt.Errorf("numaHintStr: %s indicates invalid numaSet numa_binding shared_cores", + allocationInfo.Annotations[cpuconsts.CPUStateAnnotationKeyNUMAHint]) + } + + return targetNUMAID, nil +} + +func CheckAllocationInfoTopologyAwareAssignments(ai1, ai2 *AllocationInfo) bool { + return checkCPUSetMap(ai1.TopologyAwareAssignments, ai2.TopologyAwareAssignments) +} + +func CheckAllocationInfoOriginTopologyAwareAssignments(ai1, ai2 *AllocationInfo) bool { + return checkCPUSetMap(ai1.OriginalTopologyAwareAssignments, ai2.OriginalTopologyAwareAssignments) +} + +func checkCPUSetMap(map1, map2 map[int]machine.CPUSet) bool { + if len(map1) != len(map2) { + return false + } + for numaNode, cset := range map1 { + if !map2[numaNode].Equals(cset) { + return false + } + } + return true +} diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/vpa_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/vpa_test.go new file mode 100644 index 000000000..a866e2606 --- /dev/null +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/vpa_test.go @@ -0,0 +1,1045 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamicpolicy + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/uuid" + pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" + + "github.com/kubewharf/katalyst-api/pkg/consts" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" + "github.com/kubewharf/katalyst-core/pkg/util/machine" +) + +func TestSNBVPA(t *testing.T) { + t.Parallel() + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBVPA") + as.Nil(err) + defer func() { _ = os.RemoveAll(tmpDir) }() + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) + as.Nil(err) + + dynamicPolicy.podAnnotationKeptKeys = []string{consts.PodAnnotationInplaceUpdateResizingKey} + + testName := "test" + + // allocate container + req := &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + } + + res, err := dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + hints := res.ResourceHints[string(v1.ResourceCPU)].Hints + as.NotZero(len(hints)) + + req = &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Hint: hints[0], + } + + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + resp1, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resp1.PodResources[req.PodUid]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 3, // 分配到numa0 (cpu0 -> reserved, cpu1,cpu8,cpu9 for snb) + AllocationResult: "1,8-9", + }, resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + + // resize exceed + resizeReq := &pluginapi.ResourceRequest{ + PodUid: req.PodUid, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 4, // greater than pool size + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + }, + } + + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.NotNil(err) + + // resize successfully + resizeReq1 := &pluginapi.ResourceRequest{ + PodUid: req.PodUid, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, // greater than pool size + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + }, + } + + resizeResp1, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeReq1) + as.Nil(err) + resizeHints1 := resizeResp1.ResourceHints[string(v1.ResourceCPU)].Hints + as.NotZero(len(resizeHints1)) + + resizeReq1 = &pluginapi.ResourceRequest{ + PodUid: req.PodUid, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, // greater than pool size + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + }, + Hint: resizeHints1[0], + } + + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq1) + as.Nil(err) + + resp1, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resp1.PodResources[req.PodUid]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 3, // 分配到numa0 (cpu0 -> reserved, cpu1,cpu8,cpu9 for snb) + AllocationResult: "1,8-9", + }, resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) +} + +func TestSNBVPAWithSidecar(t *testing.T) { + t.Parallel() + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBVPAWithSidecar") + as.Nil(err) + defer func() { _ = os.RemoveAll(tmpDir) }() + + cpuTopology, err := machine.GenerateDummyCPUTopology(48, 2, 4) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) + as.Nil(err) + + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + podNamespace := "test" + podName := "test" + mainContainerName := "main" + sidecarContainerName := "sidecar" + + podUID := string(uuid.NewUUID()) + // admit sidecar firstly + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 1, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + } + + // no hints for sidecar + sidecarRes, err := dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(sidecarRes.ResourceHints[string(v1.ResourceCPU)]) + + // main container doesn't allocate, return nil + sidecarReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 1, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + } + sidecarAllocateRes, err := dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(sidecarAllocateRes.AllocationResult) + + // no sidecar container record here, because main container doesn't allocate now + allocationRes, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + _, exist := allocationRes.PodResources[podUID] + as.Equal(false, exist) + + mainReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + } + + mainRes, err := dynamicPolicy.GetTopologyHints(context.Background(), mainReq) + as.Nil(err) + hints := mainRes.ResourceHints[string(v1.ResourceCPU)].Hints + as.NotZero(len(hints)) + + mainReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + Hint: hints[0], + } + + _, err = dynamicPolicy.Allocate(context.Background(), mainReq) + as.Nil(err) + + allocationRes, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(allocationRes.PodResources[mainReq.PodUid]) + // check main container + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[mainContainerName]) + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, allocationRes.PodResources[mainReq.PodUid].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + // reallocate for sidecar + // no hints for sidecar + sidecarRes, err = dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(sidecarRes.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + allocationRes, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + // check sidecar container (follow main container) + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[sidecarContainerName]) + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, allocationRes.PodResources[mainReq.PodUid].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + // check container allocation request + mainContainerAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(2), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(1), sidecarContainerAllocation.RequestQuantity) + // check pod aggregated request + aggregatedRequest, ok := mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(3), aggregatedRequest) + + resizeMainContainerReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"4\"}", + }, + } + + resizeMainContainerResp, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeMainContainerReq) + as.Nil(err) + resizeMainContainerHints := resizeMainContainerResp.ResourceHints[string(v1.ResourceCPU)].Hints + as.NotZero(len(resizeMainContainerHints)) + resizeMainContainerReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"4\"}", + }, + Hint: resizeMainContainerHints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeMainContainerReq) + as.Nil(err) + + resizeMainContainerAllocations, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resizeMainContainerAllocations.PodResources[podUID]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + mainContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(3), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(1), sidecarContainerAllocation.RequestQuantity) + + // check pod aggregated request + aggregatedRequest, ok = mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(4), aggregatedRequest) + + // resize sidecar + resizeSidecarContainerReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + }, + } + + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeSidecarContainerReq) + as.Nil(err) + + resizeSidecarContainerReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + }, + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeSidecarContainerReq) + as.Nil(err) + + resizeMainContainerReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + }, + } + resizeMainContainerResp, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeMainContainerReq) + as.Nil(err) + + resizeMainContainerReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + }, + Hint: resizeMainContainerResp.ResourceHints[string(v1.ResourceCPU)].Hints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeMainContainerReq) + as.Nil(err) + + resizeSidecarContainerAllocations, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 11, // 分配到numa0 (cpu0 -> reserved, cpu1~cpu5,cpu24~cpu29 for snb) + AllocationResult: "1-5,24-29", + }, resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + mainContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(3), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(2), sidecarContainerAllocation.RequestQuantity) + + // check pod aggregated request + aggregatedRequest, ok = mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(5), aggregatedRequest) + + // resize main exceed + resizeMainContainerReq.ResourceRequests[string(v1.ResourceCPU)] = 10 + resizeMainContainerReq.Annotations[consts.PodAnnotationAggregatedRequestsKey] = "{\"cpu\":\"12\"}" + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeMainContainerReq) + as.Nil(err) + + // resize sidecar exceed + resizeSidecarContainerReq.ResourceRequests[string(v1.ResourceCPU)] = 10 + resizeSidecarContainerReq.Annotations[consts.PodAnnotationAggregatedRequestsKey] = "{\"cpu\":\"13\"}" + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeSidecarContainerReq) + as.Nil(err) +} + +func TestNormalShareVPA(t *testing.T) { + t.Parallel() + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestNormalShareVPA") + as.Nil(err) + defer func() { _ = os.RemoveAll(tmpDir) }() + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) + as.Nil(err) + + dynamicPolicy.podAnnotationKeptKeys = []string{consts.PodAnnotationMemoryEnhancementNumaBinding, consts.PodAnnotationInplaceUpdateResizingKey} + dynamicPolicy.transitionPeriod = 10 * time.Millisecond + + testName := "test" + + // test for normal share + req := &pluginapi.ResourceRequest{ + PodUid: string(uuid.NewUUID()), + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no topology hints for normal share + res, err := dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.Nil(res.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + time.Sleep(20 * time.Millisecond) + + resp1, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + reclaim := dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + + as.NotNil(resp1.PodResources[req.PodUid]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + // share pool size: 10, reclaimed pool size: 4, reserved pool size: 2 + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 10, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + + resizeReq := &pluginapi.ResourceRequest{ + PodUid: req.PodUid, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 16, // greater than pool size + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + }, + } + + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.ErrorContains(err, "no enough") + + resizeReq1 := &pluginapi.ResourceRequest{ + PodUid: req.PodUid, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, // greater than pool size + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + }, + } + + resizeResp1, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeReq1) + as.Nil(err) + // no hints for normal share + as.Nil(resizeResp1.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq1) + as.Nil(err) + + resp1, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + reclaim = dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + + as.NotNil(resp1.PodResources[req.PodUid]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) + as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + // 10 = 16 - 2(reserved) - 4(reclaimed) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 10, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) + + allocation := dynamicPolicy.state.GetAllocationInfo(req.PodUid, testName) + as.NotNil(allocation) + as.Equal(float64(3), allocation.RequestQuantity) +} + +func TestNormalShareVPAWithSidecar(t *testing.T) { + t.Parallel() + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBVPAWithSidecar") + as.Nil(err) + defer func() { _ = os.RemoveAll(tmpDir) }() + + cpuTopology, err := machine.GenerateDummyCPUTopology(48, 2, 4) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) + as.Nil(err) + dynamicPolicy.state.SetAllowSharedCoresOverlapReclaimedCores(false) + dynamicPolicy.transitionPeriod = 10 * time.Millisecond + + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + podNamespace := "test" + podName := "test" + mainContainerName := "main" + sidecarContainerName := "sidecar" + + podUID := string(uuid.NewUUID()) + // admit sidecar firstly + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 1, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + } + + // no hints for share cores container (sidecar and main container) + sidecarRes, err := dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(sidecarRes.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + time.Sleep(20 * time.Millisecond) + + // there is sidecar hints for sidecar container here, and it was bound to the whole share pool + allocationRes, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + _, exist := allocationRes.PodResources[podUID] + as.Equal(true, exist) + as.NotNil(allocationRes.PodResources[podUID].ContainerResources[sidecarContainerName]) + as.NotNil(allocationRes.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + reclaim := dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, allocationRes.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + mainReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"3\"}", + }, + } + + // there is no hints for share cores container (sidecar and main container) + mainRes, err := dynamicPolicy.GetTopologyHints(context.Background(), mainReq) + as.Nil(err) + as.Nil(mainRes.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), mainReq) + as.Nil(err) + + // sleep to finish main container ramp up + time.Sleep(20 * time.Millisecond) + allocationRes, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + // main container was bound to whole share pool now + as.NotNil(allocationRes.PodResources[mainReq.PodUid]) + // check main container + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[mainContainerName]) + as.NotNil(allocationRes.PodResources[mainReq.PodUid].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + reclaim = dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, allocationRes.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + // no reallocate for share cores sidecar + + // check container allocation request + mainContainerAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(2), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation := dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(1), sidecarContainerAllocation.RequestQuantity) + // check pod aggregated request + aggregatedRequest, ok := mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(3), aggregatedRequest) + + resizeMainContainerReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: mainContainerName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 3, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"4\"}", + }, + } + + // there is no hints for share cores container (sidecar and main container) + resizeMainContainerResp, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeMainContainerReq) + as.Nil(err) + as.Nil(resizeMainContainerResp.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeMainContainerReq) + as.Nil(err) + + resizeMainContainerAllocations, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resizeMainContainerAllocations.PodResources[podUID]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + reclaim = dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resizeMainContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName]) + as.NotNil(resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resizeMainContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + mainContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(3), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(1), sidecarContainerAllocation.RequestQuantity) + + // check pod aggregated request + aggregatedRequest, ok = mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(4), aggregatedRequest) + + // resize sidecar + resizeSidecarContainerReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: podNamespace, + PodName: podName, + ContainerName: sidecarContainerName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceCPU), + ResourceRequests: map[string]float64{ + string(v1.ResourceCPU): 2, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationAggregatedRequestsKey: "{\"cpu\":\"5\"}", + }, + } + + // resize sidecar firstly + resizeSidecarContainerHints, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeSidecarContainerReq) + as.Nil(err) + as.Nil(resizeSidecarContainerHints.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeSidecarContainerReq) + as.Nil(err) + + // resize main container (only update pod aggregated request) + resizeMainContainerReq.Annotations[consts.PodAnnotationAggregatedRequestsKey] = "{\"cpu\":\"5\"}" + resizeMainContainerHints, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeMainContainerReq) + as.Nil(err) + as.Nil(resizeMainContainerHints.ResourceHints[string(v1.ResourceCPU)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeMainContainerReq) + as.Nil(err) + + resizeSidecarContainerAllocations, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + reclaim = dynamicPolicy.state.GetAllocationInfo(state.PoolNameReclaim, state.FakedContainerName) + as.NotNil(reclaim) + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName]) + as.NotNil(resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[mainContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + // reserve pool size: 2, reclaimed pool size: 4, share pool size: 42 + as.Equal(&pluginapi.ResourceAllocationInfo{ + OciPropertyName: util.OCIPropertyNameCPUSetCPUs, + IsNodeResource: false, + IsScalarResource: true, + AllocatedQuantity: 42, + AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).Difference(reclaim.AllocationResult).String(), + }, resizeSidecarContainerAllocations.PodResources[podUID].ContainerResources[sidecarContainerName].ResourceAllocation[string(v1.ResourceCPU)]) + + mainContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, mainContainerName) + as.Equal(float64(3), mainContainerAllocation.RequestQuantity) + sidecarContainerAllocation = dynamicPolicy.state.GetAllocationInfo(podUID, sidecarContainerName) + as.Equal(float64(2), sidecarContainerAllocation.RequestQuantity) + + // check pod aggregated request + aggregatedRequest, ok = mainContainerAllocation.GetPodAggregatedRequest() + as.Equal(true, ok) + as.Equal(float64(5), aggregatedRequest) +} diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go index 3793f5b24..125acb408 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go @@ -303,7 +303,7 @@ func (p *DynamicPolicy) Start() (err error) { general.Errorf("start %v failed, err: %v", memconsts.ClearResidualState, err) } - // we should remove this healthy check when we support vpa in all clusters. + // TODO: we should remove this healthy check when we support inplace update resize in all clusters. syncMemoryStatusFromSpec := func(_ *config.Configuration, _ interface{}, _ *dynamicconfig.DynamicAgentConfiguration, @@ -320,7 +320,7 @@ func (p *DynamicPolicy) Start() (err error) { general.Warningf("sync memory state from pod spec successfully") } } - err = periodicalhandler.RegisterPeriodicalHandler(memconsts.SyncMemoryStateFromSpec, qrm.QRMMemoryPluginPeriodicalHandlerGroupName, + err = periodicalhandler.RegisterPeriodicalHandler(qrm.QRMMemoryPluginPeriodicalHandlerGroupName, memconsts.SyncMemoryStateFromSpec, syncMemoryStatusFromSpec, stateCheckPeriod) if err != nil { general.Errorf("start %v failed, err: %v", memconsts.SyncMemoryStateFromSpec, err) @@ -510,7 +510,7 @@ func (p *DynamicPolicy) GetTopologyHints(ctx context.Context, return nil, fmt.Errorf("getReqQuantityFromResourceReq failed with error: %v", err) } - general.InfoS("GetTopologyHints is called", + general.InfoS("called", "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, @@ -612,16 +612,27 @@ func (p *DynamicPolicy) GetResourcesAllocation(_ context.Context, podResources := make(map[string]*pluginapi.ContainerResources) podEntries := p.state.GetPodResourceEntries()[v1.ResourceMemory] + needUpdateMachineState := false for podUID, containerEntries := range podEntries { if podResources[podUID] == nil { podResources[podUID] = &pluginapi.ContainerResources{} } + mainContainerAllocationInfo, _ := podEntries.GetMainContainerAllocation(podUID) for containerName, allocationInfo := range containerEntries { if allocationInfo == nil { continue } + if allocationInfo.CheckSideCar() && mainContainerAllocationInfo != nil { + if applySidecarAllocationInfoFromMainContainer(allocationInfo, mainContainerAllocationInfo) { + general.Infof("pod: %s/%s sidecar container: %s update its allocation", + allocationInfo.PodNamespace, allocationInfo.PodName, allocationInfo.ContainerName) + p.state.SetAllocationInfo(v1.ResourceMemory, podUID, containerName, allocationInfo) + needUpdateMachineState = true + } + } + if podResources[podUID].ContainerResources == nil { podResources[podUID].ContainerResources = make(map[string]*pluginapi.ResourceAllocation) } @@ -639,6 +650,17 @@ func (p *DynamicPolicy) GetResourcesAllocation(_ context.Context, } } + if needUpdateMachineState { + general.Infof("GetResourcesAllocation update machine state") + podResourceEntries := p.state.GetPodResourceEntries() + resourcesState, err := state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) + if err != nil { + general.Infof("GetResourcesAllocation GenerateMachineStateFromPodEntries failed with error: %v", err) + return nil, fmt.Errorf("calculate machineState by updated pod entries failed with error: %v", err) + } + p.state.SetMachineState(resourcesState) + } + return &pluginapi.GetResourcesAllocationResponse{ PodResources: podResources, }, nil @@ -790,7 +812,8 @@ func (p *DynamicPolicy) Allocate(ctx context.Context, "podType", req.PodType, "podRole", req.PodRole, "qosLevel", qosLevel, - "memoryReq(bytes)", reqInt) + "memoryReq(bytes)", reqInt, + "hint", req.Hint) if req.ContainerType == pluginapi.ContainerType_INIT { return &pluginapi.ResourceAllocationResponse{ @@ -863,7 +886,7 @@ func (p *DynamicPolicy) Allocate(ctx context.Context, }() allocationInfo := p.state.GetAllocationInfo(v1.ResourceMemory, req.PodUid, req.ContainerName) - if allocationInfo != nil && allocationInfo.AggregatedQuantity >= uint64(reqInt) { + if allocationInfo != nil && allocationInfo.AggregatedQuantity >= uint64(reqInt) && !util.PodInplaceUpdateResizing(req) { general.InfoS("already allocated and meet requirement", "podNamespace", req.PodNamespace, "podName", req.PodName, @@ -974,12 +997,14 @@ func (p *DynamicPolicy) getContainerRequestedMemoryBytes(allocationInfo *state.A return allocationInfo.AggregatedQuantity } - // check request for VPA + // TODO optimize this logic someday: + // only for refresh cpu request for old pod with cpu ceil and old VPA pods. + // we can remove refresh logic after upgrade all kubelet and qrm. if allocationInfo.QoSLevel == apiconsts.PodAnnotationQoSLevelSharedCores { // if there is these two annotations in memory state, it is a new pod, // we don't need to check the pod request from podWatcher if allocationInfo.Annotations[apiconsts.PodAnnotationAggregatedRequestsKey] != "" || - allocationInfo.Annotations[apiconsts.PodAnnotationVPAResizingKey] != "" { + allocationInfo.Annotations[apiconsts.PodAnnotationInplaceUpdateResizingKey] != "" { return allocationInfo.AggregatedQuantity } @@ -1075,3 +1100,48 @@ func (p *DynamicPolicy) hasLastLevelEnhancementKey(lastLevelEnhancementKey strin general.Infof("pod: %s does not have last level enhancement key: %s", podUID, lastLevelEnhancementKey) return false } + +func (p *DynamicPolicy) checkNormalShareCoresResource(req *pluginapi.ResourceRequest) (bool, error) { + reqInt, _, err := util.GetPodAggregatedRequestResource(req) + if err != nil { + return false, fmt.Errorf("GetQuantityFromResourceReq failed with error: %v", err) + } + + shareCoresAllocated := uint64(reqInt) + podEntries := p.state.GetPodResourceEntries()[v1.ResourceMemory] + for podUid, containerEntries := range podEntries { + for _, containerAllocation := range containerEntries { + // skip the current pod + if podUid == req.PodUid { + continue + } + // shareCoresAllocated should involve both main and sidecar containers + if containerAllocation.QoSLevel == apiconsts.PodAnnotationQoSLevelSharedCores && !containerAllocation.CheckNumaBinding() { + shareCoresAllocated += p.getContainerRequestedMemoryBytes(containerAllocation) + } + } + } + + machineState := p.state.GetMachineState() + resourceState := machineState[v1.ResourceMemory] + numaWithoutNUMABindingPods := resourceState.GetNUMANodesWithoutNUMABindingPods() + numaAllocatableWithoutNUMABindingPods := uint64(0) + for _, numaID := range numaWithoutNUMABindingPods.ToSliceInt() { + numaAllocatableWithoutNUMABindingPods += resourceState[numaID].Allocatable + } + + general.Infof("[checkNormalShareCoresResource] node memory allocated: %d, allocatable: %d", shareCoresAllocated, numaAllocatableWithoutNUMABindingPods) + if shareCoresAllocated > numaAllocatableWithoutNUMABindingPods { + general.Warningf("[checkNormalShareCoresResource] no enough memory resource for normal share cores pod: %s/%s, container: %s (allocated: %d, allocatable: %d)", + req.PodNamespace, req.PodName, req.ContainerName, shareCoresAllocated, numaAllocatableWithoutNUMABindingPods) + return false, nil + } + + general.InfoS("checkNormalShareCoresResource memory successfully", + "podNamespace", req.PodNamespace, + "podName", req.PodName, + "containerName", req.ContainerName, + "reqInt", reqInt) + + return true, nil +} diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go index fb17be1da..8da9becbc 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_allocation_handlers.go @@ -56,6 +56,12 @@ func (p *DynamicPolicy) reclaimedCoresAllocationHandler(ctx context.Context, return nil, fmt.Errorf("reclaimedCoresAllocationHandler got nil request") } + if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to memory inplace update resize, but not support reclaimed cores", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("not support inplace update resize for reclaiemd cores") + } + // TODO: currently we set all numas as cpuset.mems for reclaimed_cores containers, // we will support adjusting cpuset.mems for reclaimed_cores dynamically according to memory advisor. // Notice: before supporting dynamic adjustment, not to hybrid reclaimed_cores @@ -70,6 +76,12 @@ func (p *DynamicPolicy) dedicatedCoresAllocationHandler(ctx context.Context, return nil, fmt.Errorf("dedicatedCoresAllocationHandler got nil req") } + if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to memory inplace update resize, but not support dedicated cores", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("not support inplace update resize for didecated cores") + } + switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: return p.numaBindingAllocationHandler(ctx, req, apiconsts.PodAnnotationQoSLevelDedicatedCores) @@ -82,12 +94,14 @@ func (p *DynamicPolicy) numaBindingAllocationHandler(ctx context.Context, req *pluginapi.ResourceRequest, qosLevel string, ) (*pluginapi.ResourceAllocationResponse, error) { if req.ContainerType == pluginapi.ContainerType_SIDECAR { + // sidecar container admit after main container return p.numaBindingAllocationSidecarHandler(ctx, req, qosLevel) } - reqInt, _, err := util.GetQuantityFromResourceReq(req) + // use the pod aggregated request to instead of main container. + podAggregatedRequest, _, err := util.GetPodAggregatedRequestResource(req) if err != nil { - return nil, fmt.Errorf("GetQuantityFromResourceReq failed with error: %v", err) + return nil, fmt.Errorf("GetPodAggregatedRequestResource failed with error: %v", err) } machineState := p.state.GetMachineState() @@ -97,29 +111,34 @@ func (p *DynamicPolicy) numaBindingAllocationHandler(ctx context.Context, podEntries := podResourceEntries[v1.ResourceMemory] allocationInfo := p.state.GetAllocationInfo(v1.ResourceMemory, req.PodUid, req.ContainerName) - if allocationInfo != nil && allocationInfo.AggregatedQuantity >= uint64(reqInt) { - general.InfoS("already allocated and meet requirement", - "podNamespace", req.PodNamespace, - "podName", req.PodName, - "containerName", req.ContainerName, - "memoryReq(bytes)", reqInt, - "currentResult(bytes)", allocationInfo.AggregatedQuantity) + if allocationInfo != nil { + if allocationInfo.AggregatedQuantity >= uint64(podAggregatedRequest) && !util.PodInplaceUpdateResizing(req) { + general.InfoS("already allocated and meet requirement", + "podNamespace", req.PodNamespace, + "podName", req.PodName, + "containerName", req.ContainerName, + "memoryReq(bytes)", podAggregatedRequest, + "currentResult(bytes)", allocationInfo.AggregatedQuantity) - resp, packErr := packAllocationResponse(allocationInfo, req) - if packErr != nil { - general.Errorf("pod: %s/%s, container: %s packAllocationResponse failed with error: %v", - req.PodNamespace, req.PodName, req.ContainerName, packErr) - return nil, fmt.Errorf("packAllocationResponse failed with error: %v", packErr) + resp, packErr := packAllocationResponse(allocationInfo, req) + if packErr != nil { + general.Errorf("pod: %s/%s, container: %s packAllocationResponse failed with error: %v", + req.PodNamespace, req.PodName, req.ContainerName, packErr) + return nil, fmt.Errorf("packAllocationResponse failed with error: %v", packErr) + } + return resp, nil } - return resp, nil - } else if allocationInfo != nil { general.InfoS("not meet requirement, clear record and re-allocate", "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, - "memoryReq(bytes)", reqInt, + "memoryReq(bytes)", podAggregatedRequest, "currentResult(bytes)", allocationInfo.AggregatedQuantity) - delete(podEntries, req.PodUid) + + // remove the main container of this pod (the main container involve the whole pod requests), and the + // sidecar container request in state is zero. + containerEntries := podEntries[req.PodUid] + delete(containerEntries, req.ContainerName) var stateErr error memoryState, stateErr = state.GenerateMemoryStateFromPodEntries(p.state.GetMachineInfo(), podEntries, p.state.GetReservedMemory()) @@ -128,21 +147,25 @@ func (p *DynamicPolicy) numaBindingAllocationHandler(ctx context.Context, "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, - "memoryReq(bytes)", reqInt, + "memoryReq(bytes)", podAggregatedRequest, "currentResult(bytes)", allocationInfo.AggregatedQuantity) return nil, fmt.Errorf("generateMemoryMachineStateByPodEntries failed with error: %v", stateErr) } + } else if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod %s/%s, container: %s request to memory inplace update resize, but no origin allocation info", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("no origin allocation info") } // call calculateMemoryAllocation to update memoryState in-place, // and we can use this adjusted state to pack allocation results - err = p.calculateMemoryAllocation(req, memoryState, apiconsts.PodAnnotationQoSLevelDedicatedCores) + err = p.calculateMemoryAllocation(req, memoryState, qosLevel, podAggregatedRequest) if err != nil { general.ErrorS(err, "unable to allocate Memory", "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, - "memoryReq", reqInt) + "memoryReq", podAggregatedRequest) return nil, err } @@ -162,7 +185,7 @@ func (p *DynamicPolicy) numaBindingAllocationHandler(ctx context.Context, "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, - "reqMemoryQuantity", reqInt, + "reqMemoryQuantity", podAggregatedRequest, "numaAllocationResult", result.String()) allocationInfo = &state.AllocationInfo{ @@ -244,13 +267,13 @@ func (p *DynamicPolicy) numaBindingAllocationSidecarHandler(_ context.Context, ContainerIndex: req.ContainerIndex, PodRole: req.PodRole, PodType: req.PodType, - AggregatedQuantity: 0, // not count sidecar quantity - NumaAllocationResult: mainContainerAllocationInfo.NumaAllocationResult.Clone(), // pin sidecar to same cpuset.mems of the main container - TopologyAwareAllocations: nil, // not count sidecar quantity + AggregatedQuantity: 0, // not count sidecar quantity + TopologyAwareAllocations: nil, // not count sidecar quantity Labels: general.DeepCopyMap(req.Labels), Annotations: general.DeepCopyMap(req.Annotations), QoSLevel: qosLevel, } + applySidecarAllocationInfoFromMainContainer(allocationInfo, mainContainerAllocationInfo) // update pod entries directly. if one of subsequent steps is failed, // we will delete current allocationInfo from podEntries in defer function of allocation function. @@ -292,6 +315,12 @@ func (p *DynamicPolicy) allocateNUMAsWithoutNUMABindingPods(_ context.Context, req.PodNamespace, req.PodName, req.ContainerName, allocationInfo.NumaAllocationResult.String(), numaWithoutNUMABindingPods.String()) } + // use real container request size here + reqInt, _, err := util.GetQuantityFromResourceReq(req) + if err != nil { + return nil, fmt.Errorf("GetQuantityFromResourceReq failed with error: %v", err) + } + allocationInfo = &state.AllocationInfo{ PodUid: req.PodUid, PodNamespace: req.PodNamespace, @@ -305,12 +334,13 @@ func (p *DynamicPolicy) allocateNUMAsWithoutNUMABindingPods(_ context.Context, Labels: general.DeepCopyMap(req.Labels), Annotations: general.DeepCopyMap(req.Annotations), QoSLevel: qosLevel, + AggregatedQuantity: uint64(reqInt), } p.state.SetAllocationInfo(v1.ResourceMemory, allocationInfo.PodUid, allocationInfo.ContainerName, allocationInfo) podResourceEntries := p.state.GetPodResourceEntries() - machineState, err := state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) + machineState, err = state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) if err != nil { general.Errorf("pod: %s/%s, container: %s GenerateMachineStateFromPodEntries failed with error: %v", req.PodNamespace, req.PodName, req.ContainerName, err) @@ -432,8 +462,8 @@ func (p *DynamicPolicy) adjustAllocationEntries() error { } } - // we should remove it someday - // adjust container request for old VPA pods + // TODO: optimize this logic someday: + // only for refresh memory request for old inplace update resized pods. for podUID, containerEntries := range podEntries { for containerName, allocationInfo := range containerEntries { if allocationInfo == nil { @@ -529,7 +559,7 @@ func (p *DynamicPolicy) adjustAllocationEntries() error { // calculateMemoryAllocation will not store the allocation in states, instead, // it will update the passed by machineState in-place; so the function will be // called `calculateXXX` rather than `allocateXXX` -func (p *DynamicPolicy) calculateMemoryAllocation(req *pluginapi.ResourceRequest, machineState state.NUMANodeMap, qosLevel string) error { +func (p *DynamicPolicy) calculateMemoryAllocation(req *pluginapi.ResourceRequest, machineState state.NUMANodeMap, qosLevel string, podAggregatedRequest int) error { if req.Hint == nil { return fmt.Errorf("hint is nil") } else if len(req.Hint.Nodes) == 0 { @@ -540,28 +570,24 @@ func (p *DynamicPolicy) calculateMemoryAllocation(req *pluginapi.ResourceRequest return fmt.Errorf("NUMA not exclusive binding container has request larger than 1 NUMA") } - memoryReq, _, err := util.GetQuantityFromResourceReq(req) - if err != nil { - return fmt.Errorf("GetQuantityFromResourceReq failed with error: %v", err) - } - hintNumaNodes := machine.NewCPUSet(util.HintToIntArray(req.Hint)...) general.InfoS("allocate by hints", "podNamespace", req.PodNamespace, "podName", req.PodName, "containerName", req.ContainerName, "hints", hintNumaNodes.String(), - "reqMemoryQuantity", memoryReq) + "reqMemoryQuantity", podAggregatedRequest) var leftQuantity uint64 + var err error if qosutil.AnnotationsIndicateNUMAExclusive(req.Annotations) { - leftQuantity, err = calculateExclusiveMemory(req, machineState, hintNumaNodes.ToSliceInt(), uint64(memoryReq), qosLevel) + leftQuantity, err = calculateExclusiveMemory(req, machineState, hintNumaNodes.ToSliceInt(), uint64(podAggregatedRequest), qosLevel) if err != nil { return fmt.Errorf("calculateExclusiveMemory failed with error: %v", err) } } else { - leftQuantity, err = calculateMemoryInNumaNodes(req, machineState, hintNumaNodes.ToSliceInt(), uint64(memoryReq), qosLevel) + leftQuantity, err = calculateMemoryInNumaNodes(req, machineState, hintNumaNodes.ToSliceInt(), uint64(podAggregatedRequest), qosLevel) if err != nil { return fmt.Errorf("calculateMemoryInNumaNodes failed with error: %v", err) } @@ -569,7 +595,7 @@ func (p *DynamicPolicy) calculateMemoryAllocation(req *pluginapi.ResourceRequest if leftQuantity > 0 { general.Errorf("hint NUMA nodes: %s can't meet memory request: %d bytes, leftQuantity: %d bytes", - hintNumaNodes.String(), memoryReq, leftQuantity) + hintNumaNodes.String(), podAggregatedRequest, leftQuantity) return fmt.Errorf("results can't meet memory request") } diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go index 32a5f3278..8412a956a 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go @@ -27,6 +27,7 @@ import ( apiconsts "github.com/kubewharf/katalyst-api/pkg/consts" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/state" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" + "github.com/kubewharf/katalyst-core/pkg/metrics" "github.com/kubewharf/katalyst-core/pkg/util/general" "github.com/kubewharf/katalyst-core/pkg/util/machine" qosutil "github.com/kubewharf/katalyst-core/pkg/util/qos" @@ -39,19 +40,43 @@ func (p *DynamicPolicy) sharedCoresHintHandler(ctx context.Context, return nil, fmt.Errorf("got nil request") } - if !qosutil.AnnotationsIndicateNUMABinding(req.Annotations) { - return util.PackResourceHintsResponse(req, string(v1.ResourceMemory), - map[string]*pluginapi.ListOfTopologyHints{ - string(v1.ResourceMemory): nil, // indicates that there is no numa preference - }) + if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) { + return p.numaBindingHintHandler(ctx, req) + } + + // TODO: support sidecar follow main container for normal share cores in future + if req.ContainerType == pluginapi.ContainerType_MAIN { + ok, err := p.checkNormalShareCoresResource(req) + if err != nil { + general.Errorf("failed to check share cores resource: %q", err) + return nil, fmt.Errorf("failed to check share cores resource: %q", err) + } + + if !ok { + _ = p.emitter.StoreInt64(util.MetricNameShareCoresNoEnoughResourceFailed, 1, metrics.MetricTypeNameCount, metrics.ConvertMapToTags(map[string]string{ + "resource": v1.ResourceMemory.String(), + "podNamespace": req.PodNamespace, + "podName": req.PodName, + "containerName": req.ContainerName, + })...) + return nil, fmt.Errorf("no enough memory resource") + } } - return p.numaBindingHintHandler(ctx, req) + return util.PackResourceHintsResponse(req, string(v1.ResourceMemory), + map[string]*pluginapi.ListOfTopologyHints{ + string(v1.ResourceMemory): nil, // indicates that there is no numa preference + }) } func (p *DynamicPolicy) reclaimedCoresHintHandler(ctx context.Context, req *pluginapi.ResourceRequest, ) (*pluginapi.ResourceHintsResponse, error) { + if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to memory inplace update resize, but not support reclaimed cores", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("not support inplace update resize for reclaimed cores") + } return p.sharedCoresHintHandler(ctx, req) } @@ -62,6 +87,12 @@ func (p *DynamicPolicy) dedicatedCoresHintHandler(ctx context.Context, return nil, fmt.Errorf("dedicatedCoresHintHandler got nil req") } + if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to memory inplace update resize, but not support dedicated cores", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("not support inplace update resize for didecated cores") + } + switch req.Annotations[apiconsts.PodAnnotationMemoryEnhancementNumaBinding] { case apiconsts.PodAnnotationMemoryEnhancementNumaBindingEnable: return p.numaBindingHintHandler(ctx, req) @@ -82,7 +113,7 @@ func (p *DynamicPolicy) numaBindingHintHandler(_ context.Context, }) } - reqInt, _, err := util.GetQuantityFromResourceReq(req) + podAggregatedRequest, _, err := util.GetPodAggregatedRequestResource(req) if err != nil { return nil, fmt.Errorf("getReqQuantityFromResourceReq failed with error: %v", err) } @@ -92,26 +123,30 @@ func (p *DynamicPolicy) numaBindingHintHandler(_ context.Context, allocationInfo := p.state.GetAllocationInfo(v1.ResourceMemory, req.PodUid, req.ContainerName) if allocationInfo != nil { - hints = regenerateHints(uint64(reqInt), allocationInfo) - - // regenerateHints failed, and we need to clear container record and re-calculate. - if hints == nil { - podResourceEntries := p.state.GetPodResourceEntries() - for _, podEntries := range podResourceEntries { - delete(podEntries[req.PodUid], req.ContainerName) - if len(podEntries[req.PodUid]) == 0 { - delete(podEntries, req.PodUid) - } - } + if allocationInfo.NumaAllocationResult.Size() != 1 { + general.Errorf("pod: %s/%s, container: %s is share cores with numa binding, but its numa set length is %d", + req.PodNamespace, req.PodName, req.ContainerName, allocationInfo.NumaAllocationResult.Size()) + return nil, fmt.Errorf("invalid numa set size") + } + hints = regenerateHints(uint64(podAggregatedRequest), allocationInfo, util.PodInplaceUpdateResizing(req)) + // clear the current container and regenerate machine state in follow cases: + // 1. regenerateHints failed. + // 2. the container is inplace update resizing. + // hints it as a new container + if hints == nil || util.PodInplaceUpdateResizing(req) { var err error - resourcesMachineState, err = state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) + resourcesMachineState, err = p.clearContainerAndRegenerateMachineState(req) if err != nil { general.Errorf("pod: %s/%s, container: %s GenerateMachineStateFromPodEntries failed with error: %v", req.PodNamespace, req.PodName, req.ContainerName, err) return nil, fmt.Errorf("GenerateMachineStateFromPodEntries failed with error: %v", err) } } + } else if util.PodInplaceUpdateResizing(req) { + general.Errorf("pod: %s/%s, container: %s request to inplace update resize, but no origin allocation info", + req.PodNamespace, req.PodName, req.ContainerName) + return nil, fmt.Errorf("no origin allocation info") } // if hints exists in extra state-file, prefer to use them @@ -127,19 +162,83 @@ func (p *DynamicPolicy) numaBindingHintHandler(_ context.Context, } } - // otherwise, calculate hint for container without allocated memory - if hints == nil { + // check or regenerate hints for inplace update mode + if util.PodInplaceUpdateResizing(req) { + if allocationInfo.NumaAllocationResult.Size() != 1 { + general.Errorf("pod: %s/%s, container: %s is snb, but its numa size is %d", + req.PodNamespace, req.PodName, req.ContainerName, allocationInfo.NumaAllocationResult.Size()) + return nil, fmt.Errorf("invalid hints for inplace update pod") + } + + machineMemoryState := resourcesMachineState[v1.ResourceMemory] + nodeID := allocationInfo.NumaAllocationResult.ToSliceInt()[0] + nodeMemoryState := machineMemoryState[nodeID] + + // the main container aggregated quantity involve all container requests of the pod in memory admit. + originPodAggregatedRequest := allocationInfo.AggregatedQuantity + general.Infof("pod: %s/%s, main container: %s request to memory inplace update resize (%d->%d)", + req.PodNamespace, req.PodName, req.ContainerName, originPodAggregatedRequest, podAggregatedRequest) + + if uint64(podAggregatedRequest) > nodeMemoryState.Free { // no left resource to scale out + // TODO maybe support snb NUMA migrate inplace update resize later + isInplaceUpdateResizeNumaMigratable := false + if isInplaceUpdateResizeNumaMigratable { + general.Infof("pod: %s/%s, container: %s request to memory inplace update resize (%d->%d), but no enough memory"+ + "to scale out in current numa (resize request extra: %d, free: %d), migrate numa", + req.PodNamespace, req.PodName, req.ContainerName, originPodAggregatedRequest, podAggregatedRequest, uint64(podAggregatedRequest)-originPodAggregatedRequest, nodeMemoryState.Free) + + var calculateErr error + // recalculate hints for the whole pod + hints, calculateErr = p.calculateHints(uint64(podAggregatedRequest), resourcesMachineState, req.Annotations) + if calculateErr != nil { + general.Errorf("failed to calculate hints for pod: %s/%s, container: %s, error: %v", + req.PodNamespace, req.PodName, req.ContainerName, calculateErr) + return nil, fmt.Errorf("calculateHints failed with error: %v", calculateErr) + } + } else { + general.Infof("pod: %s/%s, container: %s request to memory inplace update resize (%d->%d, diff: %d), but no enough memory(%d)", + req.PodNamespace, req.PodName, req.ContainerName, originPodAggregatedRequest, podAggregatedRequest, uint64(podAggregatedRequest)-originPodAggregatedRequest, nodeMemoryState.Free) + return nil, fmt.Errorf("inplace update resize scale out failed with no enough resource") + } + } + } else if hints == nil { + // otherwise, calculate hint for container without allocated memory var calculateErr error // calculate hint for container without allocated memory - hints, calculateErr = p.calculateHints(uint64(reqInt), resourcesMachineState, req.Annotations) + hints, calculateErr = p.calculateHints(uint64(podAggregatedRequest), resourcesMachineState, req.Annotations) if calculateErr != nil { + general.Errorf("failed to calculate hints for pod: %s/%s, container: %s, error: %v", + req.PodNamespace, req.PodName, req.ContainerName, calculateErr) return nil, fmt.Errorf("calculateHints failed with error: %v", calculateErr) } } + general.Infof("memory hints for pod:%s/%s, container: %s success, hints: %v", + req.PodNamespace, req.PodName, req.ContainerName, hints) + return util.PackResourceHintsResponse(req, string(v1.ResourceMemory), hints) } +func (p *DynamicPolicy) clearContainerAndRegenerateMachineState(req *pluginapi.ResourceRequest) (state.NUMANodeResourcesMap, error) { + podResourceEntries := p.state.GetPodResourceEntries() + + for _, podEntries := range podResourceEntries { + delete(podEntries[req.PodUid], req.ContainerName) + if len(podEntries[req.PodUid]) == 0 { + delete(podEntries, req.PodUid) + } + } + + var err error + resourcesMachineState, err := state.GenerateMachineStateFromPodEntries(p.state.GetMachineInfo(), podResourceEntries, p.state.GetReservedMemory()) + if err != nil { + general.Errorf("pod: %s/%s, container: %s GenerateMachineStateFromPodEntries failed with error: %v", + req.PodNamespace, req.PodName, req.ContainerName, err) + return nil, fmt.Errorf("GenerateMachineStateFromPodEntries failed with error: %v", err) + } + return resourcesMachineState, nil +} + func (p *DynamicPolicy) dedicatedCoresWithoutNUMABindingHintHandler(_ context.Context, _ *pluginapi.ResourceRequest, ) (*pluginapi.ResourceHintsResponse, error) { @@ -262,11 +361,11 @@ func (p *DynamicPolicy) calculateHints(reqInt uint64, resourcesMachineState stat // regenerateHints regenerates hints for container that'd already been allocated memory, // and regenerateHints will assemble hints based on already-existed AllocationInfo, // without any calculation logics at all -func regenerateHints(reqInt uint64, allocationInfo *state.AllocationInfo) map[string]*pluginapi.ListOfTopologyHints { +func regenerateHints(reqInt uint64, allocationInfo *state.AllocationInfo, podInplaceUpdateResizing bool) map[string]*pluginapi.ListOfTopologyHints { hints := map[string]*pluginapi.ListOfTopologyHints{} allocatedInt := allocationInfo.AggregatedQuantity - if allocatedInt < reqInt { + if allocatedInt < reqInt && !podInplaceUpdateResizing { general.ErrorS(nil, "memory's already allocated with smaller quantity than requested", "podUID", allocationInfo.PodUid, "containerName", allocationInfo.ContainerName, "requestedResource", reqInt, "allocatedSize", allocatedInt) diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go index 314f82581..0bf54d069 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go @@ -514,7 +514,7 @@ func TestAllocate(t *testing.T) { OciPropertyName: util.OCIPropertyNameCPUSetMems, IsNodeResource: false, IsScalarResource: true, - AllocatedQuantity: 0, + AllocatedQuantity: 1073741824, AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), ResourceHints: &pluginapi.ListOfTopologyHints{ Hints: []*pluginapi.TopologyHint{nil}, @@ -1556,8 +1556,8 @@ func TestGetTopologyAwareResources(t *testing.T) { string(v1.ResourceMemory): { IsNodeResource: false, IsScalarResource: true, - AggregatedQuantity: 0, - OriginalAggregatedQuantity: 0, + AggregatedQuantity: 1073741824, + OriginalAggregatedQuantity: 1073741824, TopologyAwareQuantityList: nil, OriginalTopologyAwareQuantityList: nil, }, @@ -1736,7 +1736,7 @@ func TestGetResourcesAllocation(t *testing.T) { OciPropertyName: util.OCIPropertyNameCPUSetMems, IsNodeResource: false, IsScalarResource: true, - AllocatedQuantity: 0, + AllocatedQuantity: 1073741824, AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), }) @@ -3768,6 +3768,149 @@ func BenchmarkGetTopologyHints(b *testing.B) { } } +func TestSNBMemoryAdmitWithSidecarReallocate(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBMemoryVPAWithSidecar") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + cpuTopology, err := machine.GenerateDummyCPUTopology(4, 1, 1) + as.Nil(err) + + machineInfo, err := machine.GenerateDummyMachineInfo(1, 8) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + sidecarName := "sidecar" + + // allocate sidecar firstly + podUID := string(uuid.NewUUID()) + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no topology hints for snb sidecar + res, err := dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + // no allocation info for snb sidecar + allocationRes, err := dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(allocationRes.AllocationResult) + + // allocate main container + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + res, err = dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.NotNil(res) + as.NotNil(res.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(res.ResourceHints[string(v1.ResourceMemory)].Hints)) + + req.Hint = res.ResourceHints[string(v1.ResourceMemory)].Hints[0] + allocationRes, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + as.NotNil(allocationRes.AllocationResult) + + allocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + // allocate all resource into main container: sidecar 2G + main 2G + as.Equal(float64(4294967296), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // another pod admit + anotherReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: "pod1", + ContainerName: "container1", + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 7 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 7516192768}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + _, err = dynamicPolicy.GetTopologyHints(context.Background(), anotherReq) + as.NotNil(err) + + // reallocate sidecar here + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid]) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName]) + // allocate all resource into main container, here is zero + as.Equal(float64(0), allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) +} + func Test_getContainerRequestedMemoryBytes(t *testing.T) { t.Parallel() diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/util.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/util.go index 8987ea1d6..b388ff928 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/util.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/util.go @@ -25,6 +25,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/state" "github.com/kubewharf/katalyst-core/pkg/config" "github.com/kubewharf/katalyst-core/pkg/metaserver" "github.com/kubewharf/katalyst-core/pkg/util/general" @@ -94,3 +95,11 @@ func getReservedMemory(conf *config.Configuration, metaServer *metaserver.MetaSe } return reservedMemory, nil } + +func applySidecarAllocationInfoFromMainContainer(sidecarAllocationInfo, mainAllocationInfo *state.AllocationInfo) bool { + if !sidecarAllocationInfo.NumaAllocationResult.Equals(mainAllocationInfo.NumaAllocationResult) { + sidecarAllocationInfo.NumaAllocationResult = mainAllocationInfo.NumaAllocationResult.Clone() + return true + } + return false +} diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/vpa_test.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/vpa_test.go new file mode 100644 index 000000000..c0a1ddeac --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/vpa_test.go @@ -0,0 +1,911 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamicpolicy + +import ( + "context" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/uuid" + pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" + + "github.com/kubewharf/katalyst-api/pkg/consts" + "github.com/kubewharf/katalyst-core/pkg/util/machine" +) + +func TestSNBMemoryVPA(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBMemoryVPA") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + + podUID := string(uuid.NewUUID()) + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + res, err := dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.NotNil(res) + as.NotNil(res.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(res.ResourceHints[string(v1.ResourceMemory)].Hints)) + + req = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: res.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + allocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(2147483648), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + resizeReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // resize + resizeRes, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.Nil(err) + as.NotNil(resizeRes) + as.NotNil(resizeRes.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints)) + + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq) + as.Nil(err) + + resizeAllocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(resizeAllocation) + as.NotNil(resizeAllocation.PodResources) + as.NotNil(resizeAllocation.PodResources[req.PodUid]) + as.NotNil(resizeAllocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(resizeAllocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(4294967296), resizeAllocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize exceed (per numa 7G) + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 8 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.NotNil(err) +} + +func TestSNBMemoryVPAWithSidecar(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestSNBMemoryVPAWithSidecar") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + sidecarName := "sidecar" + + // allocate sidecar firstly + podUID := string(uuid.NewUUID()) + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no topology hints for snb sidecar + res, err := dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + // no allocation info for snb sidecar + sidecarReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + allocationRes, err := dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + as.Nil(allocationRes.AllocationResult) + + // allocate main container + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + res, err = dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.NotNil(res) + as.NotNil(res.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(res.ResourceHints[string(v1.ResourceMemory)].Hints)) + + req = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: res.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + allocationRes, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + as.NotNil(allocationRes.AllocationResult) + + allocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + // allocate all resource into main container: sidecar 2G + main 2G + as.Equal(float64(4294967296), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // reallocate sidecar here + sidecarReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 4294967296}`, // sidecar 2G + main 2G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid]) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName]) + // allocate all resource into main container, here is zero + as.Equal(float64(0), allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize main container + resizeReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 6442450944}`, // sidecar 2G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + resizeRes, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.Nil(err) + as.NotNil(resizeRes) + as.NotNil(resizeRes.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints)) + + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 6442450944}`, // sidecar 2G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq) + as.Nil(err) + + resizeAllocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(resizeAllocation) + as.NotNil(resizeAllocation.PodResources) + as.NotNil(resizeAllocation.PodResources[req.PodUid]) + as.NotNil(resizeAllocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(resizeAllocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + // allocate all resource into main container: sidecar 2G + main 4G + as.Equal(float64(6442450944), resizeAllocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName]) + // allocate all resource into main container, here is zero + as.Equal(float64(0), allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize sidecar container + resizeSidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 3 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 7516192768}`, // sidecar 3G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + resizeSidecarRes, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeSidecarReq) + as.Nil(err) + as.NotNil(resizeSidecarRes) + // no topology hints for snb sidecar + as.Nil(resizeSidecarRes.ResourceHints[string(v1.ResourceMemory)]) + + resizeSidecarReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 3 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 7516192768}`, // sidecar 3G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeSidecarReq) + as.Nil(err) + + // resize main container again (no change) + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 7516192768}`, // sidecar 2G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + resizeRes, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.Nil(err) + as.NotNil(resizeRes) + as.NotNil(resizeRes.ResourceHints[string(v1.ResourceMemory)]) + as.NotZero(len(resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints)) + + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 7516192768}`, // sidecar 2G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq) + as.Nil(err) + + resizeAllocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(resizeAllocation) + as.NotNil(resizeAllocation.PodResources) + as.NotNil(resizeAllocation.PodResources[resizeSidecarReq.PodUid]) + as.NotNil(resizeAllocation.PodResources[resizeSidecarReq.PodUid].ContainerResources) + as.NotNil(resizeAllocation.PodResources[resizeSidecarReq.PodUid].ContainerResources[testName]) + // allocate all resource into main container: sidecar 3G + main 4G + as.Equal(float64(7516192768), resizeAllocation.PodResources[resizeSidecarReq.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + as.NotNil(allocation.PodResources[resizeSidecarReq.PodUid].ContainerResources[resizeSidecarReq.ContainerName]) + // allocate all resource into main container, here is zero + as.Equal(float64(0), allocation.PodResources[resizeSidecarReq.PodUid].ContainerResources[resizeSidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize main exceeded + resizeReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 32 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 37580963840}`, // sidecar 2G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Hint: resizeRes.ResourceHints[string(v1.ResourceMemory)].Hints[0], + } + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.NotNil(err) + // resize sidecar exceeded + resizeSidecarReq = &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 32 * 1024 * 1024 * 1024, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + consts.PodAnnotationInplaceUpdateResizingKey: "true", + consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, + consts.PodAnnotationAggregatedRequestsKey: `{"memory": 37580963840}`, // sidecar 3G + main 4G + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeSidecarReq) + // TODO expected error here, but now not check sidecar + // it is not important to check sidecar in hints, because kubelet only patch pod spec when all container admit successfully + as.Nil(err) +} + +func TestNormalShareMemoryVPA(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestNormalShareMemoryVPA") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + + podUID := string(uuid.NewUUID()) + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + res, err := dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + allocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(2147483648), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize the container + resizeReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + resizeRes, err := dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.Nil(err) + as.NotNil(resizeRes) + as.Nil(resizeRes.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[resizeReq.PodUid]) + as.NotNil(allocation.PodResources[resizeReq.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[resizeReq.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(4294967296), allocation.PodResources[resizeReq.PodUid].ContainerResources[resizeReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resie exceeded + // no hints for normal share cores container + resizeReq.Annotations[consts.PodAnnotationAggregatedRequestsKey] = `{"memory": 35433480192}` + resizeReq.ResourceRequests[string(v1.ResourceMemory)] = 33 * 1024 * 1024 * 1024 + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeReq) + as.NotNil(err) +} + +func TestNormalShareMemoryVPAWithSidecar(t *testing.T) { + t.Parallel() + + as := require.New(t) + + tmpDir, err := ioutil.TempDir("", "checkpoint-TestNormalShareMemoryVPA") + as.Nil(err) + defer os.RemoveAll(tmpDir) + + cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) + as.Nil(err) + + machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) + as.Nil(err) + + dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) + as.Nil(err) + dynamicPolicy.podAnnotationKeptKeys = []string{ + consts.PodAnnotationMemoryEnhancementNumaBinding, + consts.PodAnnotationInplaceUpdateResizingKey, + consts.PodAnnotationAggregatedRequestsKey, + } + + testName := "test" + sidecarName := "sidecar" + // allocate main container + podUID := string(uuid.NewUUID()) + req := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + res, err := dynamicPolicy.GetTopologyHints(context.Background(), req) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), req) + as.Nil(err) + + allocation, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(2147483648), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // allocate sidecar container + sidecarReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 2147483648, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + res, err = dynamicPolicy.GetTopologyHints(context.Background(), sidecarReq) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), sidecarReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid]) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName]) + as.Equal(float64(2147483648), allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize main container + resizeMainReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: testName, + ContainerType: pluginapi.ContainerType_MAIN, + ContainerIndex: 0, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 4294967296, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + res, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeMainReq) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), resizeMainReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[req.PodUid]) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName]) + as.Equal(float64(4294967296), allocation.PodResources[req.PodUid].ContainerResources[req.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize sidecar container + sidecarResizeReq := &pluginapi.ResourceRequest{ + PodUid: podUID, + PodNamespace: testName, + PodName: testName, + ContainerName: sidecarName, + ContainerType: pluginapi.ContainerType_SIDECAR, + ContainerIndex: 1, + ResourceName: string(v1.ResourceMemory), + ResourceRequests: map[string]float64{ + string(v1.ResourceMemory): 3221225472, + }, + Annotations: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + Labels: map[string]string{ + consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, + }, + } + + // no hints for normal share cores container + res, err = dynamicPolicy.GetTopologyHints(context.Background(), sidecarResizeReq) + as.Nil(err) + as.NotNil(res) + as.Nil(res.ResourceHints[string(v1.ResourceMemory)]) + + _, err = dynamicPolicy.Allocate(context.Background(), sidecarResizeReq) + as.Nil(err) + + allocation, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) + as.Nil(err) + as.NotNil(allocation) + as.NotNil(allocation.PodResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid]) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources) + as.NotNil(allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName]) + as.Equal(float64(3221225472), allocation.PodResources[sidecarReq.PodUid].ContainerResources[sidecarReq.ContainerName].ResourceAllocation[string(v1.ResourceMemory)].AllocatedQuantity) + + // resize exceeded + resizeMainReq.ResourceRequests[string(v1.ResourceMemory)] = 35433480192 + _, err = dynamicPolicy.GetTopologyHints(context.Background(), resizeMainReq) + as.NotNil(err) +} diff --git a/pkg/agent/qrm-plugins/util/consts.go b/pkg/agent/qrm-plugins/util/consts.go index 5d56c3295..687cd0931 100644 --- a/pkg/agent/qrm-plugins/util/consts.go +++ b/pkg/agent/qrm-plugins/util/consts.go @@ -51,6 +51,9 @@ const ( MetricNameMemoryNumaBalance = "memory_handle_numa_balance" MetricNameMemoryNumaBalanceCost = "memory_numa_balance_cost" MetricNameMemoryNumaBalanceResult = "memory_numa_balance_result" + + // metrics for some cases + MetricNameShareCoresNoEnoughResourceFailed = "share_cores_no_enough_resource" ) // those are OCI property names to be used by QRM plugins diff --git a/pkg/agent/qrm-plugins/util/util.go b/pkg/agent/qrm-plugins/util/util.go index 6b3de1666..d7bd1348e 100644 --- a/pkg/agent/qrm-plugins/util/util.go +++ b/pkg/agent/qrm-plugins/util/util.go @@ -326,3 +326,33 @@ func GetCgroupAsyncWorkName(cgroup, topic string) string { func GetAsyncWorkNameByPrefix(prefix, topic string) string { return strings.Join([]string{prefix, topic}, asyncworker.WorkNameSeperator) } + +func PodInplaceUpdateResizing(req *pluginapi.ResourceRequest) bool { + return req.Annotations != nil && req.Annotations[apiconsts.PodAnnotationInplaceUpdateResizingKey] == "true" +} + +func GetPodAggregatedRequestResource(req *pluginapi.ResourceRequest) (int, float64, error) { + annotations := req.Annotations + if annotations == nil { + return GetQuantityFromResourceReq(req) + } + value, ok := annotations[apiconsts.PodAnnotationAggregatedRequestsKey] + if !ok { + return GetQuantityFromResourceReq(req) + } + var resourceList v1.ResourceList + if err := json.Unmarshal([]byte(value), &resourceList); err != nil { + return GetQuantityFromResourceReq(req) + } + + switch req.ResourceName { + case string(v1.ResourceCPU): + podAggregatedReqFloat64 := float64(resourceList.Cpu().MilliValue()) / 1000 + return int(math.Ceil(podAggregatedReqFloat64)), podAggregatedReqFloat64, nil + case string(v1.ResourceMemory): + podAggregatedReqFloat64 := float64(resourceList.Memory().MilliValue()) / 1000 + return int(math.Ceil(podAggregatedReqFloat64)), podAggregatedReqFloat64, nil + default: + return 0, 0, fmt.Errorf("not support resource name: %s", req.ResourceName) + } +} diff --git a/pkg/config/agent/qrm/qrm_base.go b/pkg/config/agent/qrm/qrm_base.go index 84e3bfa64..14387bc33 100644 --- a/pkg/config/agent/qrm/qrm_base.go +++ b/pkg/config/agent/qrm/qrm_base.go @@ -16,6 +16,8 @@ limitations under the License. package qrm +import "github.com/kubewharf/katalyst-api/pkg/consts" + type GenericQRMPluginConfiguration struct { StateFileDirectory string QRMPluginSocketDirs []string @@ -37,8 +39,11 @@ type QRMPluginsConfiguration struct { func NewGenericQRMPluginConfiguration() *GenericQRMPluginConfiguration { return &GenericQRMPluginConfiguration{ - PodAnnotationKeptKeys: []string{}, - PodLabelKeptKeys: []string{}, + PodAnnotationKeptKeys: []string{ + consts.PodAnnotationAggregatedRequestsKey, + consts.PodAnnotationInplaceUpdateResizingKey, + }, + PodLabelKeptKeys: []string{}, } }