Merge pull request #693 from yadzhang/feat/tmo-optimize-rebase-release

feat(tmo): block config support kcc and optimize target size calc
kubewharf · Sep 11, 2024 · f975841 · f975841
2 parents 47cc1fc + 8170d9d
commit f975841
Show file tree

Hide file tree

Showing 7 changed files with 384 additions and 28 deletions.
diff --git a/go.mod b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/h2non/gock v1.2.0
 	github.com/klauspost/cpuid/v2 v2.2.6
-	github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078
+	github.com/kubewharf/katalyst-api v0.5.1-0.20240911051124-d5c09c115a19
 	github.com/montanaflynn/stats v0.7.1
 	github.com/opencontainers/runc v1.1.6
 	github.com/opencontainers/selinux v1.10.0

diff --git a/go.sum b/go.sum
@@ -568,8 +568,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078 h1:CSBXQOe0AzlWcGaww8uqOUDu+/4bL3hVNBz86oziOis=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240820031712-7c1239991078/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
+github.com/kubewharf/katalyst-api v0.5.1-0.20240911051124-d5c09c115a19 h1:5kz2u+tLNzW94ByL9yfsfkvE4N8n/r6C1TKDiFWK8rw=
+github.com/kubewharf/katalyst-api v0.5.1-0.20240911051124-d5c09c115a19/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
 github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=

diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/advisor_test.go
@@ -816,6 +816,11 @@ func TestUpdate(t *testing.T) {
 						0: machine.MustParse("1"),
 						1: machine.MustParse("25"),
 					}, 200<<30),
+				makeContainerInfo("uid4", "default", "pod4", "c4", consts.PodAnnotationQoSLevelReclaimedCores, nil,
+					map[int]machine.CPUSet{
+						0: machine.MustParse("1"),
+						1: machine.MustParse("25"),
+					}, 200<<30),
 			},
 			pods: []*v1.Pod{
 				{
@@ -874,6 +879,62 @@ func TestUpdate(t *testing.T) {
 						},
 					},
 				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "pod3",
+						Namespace: "default",
+						UID:       "uid3",
+						Annotations: map[string]string{
+							consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores,
+						},
+						Labels: map[string]string{
+							consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores,
+						},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name: "c3",
+							},
+						},
+					},
+					Status: v1.PodStatus{
+						ContainerStatuses: []v1.ContainerStatus{
+							{
+								Name:        "c3",
+								ContainerID: "containerd://c3",
+							},
+						},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "pod4",
+						Namespace: "default",
+						UID:       "uid4",
+						Annotations: map[string]string{
+							consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores,
+						},
+						Labels: map[string]string{
+							consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores,
+						},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name: "c4",
+							},
+						},
+					},
+					Status: v1.PodStatus{
+						ContainerStatuses: []v1.ContainerStatus{
+							{
+								Name:        "c4",
+								ContainerID: "containerd://c4",
+							},
+						},
+					},
+				},
 			},
 			wantHeadroom: *resource.NewQuantity(996<<30, resource.DecimalSI),
 			nodeMetrics:  defaultNodeMetrics,
@@ -891,6 +952,18 @@ func TestUpdate(t *testing.T) {
 					podUID:        "uid1",
 					containerName: "c1",
 				},
+				{
+					metricName:    coreconsts.MetricMemCacheContainer,
+					metricValue:   metricutil.MetricData{Value: 5 << 30},
+					podUID:        "uid1",
+					containerName: "c1",
+				},
+				{
+					metricName:    coreconsts.MetricMemMappedContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid1",
+					containerName: "c1",
+				},
 				{
 					metricName:    coreconsts.MetricMemInactiveAnonContainer,
 					metricValue:   metricutil.MetricData{Value: 1 << 30},
@@ -939,9 +1012,21 @@ func TestUpdate(t *testing.T) {
 					podUID:        "uid2",
 					containerName: "c2",
 				},
+				{
+					metricName:    coreconsts.MetricMemCacheContainer,
+					metricValue:   metricutil.MetricData{Value: 3 << 30},
+					podUID:        "uid2",
+					containerName: "c2",
+				},
+				{
+					metricName:    coreconsts.MetricMemMappedContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid2",
+					containerName: "c2",
+				},
 				{
 					metricName:    coreconsts.MetricMemInactiveAnonContainer,
-					metricValue:   metricutil.MetricData{Value: 1 << 30},
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
 					podUID:        "uid2",
 					containerName: "c2",
 				},
@@ -975,6 +1060,126 @@ func TestUpdate(t *testing.T) {
 					podUID:        "uid2",
 					containerName: "c2",
 				},
+				{
+					metricName:    coreconsts.MetricMemPsiAvg60Container,
+					metricValue:   metricutil.MetricData{Value: 0.01},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemUsageContainer,
+					metricValue:   metricutil.MetricData{Value: 10 << 30},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemCacheContainer,
+					metricValue:   metricutil.MetricData{Value: 3 << 30},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemMappedContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemInactiveAnonContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemInactiveFileContainer,
+					metricValue:   metricutil.MetricData{Value: 1 << 30},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemPgscanContainer,
+					metricValue:   metricutil.MetricData{Value: 15000},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemPgstealContainer,
+					metricValue:   metricutil.MetricData{Value: 10000},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemWorkingsetRefaultContainer,
+					metricValue:   metricutil.MetricData{Value: 1000},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemWorkingsetActivateContainer,
+					metricValue:   metricutil.MetricData{Value: 1000},
+					podUID:        "uid3",
+					containerName: "c3",
+				},
+				{
+					metricName:    coreconsts.MetricMemPsiAvg60Container,
+					metricValue:   metricutil.MetricData{Value: 0.01},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemUsageContainer,
+					metricValue:   metricutil.MetricData{Value: 10 << 30},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemCacheContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemMappedContainer,
+					metricValue:   metricutil.MetricData{Value: 2 << 30},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemInactiveAnonContainer,
+					metricValue:   metricutil.MetricData{Value: 1 << 30},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemInactiveFileContainer,
+					metricValue:   metricutil.MetricData{Value: 1 << 30},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemPgscanContainer,
+					metricValue:   metricutil.MetricData{Value: 15000},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemPgstealContainer,
+					metricValue:   metricutil.MetricData{Value: 10000},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemWorkingsetRefaultContainer,
+					metricValue:   metricutil.MetricData{Value: 1000},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
+				{
+					metricName:    coreconsts.MetricMemWorkingsetActivateContainer,
+					metricValue:   metricutil.MetricData{Value: 1000},
+					podUID:        "uid4",
+					containerName: "c4",
+				},
 			},
 			cgroupMetrics: []cgroupMetric{
 				{
@@ -1007,6 +1212,16 @@ func TestUpdate(t *testing.T) {
 					metricValue: metricutil.MetricData{Value: 4 << 30},
 					cgroupPath:  "/hdfs",
 				},
+				{
+					metricName:  coreconsts.MetricMemCacheCgroup,
+					metricValue: metricutil.MetricData{Value: 3 << 30},
+					cgroupPath:  "/hdfs",
+				},
+				{
+					metricName:  coreconsts.MetricMemMappedCgroup,
+					metricValue: metricutil.MetricData{Value: 1 << 30},
+					cgroupPath:  "/hdfs",
+				},
 				{
 					metricName:  coreconsts.MetricMemInactiveAnonCgroup,
 					metricValue: metricutil.MetricData{Value: 1 << 30},
@@ -1024,19 +1239,37 @@ func TestUpdate(t *testing.T) {
 					{
 						CgroupPath: "/hdfs",
 						Values: map[string]string{
-							string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobON,
+							string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobOFF,
 							string(memoryadvisor.ControlKnowKeyMemoryOffloading): "38654705",
 						},
 					},
 				},
-				ContainerEntries: []types.ContainerMemoryAdvices{{
-					PodUID:        "uid1",
-					ContainerName: "c1",
-					Values: map[string]string{
-						string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobON,
-						string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764",
+				ContainerEntries: []types.ContainerMemoryAdvices{
+					{
+						PodUID:        "uid1",
+						ContainerName: "c1",
+						Values: map[string]string{
+							string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobOFF,
+							string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764",
+						},
 					},
-				}},
+					{
+						PodUID:        "uid3",
+						ContainerName: "c3",
+						Values: map[string]string{
+							string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobON,
+							string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764",
+						},
+					},
+					//{
+					//	PodUID:        "uid4",
+					//	ContainerName: "c4",
+					//	Values: map[string]string{
+					//		string(memoryadvisor.ControlKnobKeySwapMax):          coreconsts.ControlKnobOFF,
+					//		string(memoryadvisor.ControlKnowKeyMemoryOffloading): "96636764",
+					//	},
+					//},
+				},
 			},
 		},
 		{
@@ -2226,12 +2459,15 @@ func TestUpdate(t *testing.T) {
 			transparentMemoryOffloadingConfiguration := tmo.NewTransparentMemoryOffloadingConfiguration()
 			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations)
 			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableTMO = true
-			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableSwap = true
+			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelReclaimedCores].EnableSwap = false
+			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations)
+			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores].EnableTMO = true
+			transparentMemoryOffloadingConfiguration.QoSLevelConfigs[consts.QoSLevelSharedCores].EnableSwap = true
 
 			// cgroup level
 			transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"] = tmo.NewTMOConfigDetail(transparentMemoryOffloadingConfiguration.DefaultConfigurations)
 			transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableTMO = true
-			transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableSwap = true
+			transparentMemoryOffloadingConfiguration.CgroupConfigs["/sys/fs/cgroup/hdfs"].EnableSwap = false
 
 			advisor.conf.GetDynamicConfiguration().TransparentMemoryOffloadingConfiguration = transparentMemoryOffloadingConfiguration
 			_, advisorRecvChInterface := advisor.GetChannels()
@@ -2256,7 +2492,7 @@ func TestUpdate(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			advisor.Run(ctx)
 
-			time.Sleep(10 * time.Millisecond) // Wait some time because no signal will be sent to channel
+			time.Sleep(100 * time.Millisecond) // Wait some time because no signal will be sent to channel
 			if tt.needRecvAdvices {
 				result := <-recvCh