Mem percent watermark

examples/ensurance/evict-on-cpu-usage-percent/elastic-pod-qos.yaml

@@ -5,9 +5,6 @@ metadata:
 spec:
   allowedActions:
     - eviction
-  resourceQOS:
-    cpuQOS:
-      cpuPriority: 7
   labelSelector:
     matchLabels:
       preemptible_job: "true"

examples/ensurance/evict-on-cpu-usage-percent/pod.yaml

@@ -8,12 +8,12 @@ metadata:
 spec:
   containers:
     - command:
-        - stress-nga
+        - stress-ng
         - -c
         - "2"
         - --cpu-method
         - cpuid
-      image: ccr.ccs.tencentyun.com/tkeimages/stress-ng:v0.12.09.ln
+      image: docker.io/gocrane/stress-ng:v0.12.09
       imagePullPolicy: IfNotPresent
       name: low
       resources:

examples/ensurance/evict-on-cpu-usage-total/be-rules.yaml

@@ -5,11 +5,6 @@ metadata:
 spec:
   allowedActions:
     - eviction
-  resourceQOS:
-    cpuQOS:
-      cpuPriority: 7
-      htIsolation:
-        enable: false
   scopeSelector:
     matchExpressions:
       - operator: In

examples/ensurance/evict-on-mem-usage-percent/elastic-pod-qos.yaml

@@ -0,0 +1,10 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: PodQOS
+metadata:
+  name: all-elastic-pods
+spec:
+  allowedActions:
+    - eviction
+  labelSelector:
+    matchLabels:
+      preemptible_job: "true"

examples/ensurance/evict-on-mem-usage-percent/eviction-action.yaml

@@ -0,0 +1,9 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: AvoidanceAction
+metadata:
+  name: eviction
+spec:
+  coolDownSeconds: 300
+  description: evict low priority pods
+  eviction:
+    terminationGracePeriodSeconds: 30

examples/ensurance/evict-on-mem-usage-percent/pod.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: low-pi1
+  labels:
+    k8s-app: low
+    preemptible_job: "true"
+spec:
+  containers:
+    - image: docker.io/gocrane/stress-ng:v0.12.09
+      command:
+        - stress-ng
+        - --vm-hang
+        - "3600"
+        - --vm
+        - "2"
+        - --vm-bytes
+        - "2G"
+      name: stress
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: low-pi2
+  labels:
+    k8s-app: low
+    preemptible_job: "true"
+spec:
+  containers:
+    - image: docker.io/gocrane/stress-ng:v0.12.09
+      command:
+        - stress-ng
+        - --vm-hang
+        - "3600"
+        - --vm
+        - "2"
+        - --vm-bytes
+        - "3.5G"
+      name: stress
+

examples/ensurance/evict-on-mem-usage-percent/watermark.yaml

@@ -0,0 +1,18 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: NodeQOS
+metadata:
+  name: eviction-on-high-mem-usage-percent
+spec:
+  nodeQualityProbe:
+    nodeLocalGet:
+      localCacheTTLSeconds: 60
+    timeoutSeconds: 10
+  rules:
+    - actionName: eviction
+      avoidanceThreshold: 2
+      metricRule:
+        name: memory_total_utilization
+        value: 50
+      name: mem-usage-percent
+      restoreThreshold: 2
+      strategy: None

examples/ensurance/evict-on-mem-usage-total/elastic-pod-qos.yaml

@@ -0,0 +1,10 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: PodQOS
+metadata:
+  name: all-elastic-pods
+spec:
+  allowedActions:
+    - eviction
+  labelSelector:
+    matchLabels:
+      preemptible_job: "true"

examples/ensurance/evict-on-mem-usage-total/eviction-action.yaml

@@ -0,0 +1,9 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: AvoidanceAction
+metadata:
+  name: eviction
+spec:
+  coolDownSeconds: 300
+  description: evict low priority pods
+  eviction:
+    terminationGracePeriodSeconds: 30

examples/ensurance/evict-on-mem-usage-total/pod.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: low-pi1
+  labels:
+    k8s-app: low
+    preemptible_job: "true"
+spec:
+  containers:
+    - image: docker.io/gocrane/stress-ng:v0.12.09
+      command:
+        - stress-ng
+        - --vm-hang
+        - "3600"
+        - --vm
+        - "2"
+        - --vm-bytes
+        - "2G"
+      name: stress
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: low-pi2
+  labels:
+    k8s-app: low
+    preemptible_job: "true"
+spec:
+  containers:
+    - image: docker.io/gocrane/stress-ng:v0.12.09
+      command:
+        - stress-ng
+        - --vm-hang
+        - "3600"
+        - --vm
+        - "2"
+        - --vm-bytes
+        - "3.5G"
+      name: stress
+

examples/ensurance/evict-on-mem-usage-total/watermark.yaml

@@ -0,0 +1,18 @@
+apiVersion: ensurance.crane.io/v1alpha1
+kind: NodeQOS
+metadata:
+  name: eviction-on-high-mem-usage
+spec:
+  nodeQualityProbe:
+    nodeLocalGet:
+      localCacheTTLSeconds: 60
+    timeoutSeconds: 10
+  rules:
+    - actionName: eviction
+      avoidanceThreshold: 2
+      metricRule:
+        name: memory_total_usage
+        value: 5000000000 # means 5Gi
+      name: mem-usage
+      restoreThreshold: 2
+      strategy: None

pkg/ensurance/collector/cadvisor/cadvisor_linux.go

@@ -159,11 +159,13 @@ func (c *CadvisorCollector) Collect() (map[string][]common.TimeSeries, error) {
 			}
 
 			if hasExtMemRes {
-				extResMemUse += float64(v.Stats[0].Memory.Usage)
+				extResMemUse += float64(v.Stats[0].Memory.WorkingSet)
 			}
 
-			var containerLabels = GetContainerLabels(pod, containerId, containerName, hasExtCpuRes)
-			addSampleToStateMap(types.MetricNameContainerMemTotalUsage, composeSample(containerLabels, float64(v.Stats[0].Memory.Usage), now), stateMap)
+			hasExtRes := hasExtCpuRes || hasExtMemRes
+			var containerLabels = GetContainerLabels(pod, containerId, containerName, hasExtRes)
+			addSampleToStateMap(types.MetricNameContainerMemTotalUsage, composeSample(containerLabels, float64(v.Stats[0].Memory.WorkingSet), now), stateMap)
+			klog.V(6).Infof("Pod: %s, containerName: %s, key %s, container_mem_total_usage %#v", klog.KObj(pod), containerName, key, float64(v.Stats[0].Memory.WorkingSet))
 
 			if state, ok := c.latestContainersStates[key]; ok {
 				klog.V(6).Infof("For key %s, LatestContainersStates exist", key)

pkg/ensurance/collector/nodelocal/memory.go

@@ -39,6 +39,7 @@ func collectMemory(_ *nodeLocalContext) (map[string][]common.TimeSeries, error)
 	var data = make(map[string][]common.TimeSeries, 2)
 	data[string(types.MetricNameMemoryTotalUsage)] = []common.TimeSeries{{Samples: []common.Sample{{Value: float64(usage), Timestamp: now.Unix()}}}}
 	data[string(types.MetricNameMemoryTotalUtilization)] = []common.TimeSeries{{Samples: []common.Sample{{Value: usagePercent, Timestamp: now.Unix()}}}}
+	data[string(types.MetricNameMemoryTotal)] = []common.TimeSeries{{Samples: []common.Sample{{Value: float64(stat.Total), Timestamp: now.Unix()}}}}
 
 	return data, nil
 }

pkg/ensurance/collector/types/types.go

@@ -25,6 +25,7 @@ const (
 
 	MetricNameMemoryTotalUsage       MetricName = "memory_total_usage"
 	MetricNameMemoryTotalUtilization MetricName = "memory_total_utilization"
+	MetricNameMemoryTotal            MetricName = "memory_total"
 
 	MetricDiskReadKiBPS   MetricName = "disk_read_kibps"
 	MetricDiskWriteKiBPS  MetricName = "disk_write_kibps"

pkg/ensurance/executor/mem_usage.go

@@ -0,0 +1,70 @@
+package executor
+
+import (
+	"sync"
+
+	"k8s.io/klog/v2"
+
+	"github.com/gocrane/crane/pkg/ensurance/executor/podinfo"
+	"github.com/gocrane/crane/pkg/ensurance/executor/sort"
+	"github.com/gocrane/crane/pkg/metrics"
+	"github.com/gocrane/crane/pkg/utils"
+)
+
+func init() {
+	registerMetricMap(memUsage)
+}
+
+var memUsage = metric{
+	Name:           MemUsage,
+	ActionPriority: 5,
+	Sortable:       true,
+	SortFunc:       sort.MemUsageSort,
+
+	Throttleable:       false,
+	ThrottleQuantified: false,
+	ThrottleFunc:       nil,
+	RestoreFunc:        nil,
+
+	Evictable:       true,
+	EvictQuantified: true,
+	EvictFunc:       memUsageEvictPod,
+}
+
+func memUsageEvictPod(wg *sync.WaitGroup, ctx *ExecuteContext, index int, totalReleasedResource *ReleaseResource, EvictPods EvictPods) (errPodKeys []string, released ReleaseResource) {
+	wg.Add(1)
+
+	// Calculate release resources
+	released = releaseMemUsage(EvictPods[index])
+	totalReleasedResource.Add(released)
+
+	go func(evictPod podinfo.PodContext) {
+		defer wg.Done()
+
+		pod, err := ctx.PodLister.Pods(evictPod.Key.Namespace).Get(evictPod.Key.Name)
+		if err != nil {
+			errPodKeys = append(errPodKeys, "not found ", evictPod.Key.String())
+			return
+		}
+		klog.Warningf("Evicting pod %v", evictPod.Key)
+		err = utils.EvictPodWithGracePeriod(ctx.Client, pod, evictPod.DeletionGracePeriodSeconds)
+		if err != nil {
+			errPodKeys = append(errPodKeys, "evict failed ", evictPod.Key.String())
+			klog.Warningf("Failed to evict pod %s: %v", evictPod.Key.String(), err)
+			return
+		}
+		metrics.ExecutorEvictCountsInc()
+
+		klog.Warningf("Pod %s is evicted", klog.KObj(pod))
+	}(EvictPods[index])
+	return
+}
+
+func releaseMemUsage(pod podinfo.PodContext) ReleaseResource {
+	if pod.ActionType == podinfo.Evict {
+		return ReleaseResource{
+			MemUsage: pod.PodMemUsage,
+		}
+	}
+	return ReleaseResource{}
+}

pkg/ensurance/executor/mem_usage_percent.go

@@ -0,0 +1,70 @@
+package executor
+
+import (
+	"sync"
+
+	"k8s.io/klog/v2"
+
+	"github.com/gocrane/crane/pkg/ensurance/executor/podinfo"
+	"github.com/gocrane/crane/pkg/ensurance/executor/sort"
+	"github.com/gocrane/crane/pkg/metrics"
+	"github.com/gocrane/crane/pkg/utils"
+)
+
+func init() {
+	registerMetricMap(memUsagePercent)
+}
+
+var memUsagePercent = metric{
+	Name:           MemUsagePercent,
+	ActionPriority: 5,
+	Sortable:       true,
+	SortFunc:       sort.MemUsageSort,
+
+	Throttleable:       false,
+	ThrottleQuantified: false,
+	ThrottleFunc:       nil,
+	RestoreFunc:        nil,
+
+	Evictable:       true,
+	EvictQuantified: true,
+	EvictFunc:       memUsagePercentEvictPod,
+}
+
+func memUsagePercentEvictPod(wg *sync.WaitGroup, ctx *ExecuteContext, index int, totalReleasedResource *ReleaseResource, EvictPods EvictPods) (errPodKeys []string, released ReleaseResource) {
+	wg.Add(1)
+
+	// Calculate release resources
+	released = releaseMemUsagePercent(EvictPods[index])
+	totalReleasedResource.Add(released)
+
+	go func(evictPod podinfo.PodContext) {
+		defer wg.Done()
+
+		pod, err := ctx.PodLister.Pods(evictPod.Key.Namespace).Get(evictPod.Key.Name)
+		if err != nil {
+			errPodKeys = append(errPodKeys, "not found ", evictPod.Key.String())
+			return
+		}
+		klog.Warningf("Evicting pod %v", evictPod.Key)
+		err = utils.EvictPodWithGracePeriod(ctx.Client, pod, evictPod.DeletionGracePeriodSeconds)
+		if err != nil {
+			errPodKeys = append(errPodKeys, "evict failed ", evictPod.Key.String())
+			klog.Warningf("Failed to evict pod %s: %v", evictPod.Key.String(), err)
+			return
+		}
+		metrics.ExecutorEvictCountsInc()
+
+		klog.Warningf("Pod %s is evicted", klog.KObj(pod))
+	}(EvictPods[index])
+	return
+}
+
+func releaseMemUsagePercent(pod podinfo.PodContext) ReleaseResource {
+	if pod.ActionType == podinfo.Evict {
+		return ReleaseResource{
+			MemUsagePercent: pod.PodMemUsage,
+		}
+	}
+	return ReleaseResource{}
+}