add para execute-excess

cmd/crane-agent/app/agent.go

@@ -100,7 +100,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 	tspInformer := craneInformerFactory.Prediction().V1alpha1().TimeSeriesPredictions()
 
 	newAgent, err := agent.NewAgent(ctx, hostname, opts.RuntimeEndpoint, opts.CgroupDriver, kubeClient, craneClient, podInformer, nodeInformer,
-		nepInformer, actionInformer, tspInformer, opts.NodeResourceReserved, opts.Ifaces, healthCheck, opts.CollectInterval)
+		nepInformer, actionInformer, tspInformer, opts.NodeResourceReserved, opts.Ifaces, healthCheck, opts.CollectInterval, opts.ExecuteExcess)
 
 	if err != nil {
 		return err

cmd/crane-agent/app/options/option.go

@@ -26,6 +26,8 @@ type Options struct {
 	// Ifaces is the network devices to collect metric
 	Ifaces               []string
 	NodeResourceReserved map[string]string
+	// ExecuteExcess is the percentage of executions that exceed the gap between current usage and waterlines
+	ExecuteExcess string
 }
 
 // NewOptions builds an empty options.
@@ -54,4 +56,5 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.StringArrayVar(&o.Ifaces, "ifaces", []string{"eth0"}, "The network devices to collect metric, use comma to separated, default: eth0")
 	flags.Var(cliflag.NewMapStringString(&o.NodeResourceReserved), "node-resource-reserved", "A set of ResourceName=Percent (e.g. cpu=40%,memory=40%)")
 	flags.DurationVar(&o.MaxInactivity, "max-inactivity", 5*time.Minute, "Maximum time from last recorded activity before automatic restart, default: 5min")
+	flags.StringVar(&o.ExecuteExcess, "execute-excess", "10%", "The percentage of executions that exceed the gap between current usage and waterlines, default: 10%.")
 }

pkg/agent/agent.go

@@ -65,6 +65,7 @@ func NewAgent(ctx context.Context,
 	ifaces []string,
 	healthCheck *metrics.HealthCheck,
 	CollectInterval time.Duration,
+	executeExcess string,
 ) (*Agent, error) {
 	var managers []manager.Manager
 	var noticeCh = make(chan executor.AvoidanceExecutor)
@@ -88,7 +89,7 @@ func NewAgent(ctx context.Context,
 	managers = appendManagerIfNotNil(managers, stateCollector)
 	analyzerManager := analyzer.NewAnormalyAnalyzer(kubeClient, nodeName, podInformer, nodeInformer, nepInformer, actionInformer, stateCollector.AnalyzerChann, noticeCh)
 	managers = appendManagerIfNotNil(managers, analyzerManager)
-	avoidanceManager := executor.NewActionExecutor(kubeClient, nodeName, podInformer, nodeInformer, noticeCh, runtimeEndpoint, stateCollector.GetStateFunc())
+	avoidanceManager := executor.NewActionExecutor(kubeClient, nodeName, podInformer, nodeInformer, noticeCh, runtimeEndpoint, stateCollector.GetStateFunc(), executeExcess)
 	managers = appendManagerIfNotNil(managers, avoidanceManager)
 
 	if nodeResource := utilfeature.DefaultFeatureGate.Enabled(features.CraneNodeResource); nodeResource {

pkg/ensurance/executor/evict.go

@@ -69,7 +69,7 @@ func (e *EvictExecutor) Avoid(ctx *ExecuteContext) error {
 			errPodKeys = e.evictPods(ctx, &totalReleased, highestPrioriyMetric)
 		}
 	} else {
-		_, _, ctx.EvictGapToWaterLines = buildGapToWaterLine(ctx.getStateFunc(), ThrottleExecutor{}, *e)
+		_, _, ctx.EvictGapToWaterLines = buildGapToWaterLine(ctx.getStateFunc(), ThrottleExecutor{}, *e, ctx.executeExcessPercent)
 
 		if ctx.EvictGapToWaterLines.HasUsageMissedMetric() {
 			highestPrioriyMetric := e.EvictWaterLine.GetHighestPriorityEvictAbleMetric()

pkg/ensurance/executor/executor.go

@@ -16,6 +16,7 @@ import (
 	cruntime "github.com/gocrane/crane/pkg/ensurance/runtime"
 	"github.com/gocrane/crane/pkg/known"
 	"github.com/gocrane/crane/pkg/metrics"
+	"github.com/gocrane/crane/pkg/utils"
 )
 
 type ActionExecutor struct {
@@ -32,29 +33,38 @@ type ActionExecutor struct {
 	runtimeConn   *grpc.ClientConn
 
 	getStateFunc func() map[string][]common.TimeSeries
+
+	executeExcessPercent float64
 }
 
 // NewActionExecutor create enforcer manager
 func NewActionExecutor(client clientset.Interface, nodeName string, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer,
-	noticeCh <-chan AvoidanceExecutor, runtimeEndpoint string, getStateFunc func() map[string][]common.TimeSeries) *ActionExecutor {
+	noticeCh <-chan AvoidanceExecutor, runtimeEndpoint string, getStateFunc func() map[string][]common.TimeSeries, executeExcess string) *ActionExecutor {
 
 	runtimeClient, runtimeConn, err := cruntime.GetRuntimeClient(runtimeEndpoint)
 	if err != nil {
 		klog.Errorf("GetRuntimeClient failed %s", err.Error())
 		return nil
 	}
 
+	executeExcessPercent, err := utils.ParsePercentage(executeExcess)
+	if err != nil {
+		klog.Errorf("Parse executeExcess failed %s", err.Error())
+		return nil
+	}
+
 	return &ActionExecutor{
-		nodeName:      nodeName,
-		client:        client,
-		noticeCh:      noticeCh,
-		podLister:     podInformer.Lister(),
-		podSynced:     podInformer.Informer().HasSynced,
-		nodeLister:    nodeInformer.Lister(),
-		nodeSynced:    nodeInformer.Informer().HasSynced,
-		runtimeClient: runtimeClient,
-		runtimeConn:   runtimeConn,
-		getStateFunc:  getStateFunc,
+		nodeName:             nodeName,
+		client:               client,
+		noticeCh:             noticeCh,
+		podLister:            podInformer.Lister(),
+		podSynced:            podInformer.Informer().HasSynced,
+		nodeLister:           nodeInformer.Lister(),
+		nodeSynced:           nodeInformer.Informer().HasSynced,
+		runtimeClient:        runtimeClient,
+		runtimeConn:          runtimeConn,
+		getStateFunc:         getStateFunc,
+		executeExcessPercent: executeExcessPercent,
 	}
 }
 
@@ -101,13 +111,14 @@ func (a *ActionExecutor) Run(stop <-chan struct{}) {
 
 func (a *ActionExecutor) execute(ae AvoidanceExecutor, _ <-chan struct{}) error {
 	var ctx = &ExecuteContext{
-		NodeName:      a.nodeName,
-		Client:        a.client,
-		PodLister:     a.podLister,
-		NodeLister:    a.nodeLister,
-		RuntimeClient: a.runtimeClient,
-		RuntimeConn:   a.runtimeConn,
-		getStateFunc:  a.getStateFunc,
+		NodeName:             a.nodeName,
+		Client:               a.client,
+		PodLister:            a.podLister,
+		NodeLister:           a.nodeLister,
+		RuntimeClient:        a.runtimeClient,
+		RuntimeConn:          a.runtimeConn,
+		getStateFunc:         a.getStateFunc,
+		executeExcessPercent: a.executeExcessPercent,
 	}
 
 	//step1 do enforcer actions

pkg/ensurance/executor/interface.go

@@ -37,4 +37,6 @@ type ExecuteContext struct {
 	EvictGapToWaterLines GapToWaterLines
 
 	getStateFunc func() map[string][]common.TimeSeries
+
+	executeExcessPercent float64
 }

pkg/ensurance/executor/throttle.go

@@ -83,7 +83,7 @@ func (t *ThrottleExecutor) Avoid(ctx *ExecuteContext) error {
 			errPodKeys = t.throttlePods(ctx, &totalReleased, highestPrioriyMetric)
 		}
 	} else {
-		ctx.ThrottoleDownGapToWaterLines, _, _ = buildGapToWaterLine(ctx.getStateFunc(), *t, EvictExecutor{})
+		ctx.ThrottoleDownGapToWaterLines, _, _ = buildGapToWaterLine(ctx.getStateFunc(), *t, EvictExecutor{}, ctx.executeExcessPercent)
 
 		if ctx.ThrottoleDownGapToWaterLines.HasUsageMissedMetric() {
 			highestPrioriyMetric := t.ThrottleDownWaterLine.GetHighestPriorityThrottleAbleMetric()
@@ -161,7 +161,7 @@ func (t *ThrottleExecutor) Restore(ctx *ExecuteContext) error {
 			errPodKeys = t.restorePods(ctx, &totalReleased, highestPrioriyMetric)
 		}
 	} else {
-		_, ctx.ThrottoleUpGapToWaterLines, _ = buildGapToWaterLine(ctx.getStateFunc(), *t, EvictExecutor{})
+		_, ctx.ThrottoleUpGapToWaterLines, _ = buildGapToWaterLine(ctx.getStateFunc(), *t, EvictExecutor{}, ctx.executeExcessPercent)
 
 		if ctx.ThrottoleUpGapToWaterLines.HasUsageMissedMetric() {
 			highestPrioriyMetric := t.ThrottleUpWaterLine.GetHighestPriorityThrottleAbleMetric()

pkg/ensurance/executor/waterline.go

@@ -127,7 +127,7 @@ type GapToWaterLines map[WaterLineMetric]float64
 
 // Only calculate gap for metrics that can be quantified
 func buildGapToWaterLine(stateMap map[string][]common.TimeSeries,
-	throttleExecutor ThrottleExecutor, evictExecutor EvictExecutor) (
+	throttleExecutor ThrottleExecutor, evictExecutor EvictExecutor, executeExcessPercent float64) (
 	throttleDownGapToWaterLines, throttleUpGapToWaterLines, eviceGapToWaterLines GapToWaterLines) {
 
 	throttleDownGapToWaterLines, throttleUpGapToWaterLines, eviceGapToWaterLines = make(map[WaterLineMetric]float64), make(map[WaterLineMetric]float64), make(map[WaterLineMetric]float64)
@@ -157,7 +157,7 @@ func buildGapToWaterLine(stateMap map[string][]common.TimeSeries,
 		if !evictExist {
 			delete(eviceGapToWaterLines, m)
 		} else {
-			eviceGapToWaterLines[m] = maxUsed - float64(evictWaterLine.PopSmallest().Value())
+			eviceGapToWaterLines[m] = executeExcessPercent * (maxUsed - float64(evictWaterLine.PopSmallest().Value()))
 		}
 	}
 
@@ -188,15 +188,15 @@ func buildGapToWaterLine(stateMap map[string][]common.TimeSeries,
 		if !throttleDownExist {
 			delete(throttleDownGapToWaterLines, m)
 		} else {
-			throttleDownGapToWaterLines[m] = maxUsed - float64(throttleDownWaterLine.PopSmallest().Value())
+			throttleDownGapToWaterLines[m] = executeExcessPercent * (maxUsed - float64(throttleDownWaterLine.PopSmallest().Value()))
 		}
 
 		// If metric not exist in ThrottleUpWaterLine, throttleUpGapToWaterLines of metric will can't be calculated
 		if !throttleUpExist {
 			delete(throttleUpGapToWaterLines, m)
 		} else {
 			// Attention: different with throttleDown and evict
-			throttleUpGapToWaterLines[m] = float64(throttleUpWaterLine.PopSmallest().Value()) - maxUsed
+			throttleUpGapToWaterLines[m] = executeExcessPercent * (float64(throttleUpWaterLine.PopSmallest().Value()) - maxUsed)
 		}
 	}
 	return

pkg/resource/node_resource_manager.go

@@ -73,13 +73,13 @@ type NodeResourceManager struct {
 	tspName string
 }
 
-func NewNodeResourceManager(client clientset.Interface, nodeName string, op map[string]string, tspName string, nodeInformer coreinformers.NodeInformer,
+func NewNodeResourceManager(client clientset.Interface, nodeName string, nodeResourceReserved map[string]string, tspName string, nodeInformer coreinformers.NodeInformer,
 	tspInformer predictionv1.TimeSeriesPredictionInformer, stateChann chan map[string][]common.TimeSeries) (*NodeResourceManager, error) {
-	reserveCpuPercent, err := utils.ParsePercentage(op[v1.ResourceCPU.String()])
+	reserveCpuPercent, err := utils.ParsePercentage(nodeResourceReserved[v1.ResourceCPU.String()])
 	if err != nil {
 		return nil, err
 	}
-	reserveMemoryPercent, err := utils.ParsePercentage(op[v1.ResourceMemory.String()])
+	reserveMemoryPercent, err := utils.ParsePercentage(nodeResourceReserved[v1.ResourceMemory.String()])
 	if err != nil {
 		return nil, err
 	}