From 5531fc4e60da59d8dbaba5269091bfd58328c008 Mon Sep 17 00:00:00 2001 From: qmhu Date: Wed, 9 Mar 2022 11:46:18 +0800 Subject: [PATCH] ehpa recmmendation --- deploy/craned/deployment.yaml | 9 +- examples/config_set.yaml | 14 +- go.mod | 33 ++- go.sum | 10 +- .../analytics/analytics_controller.go | 2 - .../ehpa/effective_hpa_controller.go | 2 - pkg/controller/ehpa/hpa.go | 4 +- pkg/controller/ehpa/predict.go | 4 +- pkg/controller/ehpa/substitute.go | 2 - pkg/controller/ehpa/substitute_controller.go | 2 - .../recommendation_controller.go | 30 +- pkg/controller/recommendation/updater.go | 120 ++++++++ pkg/known/annotation.go | 4 + pkg/recommend/advisor/ehpa.go | 274 ++++++++++++++---- pkg/recommend/advisor/ehpa_test.go | 137 +++++++++ pkg/recommend/advisor/resource_request.go | 12 +- pkg/recommend/inspector/workload.go | 10 + pkg/recommend/recommender.go | 35 +++ pkg/recommend/types/types.go | 18 +- pkg/utils/hpa.go | 61 ++++ pkg/utils/pod.go | 14 + pkg/utils/pod_template.go | 64 ++++ .../ehpa/hpa_test.go => utils/pod_test.go} | 61 +++- 23 files changed, 805 insertions(+), 117 deletions(-) create mode 100644 pkg/controller/recommendation/updater.go create mode 100644 pkg/recommend/advisor/ehpa_test.go create mode 100644 pkg/utils/hpa.go create mode 100644 pkg/utils/pod_template.go rename pkg/{controller/ehpa/hpa_test.go => utils/pod_test.go} (59%) diff --git a/deploy/craned/deployment.yaml b/deploy/craned/deployment.yaml index 49dd47fd4..c93a32079 100644 --- a/deploy/craned/deployment.yaml +++ b/deploy/craned/deployment.yaml @@ -85,9 +85,16 @@ data: configs: - targets: [] properties: - cpu-request-percentile: "0.98" + resource.cpu-request-percentile: "0.98" ehpa.deployment-min-replicas: "1" ehpa.statefulset-min-replicas: "1" ehpa.workload-min-replicas: "1" ehpa.pod-min-ready-seconds: "30" ehpa.pod-available-ratio: "0.5" + ehpa.default-min-replicas: "2" + ehpa.max-replicas-factor: "3" + ehpa.min-cpu-usage-threshold: "10" + ehpa.fluctuation-threshold: "3" + ehpa.min-cpu-target-utilization: "30" + ehpa.max-cpu-target-utilization: "75" + ehpa.reference-hpa: "true" \ No newline at end of file diff --git a/examples/config_set.yaml b/examples/config_set.yaml index 47316297f..debf4260e 100644 --- a/examples/config_set.yaml +++ b/examples/config_set.yaml @@ -3,4 +3,16 @@ kind: ConfigSet configs: - targets: [] properties: - cpu-request-percentile: "0.98" \ No newline at end of file + resource.cpu-request-percentile: "0.98" + ehpa.deployment-min-replicas: "1" + ehpa.statefulset-min-replicas: "1" + ehpa.workload-min-replicas: "1" + ehpa.pod-min-ready-seconds: "30" + ehpa.pod-available-ratio: "0.5" + ehpa.default-min-replicas: "2" + ehpa.max-replicas-factor: "3" + ehpa.min-cpu-usage-threshold: "10" + ehpa.fluctuation-threshold: "3" + ehpa.min-cpu-target-utilization: "30" + ehpa.max-cpu-target-utilization: "75" + ehpa.reference-hpa: "true" \ No newline at end of file diff --git a/go.mod b/go.mod index c27e0e8cf..f824013ab 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/go-echarts/go-echarts/v2 v2.2.4 - github.com/gocrane/api v0.2.1-0.20220308075341-c4f28c1981e6 + github.com/gocrane/api v0.2.1-0.20220309033244-699efd59d009 github.com/google/cadvisor v0.39.2 github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12 github.com/prometheus/client_golang v1.11.0 @@ -31,6 +31,22 @@ require ( ) require ( + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/gin-contrib/cors v1.3.1 + github.com/gin-contrib/pprof v1.3.0 + github.com/gin-gonic/gin v1.7.7 + github.com/golang/mock v1.5.0 + github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 + github.com/montanaflynn/stats v0.6.6 + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/zsais/go-gin-prometheus v0.1.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + gopkg.in/gcfg.v1 v1.2.0 + k8s.io/kube-openapi v0.0.0-20210817084001-7fbd8d59e5b8 // indirect +) + +require ( + github.com/Microsoft/go-winio v0.5.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -139,23 +155,8 @@ require ( sigs.k8s.io/yaml v1.2.0 // indirect ) -require ( - github.com/StackExchange/wmi v1.2.1 // indirect - github.com/gin-contrib/cors v1.3.1 - github.com/gin-contrib/pprof v1.3.0 - github.com/gin-gonic/gin v1.7.7 - github.com/golang/mock v1.5.0 - github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/zsais/go-gin-prometheus v0.1.0 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - gopkg.in/gcfg.v1 v1.2.0 - k8s.io/kube-openapi v0.0.0-20210817084001-7fbd8d59e5b8 // indirect -) - require ( cloud.google.com/go v0.84.0 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index f03e67e91..74defa538 100644 --- a/go.sum +++ b/go.sum @@ -306,10 +306,10 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ= github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= -github.com/gocrane/api v0.2.0 h1:6e8urUDsN8dx/KvLk01m8uMsox+2Y09N9rm6H9AquIM= -github.com/gocrane/api v0.2.0/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= -github.com/gocrane/api v0.2.1-0.20220308075341-c4f28c1981e6 h1:++2IaGyBIoUff3082SG7BNjBR4Nn/X4qjH6+bnzJIv4= -github.com/gocrane/api v0.2.1-0.20220308075341-c4f28c1981e6/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= +github.com/gocrane/api v0.2.1-0.20220307082411-6171d03c2dc5 h1:Q8ZYSeMCoz8VLor5nFZ8BIVuKVm+KgnfSMOr+XTLyOU= +github.com/gocrane/api v0.2.1-0.20220307082411-6171d03c2dc5/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= +github.com/gocrane/api v0.2.1-0.20220309033244-699efd59d009 h1:xd175jH+TT03ea52N4187vD5uoZmHQUAvMWTUPOIj2Y= +github.com/gocrane/api v0.2.1-0.20220309033244-699efd59d009/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -556,6 +556,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= diff --git a/pkg/controller/analytics/analytics_controller.go b/pkg/controller/analytics/analytics_controller.go index f906a58a7..6cb9af091 100644 --- a/pkg/controller/analytics/analytics_controller.go +++ b/pkg/controller/analytics/analytics_controller.go @@ -358,8 +358,6 @@ func (c *Controller) GetIdentities(ctx context.Context, analytics *analysisv1alp func (c *Controller) UpdateStatus(ctx context.Context, analytics *analysisv1alph1.Analytics, newStatus *analysisv1alph1.AnalyticsStatus) { if !equality.Semantic.DeepEqual(&analytics.Status, newStatus) { - klog.V(4).Infof("Analytics status should be updated, currentStatus %v newStatus %v", &analytics.Status, newStatus) - analytics.Status = *newStatus err := c.Update(ctx, analytics) if err != nil { diff --git a/pkg/controller/ehpa/effective_hpa_controller.go b/pkg/controller/ehpa/effective_hpa_controller.go index ef2db01ea..e11b0a2f4 100644 --- a/pkg/controller/ehpa/effective_hpa_controller.go +++ b/pkg/controller/ehpa/effective_hpa_controller.go @@ -128,8 +128,6 @@ func (c *EffectiveHPAController) Reconcile(ctx context.Context, req ctrl.Request func (c *EffectiveHPAController) UpdateStatus(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, newStatus *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) { if !equality.Semantic.DeepEqual(&ehpa.Status, newStatus) { - klog.V(4).Infof("EffectiveHorizontalPodAutoscaler status should be updated, currentStatus %v newStatus %v", &ehpa.Status, newStatus) - ehpa.Status = *newStatus err := c.Status().Update(ctx, ehpa) if err != nil { diff --git a/pkg/controller/ehpa/hpa.go b/pkg/controller/ehpa/hpa.go index a16f35d4f..ce744072b 100644 --- a/pkg/controller/ehpa/hpa.go +++ b/pkg/controller/ehpa/hpa.go @@ -62,7 +62,7 @@ func (c *EffectiveHPAController) CreateHPA(ctx context.Context, ehpa *autoscalin hpa, err := c.NewHPAObject(ctx, ehpa, substitute) if err != nil { c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedCreateHPAObject", err.Error()) - klog.Errorf("Failed to create object, HorizontalPodAutoscaler %s error %v", hpa, err) + klog.Errorf("Failed to create object, HorizontalPodAutoscaler %s error %v", klog.KObj(hpa), err) return nil, err } @@ -160,8 +160,6 @@ func (c *EffectiveHPAController) UpdateHPAIfNeed(ctx context.Context, ehpa *auto } if needUpdate { - klog.V(4).Infof("HorizontalPodAutoscaler is unsynced according to EffectiveHorizontalPodAutoscaler, should be updated, currentHPA %v expectHPA %v", hpaExist, hpa) - err := c.Update(ctx, hpaExist) if err != nil { c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedUpdateHPA", err.Error()) diff --git a/pkg/controller/ehpa/predict.go b/pkg/controller/ehpa/predict.go index 4463981f2..b88a95f4e 100644 --- a/pkg/controller/ehpa/predict.go +++ b/pkg/controller/ehpa/predict.go @@ -80,13 +80,11 @@ func (c *EffectiveHPAController) UpdatePredictionIfNeed(ctx context.Context, ehp prediction, err := c.NewPredictionObject(ehpa) if err != nil { c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedCreatePredictionObject", err.Error()) - klog.Errorf("Failed to create object, TimeSeriesPrediction %s error %v", prediction, err) + klog.Errorf("Failed to create object, TimeSeriesPrediction %s error %v", klog.KObj(prediction), err) return nil, err } if !equality.Semantic.DeepEqual(&predictionExist.Spec, &prediction.Spec) { - klog.V(4).Infof("TimeSeriesPrediction is unsynced according to EffectiveHorizontalPodAutoscaler, should be updated, currentTimeSeriesPrediction %v expectTimeSeriesPrediction %v", predictionExist.Spec, prediction.Spec) - predictionExist.Spec = prediction.Spec err := c.Update(ctx, predictionExist) if err != nil { diff --git a/pkg/controller/ehpa/substitute.go b/pkg/controller/ehpa/substitute.go index caaae6fb8..d90eb67f2 100644 --- a/pkg/controller/ehpa/substitute.go +++ b/pkg/controller/ehpa/substitute.go @@ -89,8 +89,6 @@ func (c *EffectiveHPAController) NewSubstituteObject(ehpa *autoscalingapi.Effect func (c *EffectiveHPAController) UpdateSubstituteIfNeed(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substituteExist *autoscalingapi.Substitute, scale *autoscalingapiv1.Scale) (*autoscalingapi.Substitute, error) { if !equality.Semantic.DeepEqual(&substituteExist.Spec.SubstituteTargetRef, &ehpa.Spec.ScaleTargetRef) { - klog.V(4).Infof("Substitute is unsynced according to EffectiveHorizontalPodAutoscaler, should be updated, currentTarget %v expectTarget %v", substituteExist.Spec.SubstituteTargetRef, ehpa.Spec.ScaleTargetRef) - substituteExist.Spec.SubstituteTargetRef = ehpa.Spec.ScaleTargetRef err := c.Update(ctx, substituteExist) if err != nil { diff --git a/pkg/controller/ehpa/substitute_controller.go b/pkg/controller/ehpa/substitute_controller.go index 25e9682fe..7352a479d 100644 --- a/pkg/controller/ehpa/substitute_controller.go +++ b/pkg/controller/ehpa/substitute_controller.go @@ -57,8 +57,6 @@ func (c *SubstituteController) Reconcile(ctx context.Context, req ctrl.Request) } if !equality.Semantic.DeepEqual(&substitute.Status, &newStatus) { - klog.V(4).Infof("Substitute status should be updated", "current %v new %v", substitute.Status, newStatus) - substitute.Status = newStatus err := c.Status().Update(ctx, substitute) if err != nil { diff --git a/pkg/controller/recommendation/recommendation_controller.go b/pkg/controller/recommendation/recommendation_controller.go index 4a8b93b08..2d4562d4a 100644 --- a/pkg/controller/recommendation/recommendation_controller.go +++ b/pkg/controller/recommendation/recommendation_controller.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" @@ -56,6 +55,11 @@ func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, nil } + // defaulting for TargetRef.Namespace + if recommendation.Spec.TargetRef.Namespace == "" { + recommendation.Spec.TargetRef.Namespace = recommendation.Namespace + } + c.DoRecommend(ctx, recommendation) if recommendation.Spec.CompletionStrategy.CompletionStrategyType == analysisv1alph1.CompletionStrategyPeriodical { @@ -98,7 +102,7 @@ func (c *Controller) DoRecommend(ctx context.Context, recommendation *analysisv1 recommender, err := recommend.NewRecommender(c.Client, c.RestMapper, c.ScaleClient, recommendation, c.Predictors, c.Provider, c.ConfigSet) if err != nil { - c.Recorder.Event(recommendation, v1.EventTypeNormal, "FailedCreateRecommender", err.Error()) + c.Recorder.Event(recommendation, v1.EventTypeWarning, "FailedCreateRecommender", err.Error()) msg := fmt.Sprintf("Failed to create recommender, Recommendation %s error %v", klog.KObj(recommendation), err) klog.Errorf(msg) setReadyCondition(newStatus, metav1.ConditionFalse, "FailedCreateRecommender", msg) @@ -108,7 +112,7 @@ func (c *Controller) DoRecommend(ctx context.Context, recommendation *analysisv1 proposed, err := recommender.Offer() if err != nil { - c.Recorder.Event(recommendation, v1.EventTypeNormal, "FailedOfferRecommendation", err.Error()) + c.Recorder.Event(recommendation, v1.EventTypeWarning, "FailedOfferRecommendation", err.Error()) msg := fmt.Sprintf("Failed to offer recommend, Recommendation %s: %v", klog.KObj(recommendation), err) klog.Errorf(msg) setReadyCondition(newStatus, metav1.ConditionFalse, "FailedOfferRecommend", msg) @@ -116,14 +120,14 @@ func (c *Controller) DoRecommend(ctx context.Context, recommendation *analysisv1 return } - if proposed != nil { - if proposed.ResourceRequest != nil { - val, _ := yaml.Marshal(proposed.ResourceRequest) - newStatus.RecommendedValue = string(val) - } else if proposed.EffectiveHPA != nil { - val, _ := yaml.Marshal(proposed.EffectiveHPA) - newStatus.RecommendedValue = string(val) - } + err = c.UpdateRecommendation(ctx, recommendation, proposed, newStatus) + if err != nil { + c.Recorder.Event(recommendation, v1.EventTypeWarning, "FailedUpdateRecommendationValue", err.Error()) + msg := fmt.Sprintf("Failed to update recommendation value, Recommendation %s: %v", klog.KObj(recommendation), err) + klog.Errorf(msg) + setReadyCondition(newStatus, metav1.ConditionFalse, "FailedUpdateRecommendationValue", msg) + c.UpdateStatus(ctx, recommendation, newStatus) + return } setReadyCondition(newStatus, metav1.ConditionTrue, "RecommendationReady", "Recommendation is ready") @@ -132,15 +136,13 @@ func (c *Controller) DoRecommend(ctx context.Context, recommendation *analysisv1 func (c *Controller) UpdateStatus(ctx context.Context, recommendation *analysisv1alph1.Recommendation, newStatus *analysisv1alph1.RecommendationStatus) { if !equality.Semantic.DeepEqual(&recommendation.Status, newStatus) { - klog.V(4).Infof("Recommendation status should be updated, currentStatus %v newStatus %v", &recommendation.Status, newStatus) - recommendation.Status = *newStatus timeNow := metav1.Now() recommendation.Status.LastUpdateTime = &timeNow err := c.Update(ctx, recommendation) if err != nil { - c.Recorder.Event(recommendation, v1.EventTypeNormal, "FailedUpdateStatus", err.Error()) + c.Recorder.Event(recommendation, v1.EventTypeWarning, "FailedUpdateStatus", err.Error()) klog.Errorf("Failed to update status, Recommendation %s error %v", klog.KObj(recommendation), err) return } diff --git a/pkg/controller/recommendation/updater.go b/pkg/controller/recommendation/updater.go new file mode 100644 index 000000000..c2248d6b9 --- /dev/null +++ b/pkg/controller/recommendation/updater.go @@ -0,0 +1,120 @@ +package recommendation + +import ( + "context" + "fmt" + + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + analysisapi "github.com/gocrane/api/analysis/v1alpha1" + autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" + + "github.com/gocrane/crane/pkg/known" + "github.com/gocrane/crane/pkg/recommend/types" + "github.com/gocrane/crane/pkg/utils" +) + +func (c *Controller) UpdateRecommendation(ctx context.Context, recommendation *analysisapi.Recommendation, proposed *types.ProposedRecommendation, status *analysisapi.RecommendationStatus) error { + var value string + if proposed.ResourceRequest != nil { + valueBytes, err := yaml.Marshal(proposed.ResourceRequest) + if err != nil { + return err + } + value = string(valueBytes) + } else if proposed.EffectiveHPA != nil { + valueBytes, err := yaml.Marshal(proposed.EffectiveHPA) + if err != nil { + return err + } + value = string(valueBytes) + } + + status.RecommendedValue = value + if recommendation.Spec.AdoptionType == analysisapi.AdoptionTypeStatus { + return nil + } + + unstructed := &unstructured.Unstructured{} + unstructed.SetAPIVersion(recommendation.Spec.TargetRef.APIVersion) + unstructed.SetKind(recommendation.Spec.TargetRef.Kind) + err := c.Client.Get(ctx, client.ObjectKey{Name: recommendation.Spec.TargetRef.Name, Namespace: recommendation.Spec.TargetRef.Namespace}, unstructed) + if err != nil { + return fmt.Errorf("Get target object failed: %v. ", err) + } + + if recommendation.Spec.AdoptionType == analysisapi.AdoptionTypeStatusAndAnnotation || recommendation.Spec.AdoptionType == analysisapi.AdoptionTypeAuto { + annotation := unstructed.GetAnnotations() + if annotation == nil { + annotation = map[string]string{} + } + + annotation[known.RecommendationValueAnnotation] = value + unstructed.SetAnnotations(annotation) + err = c.Client.Update(ctx, unstructed) + if err != nil { + return fmt.Errorf("Update target annotation failed: %v. ", err) + } + } + + // Only support Auto Type for EHPA recommendation + if recommendation.Spec.AdoptionType == analysisapi.AdoptionTypeAuto && proposed.EffectiveHPA != nil { + ehpa, err := utils.GetEHPAFromScaleTarget(ctx, c.Client, recommendation.Namespace, recommendation.Spec.TargetRef) + if err != nil { + return fmt.Errorf("Get EHPA from target failed: %v. ", err) + } + if ehpa == nil { + ehpa = &autoscalingapi.EffectiveHorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: recommendation.Spec.TargetRef.Namespace, + Name: recommendation.Spec.TargetRef.Name, + }, + Spec: autoscalingapi.EffectiveHorizontalPodAutoscalerSpec{ + MinReplicas: proposed.EffectiveHPA.MinReplicas, + MaxReplicas: *proposed.EffectiveHPA.MaxReplicas, + Metrics: proposed.EffectiveHPA.Metrics, + ScaleStrategy: autoscalingapi.ScaleStrategyPreview, + Prediction: proposed.EffectiveHPA.Prediction, + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + Kind: recommendation.Spec.TargetRef.Kind, + APIVersion: recommendation.Spec.TargetRef.APIVersion, + Name: recommendation.Spec.TargetRef.Name, + }, + }, + } + + err = c.Client.Create(ctx, ehpa) + if err == nil { + c.Recorder.Event(ehpa, v1.EventTypeNormal, "UpdateValue", "Created EffectiveHorizontalPodAutoscaler.") + klog.Infof("Create EffectiveHorizontalPodAutoscaler successfully, recommendation %s", klog.KObj(recommendation)) + } + return err + } else { + // we don't override ScaleStrategy, because we always use preview to be the default version, + // if user change it, we don't want to override it. + // The reason for Prediction is the same. + ehpaUpdate := ehpa.DeepCopy() + ehpaUpdate.Spec.MinReplicas = proposed.EffectiveHPA.MinReplicas + ehpaUpdate.Spec.MaxReplicas = *proposed.EffectiveHPA.MaxReplicas + ehpaUpdate.Spec.Metrics = proposed.EffectiveHPA.Metrics + + if !equality.Semantic.DeepEqual(&ehpaUpdate.Spec, &ehpa.Spec) { + err = c.Client.Update(ctx, ehpaUpdate) + if err == nil { + c.Recorder.Event(ehpa, v1.EventTypeNormal, "UpdateValue", "Updated EffectiveHorizontalPodAutoscaler.") + klog.Infof("Update EffectiveHorizontalPodAutoscaler successfully, recommendation %s", klog.KObj(recommendation)) + } + return err + } + } + } + + return nil +} diff --git a/pkg/known/annotation.go b/pkg/known/annotation.go index 5d2fc0444..821d0460d 100644 --- a/pkg/known/annotation.go +++ b/pkg/known/annotation.go @@ -1 +1,5 @@ package known + +const ( + RecommendationValueAnnotation = "analysis.crane.io/recommendation-value" +) diff --git a/pkg/recommend/advisor/ehpa.go b/pkg/recommend/advisor/ehpa.go index 51035ba9b..db3df4f18 100644 --- a/pkg/recommend/advisor/ehpa.go +++ b/pkg/recommend/advisor/ehpa.go @@ -3,8 +3,10 @@ package advisor import ( "fmt" "math" + "strconv" "time" + "github.com/montanaflynn/stats" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -12,6 +14,7 @@ import ( autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" + "github.com/gocrane/crane/pkg/common" "github.com/gocrane/crane/pkg/prediction/config" "github.com/gocrane/crane/pkg/recommend/types" "github.com/gocrane/crane/pkg/utils" @@ -57,61 +60,52 @@ func (a *EHPAAdvisor) Advise(proposed *types.ProposedRecommendation) error { return fmt.Errorf("EHPAAdvisor prediction metrics data is unexpected, List length is %d ", len(tsListPrediction)) } - requestTotal, err := utils.CalculatePodRequests(a.Pods, resourceCpu) - if err != nil { - return fmt.Errorf("EHPAAdvisor calculate pod requests meet error %v ", err) - } - - maxCpuUsage := math.SmallestNonzeroFloat64 - minCpuUsage := math.MaxFloat64 - // compute historic metric + var cpuMax float64 + var cpuUsages []float64 + // combine values from historic and prediction for _, sample := range tsList[0].Samples { - if sample.Value > maxCpuUsage { - maxCpuUsage = sample.Value - } - if sample.Value < minCpuUsage { - minCpuUsage = sample.Value + cpuUsages = append(cpuUsages, sample.Value) + if sample.Value > cpuMax { + cpuMax = sample.Value } } - // compute prediction metric for _, sample := range tsListPrediction[0].Samples { - if sample.Value > maxCpuUsage { - maxCpuUsage = sample.Value - } - - if sample.Value < minCpuUsage { - minCpuUsage = sample.Value + cpuUsages = append(cpuUsages, sample.Value) + if sample.Value > cpuMax { + cpuMax = sample.Value } } - klog.V(4).Info("EHPAAdvisor maxCpuUsage %f minCpuUsage %f", maxCpuUsage, minCpuUsage) - - targetCpuUtilization := int32(50) // todo: configurable - maxReplicasFactor := 1.2 // todo: configurable + err = a.checkMinCpuUsageThreshold(cpuMax) + if err != nil { + return fmt.Errorf("EHPAAdvisor checkMinCpuUsageThreshold failed: %v", err) + } - maxCpuUtilization := int32(int64(maxCpuUsage) * 1000 * 100 / requestTotal) - proposedMaxRatio := float64(maxCpuUtilization) / float64(targetCpuUtilization) - maxReplicasProposed := int32(math.Ceil(proposedMaxRatio * float64(a.ReadyPodNumber) * maxReplicasFactor)) + err = a.checkFluctuation(tsListPrediction) + if err != nil { + return fmt.Errorf("EHPAAdvisor checkFluctuation failed: %v", err) + } - minCpuUtilization := int32(int64(minCpuUsage) * 1000 * 100 / requestTotal) - proposedMinRatio := float64(minCpuUtilization) / float64(targetCpuUtilization) - minReplicasProposed := int32(math.Ceil(proposedMinRatio * float64(a.ReadyPodNumber))) + minReplicas, err := a.proposeMinReplicas() + if err != nil { + return fmt.Errorf("EHPAAdvisor proposeMinReplicas failed: %v", err) + } - // minReplicasProposed should be larger than 0 - if minReplicasProposed < 1 { - minReplicasProposed = 1 + targetUtilization, err := a.proposeTargetUtilization() + if err != nil { + return fmt.Errorf("EHPAAdvisor proposeTargetUtilization failed: %v", err) } - // maxReplicas should be always larger than minReplicas - if maxReplicasProposed < minReplicasProposed { - maxReplicasProposed = minReplicasProposed + maxReplicas, err := a.proposeMaxReplicas(cpuUsages, targetUtilization, minReplicas) + if err != nil { + return fmt.Errorf("EHPAAdvisor proposeMaxReplicas failed: %v", err) } defaultPredictionWindow := int32(3600) proposedEHPA := &types.EffectiveHorizontalPodAutoscalerRecommendation{ - MaxReplicas: &maxReplicasProposed, - MinReplicas: &minReplicasProposed, + MaxReplicas: &maxReplicas, + MinReplicas: &minReplicas, Metrics: []autoscalingv2.MetricSpec{ { Type: autoscalingv2.ResourceMetricSourceType, @@ -119,7 +113,7 @@ func (a *EHPAAdvisor) Advise(proposed *types.ProposedRecommendation) error { Name: resourceCpu, Target: autoscalingv2.MetricTarget{ Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: &targetCpuUtilization, + AverageUtilization: &targetUtilization, }, }, }, @@ -133,6 +127,23 @@ func (a *EHPAAdvisor) Advise(proposed *types.ProposedRecommendation) error { }, } + referenceHpa, err := strconv.ParseBool(a.Context.ConfigProperties["ehpa.reference-hpa"]) + if err != nil { + return err + } + + // get metric spec from existing hpa and use them + if referenceHpa && a.HPA != nil { + for _, metricSpec := range a.HPA.Spec.Metrics { + // don't use resource cpu, since we already configuration it before + if metricSpec.Type == autoscalingv2.ResourceMetricSourceType && metricSpec.Resource != nil && metricSpec.Resource.Name == resourceCpu { + continue + } + + proposedEHPA.Metrics = append(proposedEHPA.Metrics, metricSpec) + } + } + proposed.EffectiveHPA = proposedEHPA return nil } @@ -141,25 +152,190 @@ func (a *EHPAAdvisor) Name() string { return "EHPAAdvisor" } +// checkMinCpuUsageThreshold check if the max cpu for target is reach to ehpa.min-cpu-usage-threshold +func (a *EHPAAdvisor) checkMinCpuUsageThreshold(cpuMax float64) error { + minCpuUsageThreshold, err := strconv.ParseFloat(a.Context.ConfigProperties["ehpa.min-cpu-usage-threshold"], 64) + if err != nil { + return err + } + + klog.V(4).Infof("EHPAAdvisor checkMinCpuUsageThreshold, cpuMax %f threshold %f", cpuMax, minCpuUsageThreshold) + if cpuMax < minCpuUsageThreshold { + return fmt.Errorf("Target cpuusage %f is under ehpa.min-cpu-usage-threshold %f. ", cpuMax, minCpuUsageThreshold) + } + + return nil +} + +// checkFluctuation check if the time series fluctuation is reach to ehpa.fluctuation-threshold +func (a *EHPAAdvisor) checkFluctuation(predictionTs []*common.TimeSeries) error { + fluctuationThreshold, err := strconv.ParseFloat(a.Context.ConfigProperties["ehpa.fluctuation-threshold"], 64) + if err != nil { + return err + } + + // aggregate with time's hour + cpuUsagePredictionMap := make(map[int][]float64) + for _, sample := range predictionTs[0].Samples { + sampleTime := time.Unix(sample.Timestamp, 0) + if _, exist := cpuUsagePredictionMap[sampleTime.Hour()]; exist { + cpuUsagePredictionMap[sampleTime.Hour()] = append(cpuUsagePredictionMap[sampleTime.Hour()], sample.Value) + } else { + newUsageInHour := make([]float64, 0) + newUsageInHour = append(newUsageInHour, sample.Value) + cpuUsagePredictionMap[sampleTime.Hour()] = newUsageInHour + } + } + + // use median to deburring data + var medianUsages []float64 + for _, usageInHour := range cpuUsagePredictionMap { + medianUsage, err := stats.Median(usageInHour) + if err != nil { + return err + } + medianUsages = append(medianUsages, medianUsage) + } + + medianMax := math.SmallestNonzeroFloat64 + medianMin := math.MaxFloat64 + for _, value := range medianUsages { + if value > medianMax { + medianMax = value + } + + if value < medianMin { + medianMin = value + } + } + + if medianMin == 0 { + return fmt.Errorf("Mean cpu usage is zero. ") + } + + klog.V(4).Infof("EHPAAdvisor checkFluctuation, medianMax %f medianMin %f medianUsages %v", medianMax, medianMin, medianUsages) + fluctuation := medianMax / medianMin + if fluctuation < fluctuationThreshold { + return fmt.Errorf("Target cpu fluctuation %f is under ehpa.fluctuation-threshold %f. ", fluctuation, fluctuationThreshold) + } + + return nil +} + +// proposeTargetUtilization use the 99 percentile cpu usage to propose target utilization, +// since we think if pod have reach the top usage before, maybe this is a suitable target to running. +// Considering too high or too low utilization are both invalid, we will be capping target utilization finally. +func (a *EHPAAdvisor) proposeTargetUtilization() (int32, error) { + minCpuTargetUtilization, err := strconv.ParseInt(a.Context.ConfigProperties["ehpa.min-cpu-target-utilization"], 10, 32) + if err != nil { + return 0, err + } + + maxCpuTargetUtilization, err := strconv.ParseInt(a.Context.ConfigProperties["ehpa.max-cpu-target-utilization"], 10, 32) + if err != nil { + return 0, err + } + + percentilePredictor := a.Predictors[predictionapi.AlgorithmTypePercentile] + + var cpuUsage float64 + // use percentile algo to get the 99 percentile cpu usage for this target + for _, container := range a.PodTemplate.Spec.Containers { + queryExpr := fmt.Sprintf(cpuQueryExprTemplate, container.Name, a.Recommendation.Spec.TargetRef.Namespace, a.Recommendation.Spec.TargetRef.Name) + cpuConfig := makeCpuConfig(a.ConfigProperties) + tsList, err := utils.QueryPredictedValuesOnce(a.Recommendation, + percentilePredictor, + fmt.Sprintf(callerFormat, a.Recommendation.UID), + cpuConfig, + queryExpr) + if err != nil { + return 0, err + } + if len(tsList) < 1 || len(tsList[0].Samples) < 1 { + return 0, fmt.Errorf("no value retured for queryExpr: %s", queryExpr) + } + cpuUsage += tsList[0].Samples[0].Value + } + + requestsPod, err := utils.CalculatePodTemplateRequests(a.PodTemplate, corev1.ResourceCPU) + if err != nil { + return 0, err + } + + klog.V(4).Infof("EHPAAdvisor propose targetUtilization, cpuUsage %f requestsPod %d", cpuUsage, requestsPod) + targetUtilization := int32(math.Ceil((cpuUsage * 1000 / float64(requestsPod)) * 100)) + + // capping + if targetUtilization < int32(minCpuTargetUtilization) { + targetUtilization = int32(minCpuTargetUtilization) + } + + // capping + if targetUtilization > int32(maxCpuTargetUtilization) { + targetUtilization = int32(maxCpuTargetUtilization) + } + + return targetUtilization, nil +} + +// proposeMinReplicas calculate min replicas based on ehpa.default-min-replicas +func (a *EHPAAdvisor) proposeMinReplicas() (int32, error) { + defaultMinReplicas, err := strconv.ParseInt(a.Context.ConfigProperties["ehpa.default-min-replicas"], 10, 32) + if err != nil { + return 0, err + } + + minReplicas := int32(defaultMinReplicas) + + // minReplicas should be larger than 0 + if minReplicas < 1 { + minReplicas = 1 + } + + return minReplicas, nil +} + +// proposeMaxReplicas use max cpu usage to compare with target pod cpu usage to get the max replicas. +func (a *EHPAAdvisor) proposeMaxReplicas(cpuUsages []float64, targetUtilization int32, minReplicas int32) (int32, error) { + maxReplicasFactor, err := strconv.ParseFloat(a.Context.ConfigProperties["ehpa.max-replicas-factor"], 64) + if err != nil { + return 0, err + } + // use percentile to deburring data + p95thCpu, err := stats.Percentile(cpuUsages, 95) + if err != nil { + return 0, err + } + requestsPod, err := utils.CalculatePodTemplateRequests(a.PodTemplate, corev1.ResourceCPU) + if err != nil { + return 0, err + } + + klog.V(4).Infof("EHPAAdvisor proposeMaxReplicas, p95thCpu %f requestsPod %d targetUtilization %d", p95thCpu, requestsPod, targetUtilization) + + // request * targetUtilization is the target average cpu usage, use total p95thCpu to divide, we can get the expect max replicas. + calcMaxReplicas := (p95thCpu * 100 * 1000 * maxReplicasFactor) / float64(int32(requestsPod)*targetUtilization) + maxReplicas := int32(math.Ceil(calcMaxReplicas)) + + // maxReplicas should be always larger than minReplicas + if maxReplicas < minReplicas { + maxReplicas = minReplicas + } + + return maxReplicas, nil +} + func getPredictionCpuConfig(expr string) *config.Config { return &config.Config{ Expression: &predictionapi.ExpressionQuery{Expression: expr}, - DSP: &predictionapi.DSP{ - SampleInterval: "1m", - HistoryLength: "5d", - Estimators: predictionapi.Estimators{ - FFTEstimators: []*predictionapi.FFTEstimator{ - {MarginFraction: "0.05", LowAmplitudeThreshold: "1.0", HighFrequencyThreshold: "0.05"}, - }, - }, - }, + DSP: &predictionapi.DSP{}, // use default DSP config will let prediction tuning by itself } } func ResourceToPromQueryExpr(namespace string, name string, resourceName *corev1.ResourceName) string { switch *resourceName { case corev1.ResourceCPU: - return fmt.Sprintf(config.WorkloadCpuUsagePromQLFmtStr, namespace, name, "3m") + return fmt.Sprintf(config.WorkloadCpuUsagePromQLFmtStr, namespace, name, "5m") case corev1.ResourceMemory: return fmt.Sprintf(config.WorkloadMemUsagePromQLFmtStr, namespace, name) } diff --git a/pkg/recommend/advisor/ehpa_test.go b/pkg/recommend/advisor/ehpa_test.go new file mode 100644 index 000000000..12045448c --- /dev/null +++ b/pkg/recommend/advisor/ehpa_test.go @@ -0,0 +1,137 @@ +package advisor + +import ( + "math/rand" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/gocrane/crane/pkg/common" + "github.com/gocrane/crane/pkg/recommend/types" +) + +func TestCheckFluctuation(t *testing.T) { + a := &EHPAAdvisor{ + Context: &types.Context{ + ConfigProperties: map[string]string{}, + }, + } + + var samples []common.Sample + timeSample := time.Now() + for i := 1; i < 1440; i++ { + sample := common.Sample{ + Timestamp: timeSample.Unix(), + Value: float64(i), + } + timeSample = timeSample.Add(time.Duration(1) * time.Minute) + samples = append(samples, sample) + } + + tsList := []*common.TimeSeries{{Samples: samples}} + + tests := []struct { + description string + threshold string + expectError bool + }{ + { + description: "check fluctuation passed", + threshold: "3", + expectError: false, + }, + { + description: "check fluctuation failed", + threshold: "100", + expectError: true, + }, + } + + for _, test := range tests { + a.Context.ConfigProperties["ehpa.fluctuation-threshold"] = test.threshold + err := a.checkFluctuation(tsList) + if err != nil && !test.expectError { + t.Errorf("Failed to checkFluctuation: %v", err) + } + } +} + +func TestProposeMaxReplicas(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + a := &EHPAAdvisor{ + Context: &types.Context{ + ConfigProperties: map[string]string{ + "ehpa.max-replicas-factor": "3", + }, + PodTemplate: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "podTemplateTest", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "container1", + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + }, { + Name: "container2", + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + }}, + }, + }, + }, + } + + var cpuUsages []float64 + for i := 1; i < 1000; i++ { + value := 1 + rand.Float64()*(20-1) // random float from [1,100] + cpuUsages = append(cpuUsages, value) + } + + tests := []struct { + description string + targetUtilization int32 + minReplicas int32 + expertReplicasAtLeast int32 + }{ + { + description: "use targetUtilization==50", + targetUtilization: 50, + minReplicas: 20, + expertReplicasAtLeast: 30, + }, + { + description: "use targetUtilization==10", + targetUtilization: 10, + minReplicas: 20, + expertReplicasAtLeast: 100, + }, + { + description: "capping by minReplicas", + targetUtilization: 50, + minReplicas: 60, + expertReplicasAtLeast: 60, + }, + } + + for _, test := range tests { + maxReplicas, err := a.proposeMaxReplicas(cpuUsages, test.targetUtilization, test.minReplicas) + if err != nil { + t.Errorf("Failed to checkFluctuation: %v", err) + } + if maxReplicas < test.expertReplicasAtLeast { + t.Errorf("Failed to proposeMaxReplicas, expect at least %d actual %d.", test.expertReplicasAtLeast, maxReplicas) + } + } +} diff --git a/pkg/recommend/advisor/resource_request.go b/pkg/recommend/advisor/resource_request.go index 8700acf64..c4c034df9 100644 --- a/pkg/recommend/advisor/resource_request.go +++ b/pkg/recommend/advisor/resource_request.go @@ -30,15 +30,15 @@ type ResourceRequestAdvisor struct { } func makeCpuConfig(props map[string]string) *config.Config { - sampleInterval, exists := props["cpu-sample-interval"] + sampleInterval, exists := props["resource.cpu-sample-interval"] if !exists { sampleInterval = "1m" } - percentile, exists := props["cpu-request-percentile"] + percentile, exists := props["resource.cpu-request-percentile"] if !exists { percentile = "0.99" } - marginFraction, exists := props["cpu-request-margin-fraction"] + marginFraction, exists := props["resource.cpu-request-margin-fraction"] if !exists { marginFraction = "0.15" } @@ -59,15 +59,15 @@ func makeCpuConfig(props map[string]string) *config.Config { } func makeMemConfig(props map[string]string) *config.Config { - sampleInterval, exists := props["mem-sample-interval"] + sampleInterval, exists := props["resource.mem-sample-interval"] if !exists { sampleInterval = "1m" } - percentile, exists := props["mem-request-percentile"] + percentile, exists := props["resource.mem-request-percentile"] if !exists { percentile = "0.99" } - marginFraction, exists := props["mem-request-margin-fraction"] + marginFraction, exists := props["resource.mem-request-margin-fraction"] if !exists { marginFraction = "0.15" } diff --git a/pkg/recommend/inspector/workload.go b/pkg/recommend/inspector/workload.go index 8dd8377bc..2bf34a9d8 100644 --- a/pkg/recommend/inspector/workload.go +++ b/pkg/recommend/inspector/workload.go @@ -39,6 +39,16 @@ func (i *WorkloadInspector) Inspect() error { return fmt.Errorf("Workload replicas %d should be larger than %d ", i.Context.Scale.Spec.Replicas, int32(workloadMinReplicas)) } + for _, container := range i.Context.PodTemplate.Spec.Containers { + if container.Resources.Requests.Cpu() == nil { + return fmt.Errorf("Container %s resource cpu request is empty ", container.Name) + } + + if container.Resources.Limits.Cpu() == nil { + return fmt.Errorf("Container %s resource cpu limit is empty ", container.Name) + } + } + return nil } diff --git a/pkg/recommend/recommender.go b/pkg/recommend/recommender.go index 8f641665c..14e7aca74 100644 --- a/pkg/recommend/recommender.go +++ b/pkg/recommend/recommender.go @@ -18,6 +18,7 @@ import ( analysisapi "github.com/gocrane/api/analysis/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" + "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/prediction" "github.com/gocrane/crane/pkg/providers" "github.com/gocrane/crane/pkg/recommend/advisor" @@ -155,6 +156,40 @@ func GetContext(kubeClient client.Client, restMapper meta.RESTMapper, return nil, err } + if recommendation.Spec.Type == analysisapi.AnalysisTypeHPA { + c.PodTemplate, err = utils.GetPodTemplate(context.TODO(), + recommendation.Spec.TargetRef.Namespace, + recommendation.Spec.TargetRef.Name, + recommendation.Spec.TargetRef.Kind, + recommendation.Spec.TargetRef.APIVersion, + kubeClient) + if err != nil { + return nil, err + } + + hpaList := &autoscalingv2.HorizontalPodAutoscalerList{} + opts := []client.ListOption{ + client.InNamespace(recommendation.Spec.TargetRef.Namespace), + } + err := kubeClient.List(context.TODO(), hpaList, opts...) + if err != nil { + return nil, err + } + + for _, hpa := range hpaList.Items { + // bypass hpa that controller by ehpa + if hpa.Labels != nil && hpa.Labels["app.kubernetes.io/managed-by"] == known.EffectiveHorizontalPodAutoscalerManagedBy { + continue + } + + if hpa.Spec.ScaleTargetRef.Name == recommendation.Spec.TargetRef.Name && + hpa.Spec.ScaleTargetRef.Kind == recommendation.Spec.TargetRef.APIVersion && + hpa.Spec.ScaleTargetRef.APIVersion == recommendation.Spec.TargetRef.APIVersion { + c.HPA = &hpa + } + } + } + c.Pods = pods c.Predictors = predictors c.DataSource = dataSource diff --git a/pkg/recommend/types/types.go b/pkg/recommend/types/types.go index 92797489a..d8a7f9c4c 100644 --- a/pkg/recommend/types/types.go +++ b/pkg/recommend/types/types.go @@ -1,7 +1,6 @@ package types import ( - autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" appsv1 "k8s.io/api/apps/v1" autoscalingapiv1 "k8s.io/api/autoscaling/v1" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" @@ -9,6 +8,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" analysisapi "github.com/gocrane/api/analysis/v1alpha1" + autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" "github.com/gocrane/crane/pkg/prediction" @@ -26,6 +26,8 @@ type Context struct { Deployment *appsv1.Deployment StatefulSet *appsv1.StatefulSet Pods []corev1.Pod + PodTemplate *corev1.PodTemplateSpec + HPA *autoscalingv2.HorizontalPodAutoscaler ReadyPodNumber int } @@ -39,19 +41,19 @@ type ProposedRecommendation struct { } type EffectiveHorizontalPodAutoscalerRecommendation struct { - MinReplicas *int32 `yaml:"minReplicas,omitempty"` - MaxReplicas *int32 `yaml:"maxReplicas,omitempty"` - Metrics []autoscalingv2.MetricSpec `yaml:"metrics,omitempty"` - Prediction *autoscalingapi.Prediction `yaml:"prediction,omitempty"` + MinReplicas *int32 `json:"minReplicas,omitempty"` + MaxReplicas *int32 `json:"maxReplicas,omitempty"` + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + Prediction *autoscalingapi.Prediction `json:"prediction,omitempty"` } type ResourceRequestRecommendation struct { - Containers []ContainerRecommendation `yaml:"containers,omitempty"` + Containers []ContainerRecommendation `json:"containers,omitempty"` } type ContainerRecommendation struct { - ContainerName string `yaml:"containerName,omitempty"` - Target ResourceList `yaml:"target,omitempty"` + ContainerName string `json:"containerName,omitempty"` + Target ResourceList `json:"target,omitempty"` } type ResourceList map[corev1.ResourceName]string diff --git a/pkg/utils/hpa.go b/pkg/utils/hpa.go new file mode 100644 index 000000000..cd2180516 --- /dev/null +++ b/pkg/utils/hpa.go @@ -0,0 +1,61 @@ +package utils + +import ( + "context" + "fmt" + + autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" + + "github.com/gocrane/crane/pkg/known" +) + +func GetHPAFromScaleTarget(context context.Context, kubeClient client.Client, namespace string, objRef corev1.ObjectReference) (*autoscalingv2.HorizontalPodAutoscaler, error) { + hpaList := &autoscalingv2.HorizontalPodAutoscalerList{} + opts := []client.ListOption{ + client.InNamespace(namespace), + } + err := kubeClient.List(context, hpaList, opts...) + if err != nil { + return nil, err + } + + for _, hpa := range hpaList.Items { + // bypass hpa that controller by ehpa + if hpa.Labels != nil && hpa.Labels["app.kubernetes.io/managed-by"] == known.EffectiveHorizontalPodAutoscalerManagedBy { + continue + } + + if hpa.Spec.ScaleTargetRef.Name == objRef.Name && + hpa.Spec.ScaleTargetRef.Kind == objRef.APIVersion && + hpa.Spec.ScaleTargetRef.APIVersion == objRef.APIVersion { + return &hpa, nil + } + } + + return nil, fmt.Errorf("HPA not found") +} + +func GetEHPAFromScaleTarget(context context.Context, kubeClient client.Client, namespace string, objRef corev1.ObjectReference) (*autoscalingapi.EffectiveHorizontalPodAutoscaler, error) { + ehpaList := &autoscalingapi.EffectiveHorizontalPodAutoscalerList{} + opts := []client.ListOption{ + client.InNamespace(namespace), + } + err := kubeClient.List(context, ehpaList, opts...) + if err != nil { + return nil, err + } + + for _, ehpa := range ehpaList.Items { + if ehpa.Spec.ScaleTargetRef.Name == objRef.Name && + ehpa.Spec.ScaleTargetRef.Kind == objRef.APIVersion && + ehpa.Spec.ScaleTargetRef.APIVersion == objRef.APIVersion { + return &ehpa, nil + } + } + + return nil, nil +} diff --git a/pkg/utils/pod.go b/pkg/utils/pod.go index 62ead96e7..044db8fc9 100644 --- a/pkg/utils/pod.go +++ b/pkg/utils/pod.go @@ -110,3 +110,17 @@ func GetPodContainerByName(pod *v1.Pod, containerName string) (v1.Container, err return v1.Container{}, fmt.Errorf("container not found") } + +// CalculatePodTemplateRequests sum request total from podTemplate +func CalculatePodTemplateRequests(podTemplate *v1.PodTemplateSpec, resource v1.ResourceName) (int64, error) { + var requests int64 + for _, c := range podTemplate.Spec.Containers { + if containerRequest, ok := c.Resources.Requests[resource]; ok { + requests += containerRequest.MilliValue() + } else { + return 0, fmt.Errorf("missing request for %s", resource) + } + } + + return requests, nil +} diff --git a/pkg/utils/pod_template.go b/pkg/utils/pod_template.go new file mode 100644 index 000000000..242d28241 --- /dev/null +++ b/pkg/utils/pod_template.go @@ -0,0 +1,64 @@ +package utils + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetPodTemplate(context context.Context, namespace string, name string, kind string, apiVersion string, kubeClient client.Client) (*v1.PodTemplateSpec, error) { + templateSpec := &v1.PodTemplateSpec{} + + key := client.ObjectKey{ + Name: name, + Namespace: namespace, + } + + if kind == "Deployment" && strings.HasPrefix(apiVersion, "apps") { + deployment := &appsv1.Deployment{} + err := kubeClient.Get(context, key, deployment) + if err != nil { + return nil, err + } + templateSpec = &deployment.Spec.Template + } else if kind == "StatefulSet" && strings.HasPrefix(apiVersion, "apps") { + statefulSet := &appsv1.StatefulSet{} + err := kubeClient.Get(context, key, statefulSet) + if err != nil { + return nil, err + } + templateSpec = &statefulSet.Spec.Template + } else { + unstructed := &unstructured.Unstructured{} + unstructed.SetAPIVersion(apiVersion) + unstructed.SetKind(kind) + err := kubeClient.Get(context, key, unstructed) + if err != nil { + return nil, err + } + + template, found, err := unstructured.NestedMap(unstructed.Object, "spec", "template") + if !found || err != nil { + return nil, fmt.Errorf("Get template from unstructed object %s failed. ", klog.KObj(unstructed)) + } + + templateBytes, err := json.Marshal(template) + if err != nil { + return nil, fmt.Errorf("Marshal unstructed object failed: %v. ", err) + } + + err = json.Unmarshal(templateBytes, templateSpec) + if err != nil { + return nil, fmt.Errorf("Unmarshal template bytes failed: %v. ", err) + } + } + + return templateSpec, nil +} diff --git a/pkg/controller/ehpa/hpa_test.go b/pkg/utils/pod_test.go similarity index 59% rename from pkg/controller/ehpa/hpa_test.go rename to pkg/utils/pod_test.go index 289ecf8b0..d1e64fd39 100644 --- a/pkg/controller/ehpa/hpa_test.go +++ b/pkg/utils/pod_test.go @@ -1,4 +1,4 @@ -package ehpa +package utils import ( "testing" @@ -6,8 +6,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/gocrane/crane/pkg/utils" ) func TestCalculatePodRequests(t *testing.T) { @@ -80,7 +78,7 @@ func TestCalculatePodRequests(t *testing.T) { } for _, test := range tests { - requests, err := utils.CalculatePodRequests(pods, test.resource) + requests, err := CalculatePodRequests(pods, test.resource) if err != nil { t.Errorf("Failed to calculatePodRequests: %v", err) } @@ -90,3 +88,58 @@ func TestCalculatePodRequests(t *testing.T) { } } + +func TestCalculatePodTemplateRequests(t *testing.T) { + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "podTemplateTest", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "container1", + Resources: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + }, { + Name: "container2", + Resources: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + }}, + }, + } + + tests := []struct { + description string + resource v1.ResourceName + expect int64 + }{ + { + description: "calculate cpu request total", + resource: v1.ResourceCPU, + expect: 2000, + }, + { + description: "calculate memory request total", + resource: v1.ResourceMemory, + expect: 20000, + }, + } + + for _, test := range tests { + requests, err := CalculatePodTemplateRequests(podTemplate, test.resource) + if err != nil { + t.Errorf("Failed to CalculatePodTemplateRequests: %v", err) + } + if requests != test.expect { + t.Errorf("expect requests %d actual requests %d", test.expect, requests) + } + } + +}