Merge pull request #390 from qmhu/external-metric

Prediction for external metrics

cmd/metric-adapter/main.go

@@ -54,12 +54,11 @@ type MetricAdapter struct {
 }
 
 func (a *MetricAdapter) makeCustomMetricProvider(remoteAdapter *metricprovider.RemoteAdapter, client client.Client, recorder record.EventRecorder) provider.CustomMetricsProvider {
-
 	return metricprovider.NewCustomMetricProvider(client, remoteAdapter, recorder)
 }
 
-func (a *MetricAdapter) makeExternalMetricProvider(client client.Client, recorder record.EventRecorder, scaleClient scale.ScalesGetter, restMapper meta.RESTMapper) *metricprovider.ExternalMetricProvider {
-	return metricprovider.NewExternalMetricProvider(client, recorder, scaleClient, restMapper)
+func (a *MetricAdapter) makeExternalMetricProvider(remoteAdapter *metricprovider.RemoteAdapter, client client.Client, recorder record.EventRecorder, scaleClient scale.ScalesGetter, restMapper meta.RESTMapper) *metricprovider.ExternalMetricProvider {
+	return metricprovider.NewExternalMetricProvider(client, remoteAdapter, recorder, scaleClient, restMapper)
 }
 
 func main() {
@@ -144,7 +143,7 @@ func main() {
 	ctx := signals.SetupSignalHandler()
 
 	customMetricProvider := cmd.makeCustomMetricProvider(remoteAdapter, client, recorder)
-	externalMetricProvider := cmd.makeExternalMetricProvider(client, recorder, scaleClient, restMapper)
+	externalMetricProvider := cmd.makeExternalMetricProvider(remoteAdapter, client, recorder, scaleClient, restMapper)
 
 	cmd.WithCustomMetrics(customMetricProvider)
 	cmd.WithExternalMetrics(externalMetricProvider)

deploy/metric-adapter/apiservice.yaml

@@ -11,20 +11,6 @@ spec:
   insecureSkipTLSVerify: true
   groupPriorityMinimum: 100
   versionPriority: 100
----
-apiVersion: apiregistration.k8s.io/v1
-kind: APIService
-metadata:
-  name: v1beta2.custom.metrics.k8s.io
-spec:
-  service:
-    name: metric-adapter
-    namespace: crane-system
-  group: custom.metrics.k8s.io
-  version: v1beta2
-  insecureSkipTLSVerify: true
-  groupPriorityMinimum: 100
-  versionPriority: 200
 
 ---
 apiVersion: apiregistration.k8s.io/v1

examples/autoscaling/effective-hpa-cron-local.yaml

@@ -10,7 +10,7 @@ spec:
     name: php-apache
   minReplicas: 1        # MinReplicas is the lower limit replicas to the scale target which the autoscaler can scale down to.
   maxReplicas: 10       # MaxReplicas is the upper limit replicas to the scale target which the autoscaler can scale up to.
-  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual".
+  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Preview".
   # Better to setting cron to fill the one complete time period such as one day, one week
   # Below is one day cron scheduling, it
   #100                  --------     ---------        ----------

examples/autoscaling/effective-hpa-cron-shanghai.yaml

@@ -10,7 +10,7 @@ spec:
     name: php-apache
   minReplicas: 1        # MinReplicas is the lower limit replicas to the scale target which the autoscaler can scale down to.
   maxReplicas: 10       # MaxReplicas is the upper limit replicas to the scale target which the autoscaler can scale up to.
-  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual".
+  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Preview".
   # Better to setting cron to fill the one complete time period such as one day, one week
   # Below is one day cron scheduling, it
   #100                  --------     ---------        ----------

examples/autoscaling/effective-hpa-cron.yaml

@@ -10,7 +10,7 @@ spec:
     name: php-apache
   minReplicas: 1        # MinReplicas is the lower limit replicas to the scale target which the autoscaler can scale down to.
   maxReplicas: 10       # MaxReplicas is the upper limit replicas to the scale target which the autoscaler can scale up to.
-  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual".
+  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Preview".
   # Better to setting cron to fill the one complete time period such as one day, one week
   # Below is one day cron scheduling, it
   #100                  --------     ---------        ----------

examples/autoscaling/effective-hpa.yaml

@@ -10,7 +10,7 @@ spec:
     name: php-apache
   minReplicas: 1        # MinReplicas is the lower limit replicas to the scale target which the autoscaler can scale down to.
   maxReplicas: 10       # MaxReplicas is the upper limit replicas to the scale target which the autoscaler can scale up to.
-  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual".
+  scaleStrategy: Auto   # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Preview".
   # Metrics contains the specifications for which to use to calculate the desired replica count.
   metrics:
   - type: Resource
@@ -27,4 +27,4 @@ spec:
       algorithmType: dsp
       dsp:
         sampleInterval: "60s"
-        historyLength: "3d"
+        historyLength: "7d"

go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/go-echarts/go-echarts/v2 v2.2.4
-	github.com/gocrane/api v0.4.1-0.20220520134105-09d430d903ac
+	github.com/gocrane/api v0.5.1-0.20220706040335-eaadbb4b99ed
 	github.com/google/cadvisor v0.39.2
 	github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12
 	github.com/prometheus/client_golang v1.11.0
@@ -181,6 +181,7 @@ require (
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/tools v0.1.8 // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/protobuf v1.27.1
 )
 
 replace (

go.sum

@@ -314,6 +314,8 @@ github.com/gocrane/api v0.4.1-0.20220507041258-d376db2b4ad4 h1:vGDg3G6y661KAlhjf
 github.com/gocrane/api v0.4.1-0.20220507041258-d376db2b4ad4/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
 github.com/gocrane/api v0.4.1-0.20220520134105-09d430d903ac h1:lBKVVOA4del0Plj80PCE+nglxaJxaXanCv5N6a3laVY=
 github.com/gocrane/api v0.4.1-0.20220520134105-09d430d903ac/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
+github.com/gocrane/api v0.5.1-0.20220706040335-eaadbb4b99ed h1:aARCU+Hs1ZKTqJFJT/4/or/iGR6qYwMcG99CGmBFJpg=
+github.com/gocrane/api v0.5.1-0.20220706040335-eaadbb4b99ed/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=

pkg/controller/ehpa/effective_hpa_controller.go

@@ -78,17 +78,18 @@ func (c *EffectiveHPAController) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	// reconcile prediction if enabled
+	var tsp *predictionapi.TimeSeriesPrediction
 	if utils.IsEHPAPredictionEnabled(ehpa) && utils.IsEHPAHasPredictionMetric(ehpa) {
-		prediction, err := c.ReconcilePredication(ctx, ehpa)
+		tsp, err = c.ReconcilePredication(ctx, ehpa)
 		if err != nil {
 			setCondition(newStatus, autoscalingapi.Ready, metav1.ConditionFalse, "FailedReconcilePrediction", err.Error())
 			c.UpdateStatus(ctx, ehpa, newStatus)
 			return ctrl.Result{}, err
 		}
-		setPredictionCondition(newStatus, prediction.Status.Conditions)
+		setPredictionCondition(newStatus, tsp.Status.Conditions)
 	}
 
-	hpa, err := c.ReconcileHPA(ctx, ehpa, substitute, newStatus)
+	hpa, err := c.ReconcileHPA(ctx, ehpa, substitute, tsp)
 	if err != nil {
 		setCondition(newStatus, autoscalingapi.Ready, metav1.ConditionFalse, "FailedReconcileHPA", err.Error())
 		c.UpdateStatus(ctx, ehpa, newStatus)

pkg/controller/ehpa/hpa.go

@@ -16,31 +16,32 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1"
+	predictionapi "github.com/gocrane/api/prediction/v1alpha1"
 
 	"github.com/gocrane/crane/pkg/known"
 	"github.com/gocrane/crane/pkg/metricprovider"
 	"github.com/gocrane/crane/pkg/utils"
 )
 
-func (c *EffectiveHPAController) ReconcileHPA(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) (*autoscalingv2.HorizontalPodAutoscaler, error) {
+func (c *EffectiveHPAController) ReconcileHPA(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, tsp *predictionapi.TimeSeriesPrediction) (*autoscalingv2.HorizontalPodAutoscaler, error) {
 	hpaList := &autoscalingv2.HorizontalPodAutoscalerList{}
 	opts := []client.ListOption{
 		client.MatchingLabels(map[string]string{known.EffectiveHorizontalPodAutoscalerUidLabel: string(ehpa.UID)}),
 	}
 	err := c.Client.List(ctx, hpaList, opts...)
 	if err != nil {
 		if errors.IsNotFound(err) {
-			return c.CreateHPA(ctx, ehpa, substitute, status)
+			return c.CreateHPA(ctx, ehpa, substitute, tsp)
 		} else {
 			c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedGetHPA", err.Error())
 			klog.Error("Failed to get HPA, ehpa %s error %v", klog.KObj(ehpa), err)
 			return nil, err
 		}
 	} else if len(hpaList.Items) == 0 {
-		return c.CreateHPA(ctx, ehpa, substitute, status)
+		return c.CreateHPA(ctx, ehpa, substitute, tsp)
 	}
 
-	return c.UpdateHPAIfNeed(ctx, ehpa, &hpaList.Items[0], substitute, status)
+	return c.UpdateHPAIfNeed(ctx, ehpa, &hpaList.Items[0], substitute, tsp)
 }
 
 func (c *EffectiveHPAController) GetHPA(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) (*autoscalingv2.HorizontalPodAutoscaler, error) {
@@ -58,8 +59,8 @@ func (c *EffectiveHPAController) GetHPA(ctx context.Context, ehpa *autoscalingap
 	return &hpaList.Items[0], nil
 }
 
-func (c *EffectiveHPAController) CreateHPA(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) (*autoscalingv2.HorizontalPodAutoscaler, error) {
-	hpa, err := c.NewHPAObject(ctx, ehpa, substitute, status)
+func (c *EffectiveHPAController) CreateHPA(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, tsp *predictionapi.TimeSeriesPrediction) (*autoscalingv2.HorizontalPodAutoscaler, error) {
+	hpa, err := c.NewHPAObject(ctx, ehpa, substitute, tsp)
 	if err != nil {
 		c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedCreateHPAObject", err.Error())
 		klog.Errorf("Failed to create object, HorizontalPodAutoscaler %s error %v", klog.KObj(hpa), err)
@@ -79,8 +80,8 @@ func (c *EffectiveHPAController) CreateHPA(ctx context.Context, ehpa *autoscalin
 	return hpa, nil
 }
 
-func (c *EffectiveHPAController) NewHPAObject(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) (*autoscalingv2.HorizontalPodAutoscaler, error) {
-	metrics, err := c.GetHPAMetrics(ctx, ehpa, status)
+func (c *EffectiveHPAController) NewHPAObject(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, tsp *predictionapi.TimeSeriesPrediction) (*autoscalingv2.HorizontalPodAutoscaler, error) {
+	metrics, err := c.GetHPAMetrics(ctx, ehpa, tsp)
 	if err != nil {
 		return nil, err
 	}
@@ -135,9 +136,9 @@ func (c *EffectiveHPAController) NewHPAObject(ctx context.Context, ehpa *autosca
 	return hpa, nil
 }
 
-func (c *EffectiveHPAController) UpdateHPAIfNeed(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, hpaExist *autoscalingv2.HorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) (*autoscalingv2.HorizontalPodAutoscaler, error) {
+func (c *EffectiveHPAController) UpdateHPAIfNeed(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, hpaExist *autoscalingv2.HorizontalPodAutoscaler, substitute *autoscalingapi.Substitute, tsp *predictionapi.TimeSeriesPrediction) (*autoscalingv2.HorizontalPodAutoscaler, error) {
 	var needUpdate bool
-	hpa, err := c.NewHPAObject(ctx, ehpa, substitute, status)
+	hpa, err := c.NewHPAObject(ctx, ehpa, substitute, tsp)
 	if err != nil {
 		c.Recorder.Event(ehpa, v1.EventTypeNormal, "FailedCreateHPAObject", err.Error())
 		klog.Errorf("Failed to create object, HorizontalPodAutoscaler %s error %v", klog.KObj(hpa), err)
@@ -173,15 +174,15 @@ func (c *EffectiveHPAController) UpdateHPAIfNeed(ctx context.Context, ehpa *auto
 }
 
 // GetHPAMetrics loop metricSpec in EffectiveHorizontalPodAutoscaler and generate metricSpec for HPA
-func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) ([]autoscalingv2.MetricSpec, error) {
+func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler, tsp *predictionapi.TimeSeriesPrediction) ([]autoscalingv2.MetricSpec, error) {
 	var metrics []autoscalingv2.MetricSpec
 	for _, metric := range ehpa.Spec.Metrics {
 		copyMetric := metric.DeepCopy()
 		metrics = append(metrics, *copyMetric)
 	}
 
-	if utils.IsEHPAPredictionEnabled(ehpa) && isPredictionReady(status) {
-		var customMetricsForPrediction []autoscalingv2.MetricSpec
+	if utils.IsEHPAPredictionEnabled(ehpa) {
+		var metricsForPrediction []autoscalingv2.MetricSpec
 
 		for _, metric := range metrics {
 			// generate a custom metric for resource metric
@@ -191,6 +192,11 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
 					continue
 				}
 
+				if _, err := utils.GetReadyPredictionMetric(name, tsp); err != nil {
+					// metric is not predictable
+					continue
+				}
+
 				customMetric := &autoscalingv2.PodsMetricSource{
 					Metric: autoscalingv2.MetricIdentifier{
 						Name: name,
@@ -240,11 +246,79 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
 				}
 
 				metricSpec := autoscalingv2.MetricSpec{Pods: customMetric, Type: autoscalingv2.PodsMetricSourceType}
-				customMetricsForPrediction = append(customMetricsForPrediction, metricSpec)
+				metricsForPrediction = append(metricsForPrediction, metricSpec)
+			}
+
+			// generate a external metric for external metric
+			if metric.Type == autoscalingv2.ExternalMetricSourceType {
+				name := utils.GetGeneralPredictionMetricName(metric.Type, false, metric.External.Metric.Name)
+				expressionQuery := utils.GetExpressionQuery(metric.External.Metric.Name, ehpa.Annotations)
+				if len(expressionQuery) == 0 {
+					continue
+				}
+
+				if _, err := utils.GetReadyPredictionMetric(name, tsp); err != nil {
+					// metric is not predictable
+					continue
+				}
+
+				external := &autoscalingv2.ExternalMetricSource{
+					Metric: autoscalingv2.MetricIdentifier{
+						Name: name,
+						// add known.EffectiveHorizontalPodAutoscalerUidLabel=uid in metric.selector
+						// MetricAdapter use label selector to match the matching TimeSeriesPrediction to return metrics
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								known.EffectiveHorizontalPodAutoscalerUidLabel: string(ehpa.UID),
+							},
+						},
+					},
+					Target: autoscalingv2.MetricTarget{
+						Type:         autoscalingv2.AverageValueMetricType,
+						AverageValue: metric.External.Target.AverageValue,
+					},
+				}
+
+				metricSpec := autoscalingv2.MetricSpec{External: external, Type: autoscalingv2.ExternalMetricSourceType}
+				metricsForPrediction = append(metricsForPrediction, metricSpec)
+			}
+
+			// generate a custom metric for pods metric
+			if metric.Type == autoscalingv2.PodsMetricSourceType {
+				name := utils.GetGeneralPredictionMetricName(metric.Type, false, metric.Pods.Metric.Name)
+				expressionQuery := utils.GetExpressionQuery(metric.Pods.Metric.Name, ehpa.Annotations)
+				if len(expressionQuery) == 0 {
+					continue
+				}
+
+				if _, err := utils.GetReadyPredictionMetric(name, tsp); err != nil {
+					// metric is not predictable
+					continue
+				}
+
+				podsMetric := &autoscalingv2.PodsMetricSource{
+					Metric: autoscalingv2.MetricIdentifier{
+						Name: name,
+						// add known.EffectiveHorizontalPodAutoscalerUidLabel=uid in metric.selector
+						// MetricAdapter use label selector to match the matching TimeSeriesPrediction to return metrics
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								known.EffectiveHorizontalPodAutoscalerUidLabel: string(ehpa.UID),
+							},
+						},
+					},
+					Target: autoscalingv2.MetricTarget{
+						Type:         autoscalingv2.AverageValueMetricType,
+						AverageValue: metric.Pods.Target.AverageValue,
+					},
+				}
+
+				metricSpec := autoscalingv2.MetricSpec{Pods: podsMetric, Type: autoscalingv2.PodsMetricSourceType}
+				metricsForPrediction = append(metricsForPrediction, metricSpec)
 			}
 		}
 
-		metrics = append(metrics, customMetricsForPrediction...)
+		metrics = append(metrics, metricsForPrediction...)
 	}
 
 	// Construct cron external metrics for cron scale
@@ -263,7 +337,7 @@ func GetCronMetricSpecsForHPA(ehpa *autoscalingapi.EffectiveHorizontalPodAutosca
 		Type: autoscalingv2.ExternalMetricSourceType,
 		External: &autoscalingv2.ExternalMetricSource{
 			Metric: autoscalingv2.MetricIdentifier{
-				Name: ehpa.Name,
+				Name: utils.GetGeneralPredictionMetricName(autoscalingv2.PodsMetricSourceType, true, ehpa.Name),
 				Selector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						known.EffectiveHorizontalPodAutoscalerUidLabel: string(ehpa.UID),

pkg/controller/ehpa/predict.go

@@ -144,6 +144,35 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
 				},
 			})
 		}
+		// get expressionQuery according to metric.Type
+		var expressionQuery string
+		var metricName string
+		switch metric.Type {
+		case autoscalingv2.ExternalMetricSourceType:
+			expressionQuery = utils.GetExpressionQuery(metric.External.Metric.Name, ehpa.Annotations)
+			metricName = metric.External.Metric.Name
+		case autoscalingv2.PodsMetricSourceType:
+			expressionQuery = utils.GetExpressionQuery(metric.Pods.Metric.Name, ehpa.Annotations)
+			metricName = metric.Pods.Metric.Name
+		}
+
+		if len(expressionQuery) == 0 {
+			continue
+		}
+
+		metricIdentifier := utils.GetGeneralPredictionMetricName(metric.Type, false, metricName)
+		predictionMetrics = append(predictionMetrics, predictionapi.PredictionMetric{
+			ResourceIdentifier: metricIdentifier,
+			Type:               predictionapi.ExpressionQueryMetricType,
+			ExpressionQuery: &predictionapi.ExpressionQuery{
+				Expression: expressionQuery,
+			},
+			Algorithm: predictionapi.Algorithm{
+				AlgorithmType: ehpa.Spec.Prediction.PredictionAlgorithm.AlgorithmType,
+				DSP:           ehpa.Spec.Prediction.PredictionAlgorithm.DSP,
+				Percentile:    ehpa.Spec.Prediction.PredictionAlgorithm.Percentile,
+			},
+		})
 	}
 	prediction.Spec.PredictionMetrics = predictionMetrics
 
@@ -164,13 +193,3 @@ func setPredictionCondition(status *autoscalingapi.EffectiveHorizontalPodAutosca
 		}
 	}
 }
-
-func isPredictionReady(status *autoscalingapi.EffectiveHorizontalPodAutoscalerStatus) bool {
-	for _, cond := range status.Conditions {
-		if cond.Type == string(autoscalingapi.PredictionReady) && cond.Status == metav1.ConditionTrue {
-			return true
-		}
-	}
-
-	return false
-}