Add fallback functionality (#1910)

Signed-off-by: misha <mishamo@gmail.com>

CHANGELOG.md

@@ -24,6 +24,7 @@
 - Extend Azure Monitor scaler to support custom metrics ([#1883](https://github.com/kedacore/keda/pull/1883))
 - Support non-public cloud environments in the Azure Storage Queue and Azure Storage Blob scalers ([#1863](https://github.com/kedacore/keda/pull/1863))
 - Show HashiCorp Vault Address when using `kubectl get ta` or `kubectl get cta` ([#1862](https://github.com/kedacore/keda/pull/1862))
+- Add fallback functionality ([#1872](https://github.com/kedacore/keda/issues/1872))
 
 ### Improvements
 

api/v1alpha1/condition_types.go

@@ -14,6 +14,8 @@ const (
 	// ConditionActive specifies that the resource has finished.
 	// For resource which run to completion.
 	ConditionActive ConditionType = "Active"
+	// ConditionFallback specifies that the resource has a fallback active.
+	ConditionFallback ConditionType = "Fallback"
 )
 
 // Condition to store the condition state
@@ -44,6 +46,7 @@ type Conditions []Condition
 func (c *Conditions) AreInitialized() bool {
 	foundReady := false
 	foundActive := false
+	foundFallback := false
 	if *c != nil {
 		for _, condition := range *c {
 			if condition.Type == ConditionReady {
@@ -57,14 +60,20 @@ func (c *Conditions) AreInitialized() bool {
 				break
 			}
 		}
+		for _, condition := range *c {
+			if condition.Type == ConditionFallback {
+				foundFallback = true
+				break
+			}
+		}
 	}
 
-	return foundReady && foundActive
+	return foundReady && foundActive && foundFallback
 }
 
 // GetInitializedConditions returns Conditions initialized to the default -> Status: Unknown
 func GetInitializedConditions() *Conditions {
-	return &Conditions{{Type: ConditionReady, Status: metav1.ConditionUnknown}, {Type: ConditionActive, Status: metav1.ConditionUnknown}}
+	return &Conditions{{Type: ConditionReady, Status: metav1.ConditionUnknown}, {Type: ConditionActive, Status: metav1.ConditionUnknown}, {Type: ConditionFallback, Status: metav1.ConditionUnknown}}
 }
 
 // IsTrue is true if the condition is True
@@ -107,6 +116,14 @@ func (c *Conditions) SetActiveCondition(status metav1.ConditionStatus, reason st
 	c.setCondition(ConditionActive, status, reason, message)
 }
 
+// SetFallbackCondition modifies Fallback Condition according to input parameters
+func (c *Conditions) SetFallbackCondition(status metav1.ConditionStatus, reason string, message string) {
+	if *c == nil {
+		c = GetInitializedConditions()
+	}
+	c.setCondition(ConditionFallback, status, reason, message)
+}
+
 // GetActiveCondition returns Condition of type Active
 func (c *Conditions) GetActiveCondition() Condition {
 	if *c == nil {
@@ -123,6 +140,14 @@ func (c *Conditions) GetReadyCondition() Condition {
 	return c.getCondition(ConditionReady)
 }
 
+// GetFallbackCondition returns Condition of type Ready
+func (c *Conditions) GetFallbackCondition() Condition {
+	if *c == nil {
+		c = GetInitializedConditions()
+	}
+	return c.getCondition(ConditionFallback)
+}
+
 func (c Conditions) getCondition(conditionType ConditionType) Condition {
 	for i := range c {
 		if c[i].Type == conditionType {

api/v1alpha1/scaledobject_types.go

@@ -17,6 +17,7 @@ import (
 // +kubebuilder:printcolumn:name="Authentication",type="string",JSONPath=".spec.triggers[*].authenticationRef.name"
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status"
 // +kubebuilder:printcolumn:name="Active",type="string",JSONPath=".status.conditions[?(@.type==\"Active\")].status"
+// +kubebuilder:printcolumn:name="Fallback",type="string",JSONPath=".status.conditions[?(@.type==\"Fallback\")].status"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // ScaledObject is a specification for a ScaledObject resource
@@ -29,6 +30,25 @@ type ScaledObject struct {
 	Status ScaledObjectStatus `json:"status,omitempty"`
 }
 
+// HealthStatus is the status for a ScaledObject's health
+type HealthStatus struct {
+	// +optional
+	NumberOfFailures *int32 `json:"numberOfFailures,omitempty"`
+	// +optional
+	Status HealthStatusType `json:"status,omitempty"`
+}
+
+// HealthStatusType is an indication of whether the health status is happy or failing
+type HealthStatusType string
+
+const (
+	// HealthStatusHappy means the status of the health object is happy
+	HealthStatusHappy HealthStatusType = "Happy"
+
+	// HealthStatusFailing means the status of the health object is failing
+	HealthStatusFailing HealthStatusType = "Failing"
+)
+
 // ScaledObjectSpec is the spec for a ScaledObject resource
 type ScaledObjectSpec struct {
 	ScaleTargetRef *ScaleTarget `json:"scaleTargetRef"`
@@ -44,6 +64,14 @@ type ScaledObjectSpec struct {
 	Advanced *AdvancedConfig `json:"advanced,omitempty"`
 
 	Triggers []ScaleTriggers `json:"triggers"`
+	// +optional
+	Fallback *Fallback `json:"fallback,omitempty"`
+}
+
+// Fallback is the spec for fallback options
+type Fallback struct {
+	FailureThreshold int32 `json:"failureThreshold"`
+	Replicas         int32 `json:"replicas"`
 }
 
 // AdvancedConfig specifies advance scaling options
@@ -79,8 +107,12 @@ type ScaleTriggers struct {
 	Metadata map[string]string `json:"metadata"`
 	// +optional
 	AuthenticationRef *ScaledObjectAuthRef `json:"authenticationRef,omitempty"`
+	// +optional
+	FallbackReplicas *int32 `json:"fallback,omitempty"`
 }
 
+// +k8s:openapi-gen=true
+
 // ScaledObjectStatus is the status for a ScaledObject resource
 // +optional
 type ScaledObjectStatus struct {
@@ -98,6 +130,8 @@ type ScaledObjectStatus struct {
 	ResourceMetricNames []string `json:"resourceMetricNames,omitempty"`
 	// +optional
 	Conditions Conditions `json:"conditions,omitempty"`
+	// +optional
+	Health map[string]HealthStatus `json:"health,omitempty"`
 }
 
 // +kubebuilder:object:root=true

config/crd/bases/keda.sh_scaledjobs.yaml

@@ -6760,6 +6760,9 @@ spec:
                       required:
                       - name
                       type: object
+                    fallback:
+                      format: int32
+                      type: integer
                     metadata:
                       additionalProperties:
                         type: string

config/crd/bases/keda.sh_scaledobjects.yaml

@@ -43,6 +43,9 @@ spec:
     - jsonPath: .status.conditions[?(@.type=="Active")].status
       name: Active
       type: string
+    - jsonPath: .status.conditions[?(@.type=="Fallback")].status
+      name: Fallback
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -200,6 +203,19 @@ spec:
               cooldownPeriod:
                 format: int32
                 type: integer
+              fallback:
+                description: Fallback is the spec for fallback options
+                properties:
+                  failureThreshold:
+                    format: int32
+                    type: integer
+                  replicas:
+                    format: int32
+                    type: integer
+                required:
+                - failureThreshold
+                - replicas
+                type: object
               maxReplicaCount:
                 format: int32
                 type: integer
@@ -242,6 +258,9 @@ spec:
                       required:
                       - name
                       type: object
+                    fallback:
+                      format: int32
+                      type: integer
                     metadata:
                       additionalProperties:
                         type: string
@@ -290,6 +309,19 @@ spec:
                 items:
                   type: string
                 type: array
+              health:
+                additionalProperties:
+                  description: HealthStatus is the status for a ScaledObject's health
+                  properties:
+                    numberOfFailures:
+                      format: int32
+                      type: integer
+                    status:
+                      description: HealthStatusType is an indication of whether the
+                        health status is happy or failing
+                      type: string
+                  type: object
+                type: object
               lastActiveTime:
                 format: date-time
                 type: string

controllers/hpa.go

@@ -184,6 +184,9 @@ func (r *ScaledObjectReconciler) getScaledObjectMetricSpecs(logger logr.Logger,
 	status := scaledObject.Status.DeepCopy()
 	status.ExternalMetricNames = externalMetricNames
 	status.ResourceMetricNames = resourceMetricNames
+
+	updateHealthStatus(scaledObject, externalMetricNames, status)
+
 	err = kedacontrollerutil.UpdateScaledObjectStatus(r.Client, logger, scaledObject, status)
 	if err != nil {
 		logger.Error(err, "Error updating scaledObject status with used externalMetricNames")
@@ -193,6 +196,18 @@ func (r *ScaledObjectReconciler) getScaledObjectMetricSpecs(logger logr.Logger,
 	return scaledObjectMetricSpecs, nil
 }
 
+func updateHealthStatus(scaledObject *kedav1alpha1.ScaledObject, externalMetricNames []string, status *kedav1alpha1.ScaledObjectStatus) {
+	health := scaledObject.Status.Health
+	newHealth := make(map[string]kedav1alpha1.HealthStatus)
+	for _, metricName := range externalMetricNames {
+		entry, exists := health[metricName]
+		if exists {
+			newHealth[metricName] = entry
+		}
+	}
+	status.Health = newHealth
+}
+
 // checkMinK8sVersionforHPABehavior min version (k8s v1.18) for HPA Behavior
 func (r *ScaledObjectReconciler) checkMinK8sVersionforHPABehavior(logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject) {
 	if r.kubeVersion.MinorVersion < 18 {