Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provide explicitly pausing annotations of ScaledObjects at the current replica count #4809

Merged
merged 15 commits into from
Sep 25, 2023
Prev Previous commit
Next Next commit
Update
Signed-off-by: SpiritZhou <iammrzhouzhenghan@gmail.com>
SpiritZhou committed Jul 18, 2023
commit d86b9e7442f40e3f36d4e0ff649fd4ea15e87c3c
16 changes: 13 additions & 3 deletions apis/keda/v1alpha1/scaledobject_types.go
Original file line number Diff line number Diff line change
@@ -171,10 +171,20 @@ func (so *ScaledObject) GenerateIdentifier() string {
}

func (so *ScaledObject) HasPausedAnnotation() bool {
SpiritZhou marked this conversation as resolved.
Show resolved Hide resolved
_, pausedAnnotationFound := so.GetAnnotations()[PausedReplicasAnnotation]
return pausedAnnotationFound
_, pausedAnnotationFound := so.GetAnnotations()[PausedAnnotation]
_, pausedReplicasAnnotationFound := so.GetAnnotations()[PausedReplicasAnnotation]
return pausedAnnotationFound || pausedReplicasAnnotationFound
}

func (so *ScaledObject) NeedPaused() bool {
SpiritZhou marked this conversation as resolved.
Show resolved Hide resolved
return so.Status.PausedReplicaCount == nil
_, pausedAnnotationFound := so.GetAnnotations()[PausedAnnotation]
if pausedAnnotationFound {
return true
}

_, pausedReplicasAnnotationFound := so.GetAnnotations()[PausedReplicasAnnotation]
if pausedReplicasAnnotationFound {
return so.Status.PausedReplicaCount != nil
}
return false
}
1 change: 1 addition & 0 deletions controllers/keda/scaledobject_controller.go
Original file line number Diff line number Diff line change
@@ -122,6 +122,7 @@ func (r *ScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager, options cont
// so reconcile loop is not started on Status updates
For(&kedav1alpha1.ScaledObject{}, builder.WithPredicates(
predicate.Or(
kedacontrollerutil.PausedPredicate{},
kedacontrollerutil.PausedReplicasPredicate{},
kedacontrollerutil.ScaleObjectReadyConditionPredicate{},
predicate.GenerationChangedPredicate{},
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
//go:build e2e
// +build e2e

package pause_scaledobject_explicitly_test

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes"

. "github.com/kedacore/keda/v2/tests/helper"
)

// Load environment variables from .env file

const (
testName = "pause-scaledobject-explicitly-test"
)

var (
testNamespace = fmt.Sprintf("%s-ns", testName)
deploymentName = fmt.Sprintf("%s-deployment", testName)
monitoredDeploymentName = fmt.Sprintf("%s-monitored", testName)
scaledObjectName = fmt.Sprintf("%s-so", testName)
maxReplicaCount = 1
minReplicaCount = 0
testScaleOutWaitMin = 1
testPauseAtNWaitMin = 1
testScaleInWaitMin = 1
)

type templateData struct {
TestNamespace string
DeploymentName string
ScaledObjectName string
MonitoredDeploymentName string
}

const (
monitoredDeploymentTemplate = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{.MonitoredDeploymentName}}
namespace: {{.TestNamespace}}
labels:
app: {{.MonitoredDeploymentName}}
spec:
replicas: 0
selector:
matchLabels:
app: {{.MonitoredDeploymentName}}
template:
metadata:
labels:
app: {{.MonitoredDeploymentName}}
spec:
containers:
- name: {{.MonitoredDeploymentName}}
image: nginxinc/nginx-unprivileged:alpine-slim
`

deploymentTemplate = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{.DeploymentName}}
namespace: {{.TestNamespace}}
labels:
app: {{.DeploymentName}}
spec:
replicas: 0
selector:
matchLabels:
app: {{.DeploymentName}}
template:
metadata:
labels:
app: {{.DeploymentName}}
spec:
containers:
- name: {{.DeploymentName}}
image: nginxinc/nginx-unprivileged:alpine-slim
`

scaledObjectTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
minReplicaCount: 0
maxReplicaCount: 5
cooldownPeriod: 5
triggers:
- type: kubernetes-workload
metadata:
podSelector: 'app={{.MonitoredDeploymentName}}'
value: '1'
`
)

func TestScaler(t *testing.T) {
// setup
t.Log("--- setting up ---")

// Create kubernetes resources
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

CreateKubernetesResources(t, kc, testNamespace, data, templates)

// scaling to paused replica count
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
"replica count should be 0 after 1 minute")

// test scaling
testPauseWhenScaleOut(t, kc)
testScaleOut(t, kc)
testPauseWhenScaleIn(t, kc)
testScaleIn(t, kc)

// cleanup
DeleteKubernetesResources(t, testNamespace, data, templates)
}

func getTemplateData() (templateData, []Template) {
return templateData{
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
MonitoredDeploymentName: monitoredDeploymentName,
}, []Template{
{Name: "deploymentTemplate", Config: deploymentTemplate},
{Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate},
{Name: "scaledObjectAnnotatedTemplate", Config: scaledObjectTemplate},
}
}

func upsertScaledObjectAnnotation(t assert.TestingT) {
_, err := ExecuteCommand(fmt.Sprintf("kubectl annotate scaledobject/%s -n %s autoscaling.keda.sh/paused='true' --overwrite", scaledObjectName, testNamespace))
assert.NoErrorf(t, err, "cannot execute command - %s", err)
}

func removeScaledObjectAnnotation(t assert.TestingT) {
_, err := ExecuteCommand(fmt.Sprintf("kubectl annotate scaledobject/%s -n %s autoscaling.keda.sh/paused- --overwrite", scaledObjectName, testNamespace))
assert.NoErrorf(t, err, "cannot execute command - %s", err)
}

func testPauseWhenScaleOut(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing pausing at 0 ---")

upsertScaledObjectAnnotation(t)
KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 2, testNamespace)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 2, 60, testScaleOutWaitMin),
"monitoredDeploymentName replica count should be 2 after %d minute(s)", testScaleOutWaitMin)

AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 10)
}

func testScaleOut(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale out ---")

removeScaledObjectAnnotation(t)

KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 5, testNamespace)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 5, 60, testScaleOutWaitMin),
"monitoredDeploymentName replica count should be 5 after %d minute(s)", testScaleOutWaitMin)

assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 5, 60, testScaleOutWaitMin),
"replica count should be 5 after %d minute(s)", testScaleOutWaitMin)
}

func testPauseWhenScaleIn(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing pausing at N ---")

KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 5, testNamespace)

assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 5, 10, testPauseAtNWaitMin),
"replica count should be 5 after %d minute(s)", testPauseAtNWaitMin)

upsertScaledObjectAnnotation(t)

KubernetesScaleDeployment(t, kc, monitoredDeploymentName, 0, testNamespace)
SpiritZhou marked this conversation as resolved.
Show resolved Hide resolved

assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredDeploymentName, testNamespace, 0, 60, testPauseAtNWaitMin),
"monitoredDeploymentName replica count should be 0 after %d minute(s)", testPauseAtNWaitMin)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 5, 10, testPauseAtNWaitMin),
"replica count should be 5 after %d minute(s)", testPauseAtNWaitMin)

}

func testScaleIn(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale in ---")

removeScaledObjectAnnotation(t)
assert.Truef(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, testScaleInWaitMin),
"replica count should be 0 after %d minutes", testScaleInWaitMin)
}